fix receive

* Add changelog entry for v0.34.22

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* Prepare pending changelog for v0.34.23

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* Bump version to v0.34.22

Signed-off-by: Thane Thomson <connect@thanethomson.com>

* Reword changelog entry regarding block sync issue

Signed-off-by: Thane Thomson <connect@thanethomson.com>

Signed-off-by: Thane Thomson <connect@thanethomson.com>

.circleci/config.yml

@@ -72,30 +72,6 @@ jobs:
           paths:
             - "."
 
-  test_persistence:
-    executor: golang
-    steps:
-      - run_test:
-          script_path: test/persist/test_failure_indices.sh
-
-  test_p2p:
-    environment:
-      GOBIN: /home/circleci/.go_workspace/bin
-      GOPATH: /home/circleci/.go_workspace
-    machine:
-      image: circleci/classic:latest
-    parameters:
-      ipv:
-        type: integer
-        default: 4
-    steps:
-      - checkout
-      - run: mkdir -p $GOPATH/src/github.com/tendermint
-      - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint
-      - run: bash test/p2p/circleci.sh << parameters.ipv >>
-      - store_artifacts:
-          path: /home/circleci/project/test/p2p/logs
-
   deploy_docs:
     executor: docs
     steps:
@@ -142,115 +118,6 @@ jobs:
           paths:
             - "/go/pkg/mod"
 
-  build_artifacts:
-    executor: golang
-    parallelism: 5
-    steps:
-      - restore_cache:
-          name: "Restore source code cache"
-          keys:
-            - go-src-v1-{{ .Revision }}
-      - checkout
-      - restore_cache:
-          name: "Restore release dependencies cache"
-          keys:
-            - v2-release-deps-{{ checksum "go.sum" }}
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: Build artifact
-          command: |
-            # Setting CIRCLE_TAG because we do not tag the release ourselves.
-            source /tmp/workspace/release-version.source
-            if test ${CIRCLE_NODE_INDEX:-0} == 0 ;then export GOOS=linux GOARCH=amd64   && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
-            if test ${CIRCLE_NODE_INDEX:-0} == 1 ;then export GOOS=darwin GOARCH=amd64  && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
-            if test ${CIRCLE_NODE_INDEX:-0} == 2 ;then export GOOS=windows GOARCH=amd64 && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
-            if test ${CIRCLE_NODE_INDEX:-0} == 3 ;then export GOOS=linux GOARCH=arm     && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
-            if test ${CIRCLE_NODE_INDEX:-0} == 4 ;then export GOOS=linux GOARCH=arm64   && export OUTPUT=build/tendermint_${GOOS}_${GOARCH} && make build && python -u scripts/release_management/zip-file.py ;fi
-      - persist_to_workspace:
-          root: build
-          paths:
-            - "*.zip"
-            - "tendermint_linux_amd64"
-
-  release_artifacts:
-    executor: golang
-    steps:
-      - restore_cache:
-          name: "Restore source code cache"
-          keys:
-            - go-src-v1-{{ .Revision }}
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: "Deploy to GitHub"
-          command: |
-            # Setting CIRCLE_TAG because we do not tag the release ourselves.
-            source /tmp/workspace/release-version.source
-            echo "---"
-            ls -la /tmp/workspace/*.zip
-            echo "---"
-            python -u scripts/release_management/sha-files.py
-            echo "---"
-            cat /tmp/workspace/SHA256SUMS
-            echo "---"
-            export RELEASE_ID="`python -u scripts/release_management/github-draft.py`"
-            echo "Release ID: ${RELEASE_ID}"
-            #Todo: Parallelize uploads
-            export GOOS=linux GOARCH=amd64   && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
-            export GOOS=darwin GOARCH=amd64  && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
-            export GOOS=windows GOARCH=amd64 && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
-            export GOOS=linux GOARCH=arm     && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
-            export GOOS=linux GOARCH=arm64   && python -u scripts/release_management/github-upload.py --id "${RELEASE_ID}"
-            python -u scripts/release_management/github-upload.py --file "/tmp/workspace/SHA256SUMS" --id "${RELEASE_ID}"
-            python -u scripts/release_management/github-publish.py --id "${RELEASE_ID}"
-  release_docker:
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-      - checkout
-      - attach_workspace:
-          at: /tmp/workspace
-      - run:
-          name: "Deploy to Docker Hub"
-          command: |
-            # Setting CIRCLE_TAG because we do not tag the release ourselves.
-            source /tmp/workspace/release-version.source
-            cp /tmp/workspace/tendermint_linux_amd64 DOCKER/tendermint
-            docker build --label="tendermint" --tag="tendermint/tendermint:${CIRCLE_TAG}" --tag="tendermint/tendermint:latest" "DOCKER"
-            docker login -u "${DOCKERHUB_USER}" --password-stdin \<<< "${DOCKERHUB_PASS}"
-            docker push "tendermint/tendermint"
-            docker logout
-  reproducible_builds:
-    executor: golang
-    steps:
-      - attach_workspace:
-          at: /tmp/workspace
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      - run:
-          name: Build tendermint
-          no_output_timeout: 20m
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y ruby
-            bash -x ./scripts/gitian-build.sh all
-            for os in darwin linux windows; do
-              cp gitian-build-${os}/result/tendermint-${os}-res.yml .
-              cp gitian-build-${os}/build/out/tendermint-*.tar.gz .
-              rm -rf gitian-build-${os}/
-            done
-      - store_artifacts:
-          path: /go/src/github.com/tendermint/tendermint/tendermint-darwin-res.yml
-      - store_artifacts:
-          path: /go/src/github.com/tendermint/tendermint/tendermint-linux-res.yml
-      - store_artifacts:
-          path: /go/src/github.com/tendermint/tendermint/tendermint-windows-res.yml
-      - store_artifacts:
-          path: /go/src/github.com/tendermint/tendermint/tendermint-*.tar.gz
-
   # # Test RPC implementation against the swagger documented specs
   # contract_tests:
   #   working_directory: /home/circleci/.go_workspace/src/github.com/tendermint/tendermint
@@ -279,7 +146,7 @@ jobs:
 
 workflows:
   version: 2
-  test-suite:
+  docs:
     jobs:
       - deploy_docs:
           context: tendermint-docs
@@ -296,44 +163,6 @@ workflows:
             branches:
               only:
                 - docs-staging
-      - setup_dependencies
-      - test_persistence:
-          requires:
-            - setup_dependencies
-      - test_p2p
-      - test_p2p:
-          name: test_p2p_ipv6
-          ipv: 6
-      - reproducible_builds:
-          filters:
-            branches:
-              only:
-                - master
-                - /v[0-9]+\.[0-9]+/
       # - contract_tests:
       #     requires:
       #       - setup_dependencies
-
-  release:
-    jobs:
-      - prepare_build
-      - build_artifacts:
-          requires:
-            - prepare_build
-      - release_artifacts:
-          requires:
-            - prepare_build
-            - build_artifacts
-          filters:
-            branches:
-              only:
-                - /v[0-9]+\.[0-9]+/
-      - release_docker:
-          requires:
-            - prepare_build
-            - build_artifacts
-          filters:
-            branches:
-              only:
-                - /v[0-9]+\.[0-9]+/
-                - master

.dockerignore

@@ -0,0 +1,5 @@
+build
+test/e2e/build
+test/e2e/networks
+test/logs
+test/p2p/data

.github/CODEOWNERS

@@ -1,27 +1,11 @@
 # CODEOWNERS: https://help.github.com/articles/about-codeowners/
 
-# Everything goes through the following "global owners" by default. 
+# Everything goes through the following "global owners" by default.
 # Unless a later match takes precedence, these three will be
-# requested for review when someone opens a PR. 
+# requested for review when someone opens a PR.
 # Note that the last matching pattern takes precedence, so
 # global owners are only requested if there isn't a more specific
-# codeowner specified below. For this reason, the global codeowners 
+# codeowner specified below. For this reason, the global codeowners
 # are often repeated in package-level definitions.
-*       @ebuchman @erikgrinaker @melekes @tessr 
-
-# Overrides for tooling packages
-.circleci/ @marbar3778  @ebuchman @erikgrinaker @melekes @tessr 
-.github/ @marbar3778 @ebuchman @erikgrinaker @melekes @tessr 
-DOCKER/ @marbar3778 @ebuchman @erikgrinaker @melekes @tessr 
-
-# Overrides for core Tendermint packages 
-abci/  @marbar3778 @ebuchman @erikgrinaker @melekes @tessr 
-evidence/ @cmwaters @ebuchman @erikgrinaker @melekes @tessr
-light/ @cmwaters @melekes @ebuchman
-
-# Overrides for docs
-*.md @marbar3778 @ebuchman @erikgrinaker @melekes @tessr  
-docs/ @marbar3778 @ebuchman @erikgrinaker @melekes @tessr 
-
-
+*     @ebuchman @tendermint/tendermint-engineering
 

.github/ISSUE_TEMPLATE/proposal.md

@@ -0,0 +1,37 @@
+---
+name: Protocol Change Proposal
+about: Create a proposal to request a change to the protocol
+
+---
+
+<!-- < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < ☺ 
+v                            ✰  Thanks for opening an issue! ✰    
+v    Before smashing the submit button please review the template.
+v    Word of caution: Under-specified proposals may be rejected summarily 
+☺ > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > >  -->
+
+# Protocol Change Proposal
+
+## Summary
+
+<!-- Short, concise description of the proposed change -->
+
+## Problem Definition
+
+<!-- Why do we need this change? 
+What problems may be addressed by introducing this change?
+What benefits does Tendermint stand to gain by including this change?
+Are there any disadvantages of including this change? -->
+
+## Proposal
+
+<!-- Detailed description of requirements of implementation -->
+
+____
+
+#### For Admin Use
+
+- [ ] Not duplicate issue
+- [ ] Appropriate labels applied
+- [ ] Appropriate contributors tagged
+- [ ] Contributor assigned/self-assigned

.github/dependabot.yml

@@ -1,28 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: github-actions
-    directory: "/"
-    schedule:
-      interval: daily
-      time: "11:00"
-    open-pull-requests-limit: 10
-  - package-ecosystem: npm
-    directory: "/docs"
-    schedule:
-      interval: daily
-      time: "11:00"
-    open-pull-requests-limit: 10
-    reviewers:
-      - fadeev
-  - package-ecosystem: gomod
-    directory: "/"
-    schedule:
-      interval: daily
-      time: "11:00"
-    open-pull-requests-limit: 10
-    reviewers:
-      - melekes
-      - tessr
-      - erikgrinaker
-    labels:
-      - T:dependencies

.github/mergify.yml

@@ -0,0 +1,19 @@
+queue_rules:
+  - name: default
+    conditions:
+      - base=v0.34.x
+      - label=S:automerge
+
+pull_request_rules:
+  - name: Automerge to v0.34.x
+    conditions:
+      - base=v0.34.x
+      - label=S:automerge
+    actions:
+      queue:
+        method: squash
+        name: default
+        commit_message_template: |
+          {{ title }} (#{{ number }})
+
+          {{ body }}

.github/workflows/check-generated.yml

@@ -0,0 +1,75 @@
+# Verify that generated code is up-to-date.
+#
+# Note that we run these checks regardless whether the input files have
+# changed, because generated code can change in response to toolchain updates
+# even if no files in the repository are modified.
+name: Check generated code
+on:
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  check-mocks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - uses: actions/checkout@v3
+
+      - name: "Check generated mocks"
+        run: |
+          set -euo pipefail
+
+          readonly MOCKERY=2.12.3  # N.B. no leading "v"
+          curl -sL "https://github.com/vektra/mockery/releases/download/v${MOCKERY}/mockery_${MOCKERY}_Linux_x86_64.tar.gz" | tar -C /usr/local/bin -xzf -
+          make mockery 2>/dev/null
+
+          if ! git diff --stat --exit-code ; then
+            echo ">> ERROR:"
+            echo ">>"
+            echo ">> Generated mocks require update (either Mockery or source files may have changed)."
+            echo ">> Ensure your tools are up-to-date, re-run 'make mockery' and update this PR."
+            echo ">>"
+            exit 1
+          fi
+
+  check-proto:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1  # we need a .git directory to run git diff
+
+      - name: "Check protobuf generated code"
+        run: |
+          set -euo pipefail
+
+          # Install buf and gogo tools, so that differences that arise from
+          # toolchain differences are also caught.
+          readonly tools="$(mktemp -d)"
+          export PATH="${PATH}:${tools}/bin"
+          export GOBIN="${tools}/bin"
+
+          go install github.com/bufbuild/buf/cmd/buf
+          go install github.com/gogo/protobuf/protoc-gen-gogofaster@latest
+
+          make proto-gen
+
+          if ! git diff --stat --exit-code ; then
+            echo ">> ERROR:"
+            echo ">>"
+            echo ">> Protobuf generated code requires update (either tools or .proto files may have changed)."
+            echo ">> Ensure your tools are up-to-date, re-run 'make proto-gen' and update this PR."
+            echo ">>"
+            exit 1
+          fi

.github/workflows/coverage.yml

@@ -10,129 +10,114 @@ jobs:
   split-test-files:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Create a file with all the pkgs
         run: go list ./... > pkgs.txt
       - name: Split pkgs into 4 files
-        run: split -n l/4 --additional-suffix=.txt ./pkgs.txt
+        run: split -d -n l/4 pkgs.txt pkgs.txt.part.
       # cache multiple
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
-          name: "${{ github.sha }}-aa"
-          path: ./xaa.txt
-      - uses: actions/upload-artifact@v2
+          name: "${{ github.sha }}-00"
+          path: ./pkgs.txt.part.00
+      - uses: actions/upload-artifact@v3
         with:
-          name: "${{ github.sha }}-ab"
-          path: ./xab.txt
-      - uses: actions/upload-artifact@v2
+          name: "${{ github.sha }}-01"
+          path: ./pkgs.txt.part.01
+      - uses: actions/upload-artifact@v3
         with:
-          name: "${{ github.sha }}-ac"
-          path: ./xac.txt
-      - uses: actions/upload-artifact@v2
+          name: "${{ github.sha }}-02"
+          path: ./pkgs.txt.part.02
+      - uses: actions/upload-artifact@v3
         with:
-          name: "${{ github.sha }}-ad"
-          path: ./xad.txt
+          name: "${{ github.sha }}-03"
+          path: ./pkgs.txt.part.03
 
-  test-coverage-part-1:
+  build-linux:
+    name: Build
     runs-on: ubuntu-latest
-    needs: split-test-files
+    strategy:
+      fail-fast: false
+      matrix:
+        goarch: ["arm", "amd64"]
+    timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/setup-go@v3
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - uses: actions/download-artifact@v2
+          go-version: "1.18"
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
         with:
-          name: "${{ github.sha }}-aa"
-        if: "env.GIT_DIFF != ''"
-      - name: test & coverage report creation
-        run: |
-          cat xaa.txt | xargs go test -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic
-        if: "env.GIT_DIFF != ''"
-      - uses: codecov/codecov-action@v1.0.13
-        with:
-          file: ./coverage.txt
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - name: install
+        run: GOOS=linux GOARCH=${{ matrix.goarch }} make build
         if: "env.GIT_DIFF != ''"
 
-  test-coverage-part-2:
+  tests:
     runs-on: ubuntu-latest
     needs: split-test-files
+    strategy:
+      fail-fast: false
+      matrix:
+        part: ["00", "01", "02", "03"]
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/setup-go@v3
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - uses: actions/download-artifact@v2
+          go-version: "1.18"
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
         with:
-          name: "${{ github.sha }}-ab"
-        if: "env.GIT_DIFF != ''"
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - uses: actions/download-artifact@v3
+        with:
+          name: "${{ github.sha }}-${{ matrix.part }}"
+        if: env.GIT_DIFF
       - name: test & coverage report creation
         run: |
-          cat xab.txt | xargs go test -mod=readonly -timeout 5m -race -coverprofile=coverage.txt -covermode=atomic
-        if: "env.GIT_DIFF != ''"
-      - uses: codecov/codecov-action@v1.0.13
+          cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic
+        if: env.GIT_DIFF
+      - uses: actions/upload-artifact@v3
         with:
-          file: ./coverage.txt
-        if: "env.GIT_DIFF != ''"
+          name: "${{ github.sha }}-${{ matrix.part }}-coverage"
+          path: ./${{ matrix.part }}profile.out
 
-  test-coverage-part-3:
+  upload-coverage-report:
     runs-on: ubuntu-latest
-    needs: split-test-files
+    needs: tests
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - uses: actions/download-artifact@v2
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - uses: actions/download-artifact@v3
         with:
-          name: "${{ github.sha }}-ac"
-        if: "env.GIT_DIFF != ''"
-      - name: test & coverage report creation
-        run: |
-          cat xac.txt | xargs go test -mod=readonly -timeout 10m -race -coverprofile=coverage.txt -covermode=atomic
-        if: "env.GIT_DIFF != ''"
-      - uses: codecov/codecov-action@v1.0.13
+          name: "${{ github.sha }}-00-coverage"
+        if: env.GIT_DIFF
+      - uses: actions/download-artifact@v3
+        with:
+          name: "${{ github.sha }}-01-coverage"
+        if: env.GIT_DIFF
+      - uses: actions/download-artifact@v3
+        with:
+          name: "${{ github.sha }}-02-coverage"
+        if: env.GIT_DIFF
+      - uses: actions/download-artifact@v3
+        with:
+          name: "${{ github.sha }}-03-coverage"
+        if: env.GIT_DIFF
+      - run: |
+          cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt
+        if: env.GIT_DIFF
+      - uses: codecov/codecov-action@v3
         with:
           file: ./coverage.txt
-        if: "env.GIT_DIFF != ''"
-
-  test-coverage-part-4:
-    runs-on: ubuntu-latest
-    needs: split-test-files
-    steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
-        with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - uses: actions/download-artifact@v2
-        with:
-          name: "${{ github.sha }}-ad"
-        if: "env.GIT_DIFF != ''"
-      - name: test & coverage report creation
-        run: |
-          cat xad.txt | xargs go test -mod=readonly -timeout 5m -race -coverprofile=coverage.txt -covermode=atomic
-        if: "env.GIT_DIFF != ''"
-      - uses: codecov/codecov-action@v1.0.13
-        with:
-          file: ./coverage.txt
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF

.github/workflows/docker.yml

@@ -0,0 +1,59 @@
+name: Build & Push
+#  Build & Push rebuilds the tendermint docker image on every push to master and creation of tags
+# and pushes the image to https://hub.docker.com/r/interchainio/simapp/tags
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+" # Push events to matching v*, i.e. v1.0, v20.15.10
+      - "v[0-9]+.[0-9]+.[0-9]+-rc*" # Push events to matching v*, i.e. v1.0-rc1, v20.15.10-rc5
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=tendermint/tendermint
+          VERSION=noop
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+            if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then
+              VERSION=latest
+            fi
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION}"
+          if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:${VERSION}"
+          fi
+          echo ::set-output name=tags::${TAGS}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@master
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to DockerHub
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Publish to Docker Hub
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./DOCKER/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}

.github/workflows/e2e-manual.yml

@@ -0,0 +1,35 @@
+# Manually run randomly generated E2E testnets (as nightly).
+name: e2e-manual
+on:
+  workflow_dispatch:
+
+jobs:
+  e2e-nightly-test:
+    # Run parallel jobs for the listed testnet groups (must match the
+    # ./build/generator -g flag)
+    strategy:
+      fail-fast: false
+      matrix:
+        group: ['00', '01', '02', '03']
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - uses: actions/checkout@v3
+
+      - name: Build
+        working-directory: test/e2e
+        # Run make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker generator runner tests
+
+      - name: Generate testnets
+        working-directory: test/e2e
+        # When changing -g, also change the matrix groups above
+        run: ./build/generator -g 4 -d networks/nightly/
+
+      - name: Run ${{ matrix.p2p }} p2p testnets
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml

.github/workflows/e2e-nightly-34x.yml

@@ -0,0 +1,76 @@
+# Runs randomly generated E2E testnets nightly
+# on the 0.34.x release branch
+
+# !! If you change something in this file, you probably want
+# to update the e2e-nightly-master workflow as well!
+
+name: e2e-nightly-34x
+on:
+  workflow_dispatch: # allow running workflow manually, in theory
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  e2e-nightly-test:
+    # Run parallel jobs for the listed testnet groups (must match the
+    # ./build/generator -g flag)
+    strategy:
+      fail-fast: false
+      matrix:
+        group: ['00', '01', '02', '03']
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - uses: actions/checkout@v3
+        with:
+          ref: 'v0.34.x'
+
+      - name: Build
+        working-directory: test/e2e
+        # Run make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker generator runner
+
+      - name: Generate testnets
+        working-directory: test/e2e
+        # When changing -g, also change the matrix groups above
+        run: ./build/generator -g 4 -d networks/nightly
+
+      - name: Run testnets in group ${{ matrix.group }}
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml
+
+  e2e-nightly-fail:
+    needs: e2e-nightly-test
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack on failure
+        uses: rtCamp/action-slack-notify@f565a63638bd3615e76249bffab00fcb9dab90f7
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_CHANNEL: tendermint-internal
+          SLACK_USERNAME: Nightly E2E Tests
+          SLACK_ICON_EMOJI: ':skull:'
+          SLACK_COLOR: danger
+          SLACK_MESSAGE: Nightly E2E tests failed on v0.34.x
+          SLACK_FOOTER: ''
+
+  e2e-nightly-success: # may turn this off once they seem to pass consistently
+    needs: e2e-nightly-test
+    if: ${{ success() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack on success
+        uses: rtCamp/action-slack-notify@f565a63638bd3615e76249bffab00fcb9dab90f7
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_CHANNEL: tendermint-internal
+          SLACK_USERNAME: Nightly E2E Tests
+          SLACK_ICON_EMOJI: ':white_check_mark:'
+          SLACK_COLOR: good
+          SLACK_MESSAGE: Nightly E2E tests passed on v0.34.x
+          SLACK_FOOTER: ''

.github/workflows/e2e-nightly-master.yml

@@ -0,0 +1,73 @@
+# Runs randomly generated E2E testnets nightly on master
+
+# !! If you change something in this file, you probably want
+# to update the e2e-nightly-34x workflow as well!
+
+name: e2e-nightly-master
+on:
+  workflow_dispatch: # allow running workflow manually
+  schedule:
+    - cron: '0 2 * * *'
+
+jobs:
+  e2e-nightly-test-2:
+    # Run parallel jobs for the listed testnet groups (must match the
+    # ./build/generator -g flag)
+    strategy:
+      fail-fast: false
+      matrix:
+        group: ['00', '01', '02', '03']
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - uses: actions/checkout@v3
+
+      - name: Build
+        working-directory: test/e2e
+        # Run make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker generator runner
+
+      - name: Generate testnets
+        working-directory: test/e2e
+        # When changing -g, also change the matrix groups above
+        run: ./build/generator -g 4 -d networks/nightly
+
+      - name: Run testnets in group ${{ matrix.group }}
+        working-directory: test/e2e
+        run: ./run-multiple.sh networks/nightly/*-group${{ matrix.group }}-*.toml
+
+  e2e-nightly-fail-2:
+    needs: e2e-nightly-test-2
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack on failure
+        uses: rtCamp/action-slack-notify@f565a63638bd3615e76249bffab00fcb9dab90f7
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_CHANNEL: tendermint-internal
+          SLACK_USERNAME: Nightly E2E Tests
+          SLACK_ICON_EMOJI: ':skull:'
+          SLACK_COLOR: danger
+          SLACK_MESSAGE: Nightly E2E tests failed on master
+          SLACK_FOOTER: ''
+
+  e2e-nightly-success: # may turn this off once they seem to pass consistently
+    needs: e2e-nightly-test-2
+    if: ${{ success() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack on success
+        uses: rtCamp/action-slack-notify@f565a63638bd3615e76249bffab00fcb9dab90f7
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_CHANNEL: tendermint-internal
+          SLACK_USERNAME: Nightly E2E Tests
+          SLACK_ICON_EMOJI: ':white_check_mark:'
+          SLACK_COLOR: good
+          SLACK_MESSAGE: Nightly E2E tests passed on master
+          SLACK_FOOTER: ''

.github/workflows/e2e.yml

@@ -0,0 +1,41 @@
+name: e2e
+# Runs the CI end-to-end test network on all pushes to master or release branches
+# and every pull request, but only if any Go files have been changed.
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - release/**
+
+jobs:
+  e2e-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+
+      - name: Build
+        working-directory: test/e2e
+        # Run two make jobs in parallel, since we can't run steps in parallel.
+        run: make -j2 docker runner
+        if: "env.GIT_DIFF != ''"
+
+      - name: Run CI testnet
+        working-directory: test/e2e
+        run: ./build/runner -f networks/ci.toml
+        if: "env.GIT_DIFF != ''"
+
+      - name: Emit logs on failure
+        if: ${{ failure() }}
+        working-directory: test/e2e
+        run: ./build/runner -f networks/ci.toml logs

.github/workflows/fuzz-nightly.yml

@@ -0,0 +1,83 @@
+# Runs fuzzing nightly.
+name: fuzz-nightly
+on:
+  workflow_dispatch: # allow running workflow manually
+  schedule:
+    - cron: '0 3 * * *'
+
+jobs:
+  fuzz-nightly-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - uses: actions/checkout@v3
+
+      - name: Install go-fuzz
+        working-directory: test/fuzz
+        run: go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
+
+      - name: Fuzz mempool
+        working-directory: test/fuzz
+        run: timeout -s SIGINT --preserve-status 10m make fuzz-mempool
+        continue-on-error: true
+
+      - name: Fuzz p2p-addrbook
+        working-directory: test/fuzz
+        run: timeout -s SIGINT --preserve-status 10m make fuzz-p2p-addrbook
+        continue-on-error: true
+
+      - name: Fuzz p2p-pex
+        working-directory: test/fuzz
+        run: timeout -s SIGINT --preserve-status 10m make fuzz-p2p-pex
+        continue-on-error: true
+
+      - name: Fuzz p2p-sc
+        working-directory: test/fuzz
+        run: timeout -s SIGINT --preserve-status 10m make fuzz-p2p-sc
+        continue-on-error: true
+
+      - name: Fuzz p2p-rpc-server
+        working-directory: test/fuzz
+        run: timeout -s SIGINT --preserve-status 10m make fuzz-rpc-server
+        continue-on-error: true
+
+      - name: Archive crashers
+        uses: actions/upload-artifact@v3
+        with:
+          name: crashers
+          path: test/fuzz/**/crashers
+          retention-days: 1
+
+      - name: Archive suppressions
+        uses: actions/upload-artifact@v3
+        with:
+          name: suppressions
+          path: test/fuzz/**/suppressions
+          retention-days: 1
+
+      - name: Set crashers count
+        working-directory: test/fuzz
+        run: echo "::set-output name=count::$(find . -type d -name 'crashers' | xargs -I % sh -c 'ls % | wc -l' | awk '{total += $1} END {print total}')"
+        id: set-crashers-count
+
+    outputs:
+      crashers-count: ${{ steps.set-crashers-count.outputs.count }}
+
+  fuzz-nightly-fail:
+    needs: fuzz-nightly-test
+    if: ${{ needs.fuzz-nightly-test.outputs.crashers-count != 0 }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Notify Slack if any crashers
+        uses: rtCamp/action-slack-notify@f565a63638bd3615e76249bffab00fcb9dab90f7
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+          SLACK_CHANNEL: tendermint-internal
+          SLACK_USERNAME: Nightly Fuzz Tests
+          SLACK_ICON_EMOJI: ':firecracker:'
+          SLACK_COLOR: danger
+          SLACK_MESSAGE: Crashers found in Nightly Fuzz tests
+          SLACK_FOOTER: ''

.github/workflows/linkchecker.yml

@@ -1,12 +1,12 @@
 name: Check Markdown links
-on: 
+on:
   schedule:
     - cron: '* */24 * * *'
 jobs:
   markdown-link-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
-      - uses: gaurav-nelson/github-action-markdown-link-check@1.0.7
+      - uses: actions/checkout@v3
+      - uses: creachadair/github-action-markdown-link-check@master
         with:
           folder-path: "docs"

.github/workflows/lint.yml

@@ -11,19 +11,22 @@ jobs:
   golangci:
     name: golangci-lint
     runs-on: ubuntu-latest
-    timeout-minutes: 4
+    timeout-minutes: 8
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-      - uses: golangci/golangci-lint-action@v2.2.0
+          go-version: '1.18'
+      - uses: technote-space/get-diff-action@v6
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - uses: golangci/golangci-lint-action@v3
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.30
+          version: v1.47.3
           args: --timeout 10m
           github-token: ${{ secrets.github_token }}
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF

.github/workflows/linter.yml

@@ -11,6 +11,7 @@ on:
     branches: [master]
     paths:
       - "**.md"
+      - "**.yml"
 
 jobs:
   build:
@@ -18,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Lint Code Base
         uses: docker://github/super-linter:v3
         env:
@@ -27,6 +28,5 @@ jobs:
           DEFAULT_BRANCH: master
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_MD: true
-          MARKDOWN_CONFIG_FILE: .markdownlint.yml
-          VALIDATE_OPAENAPI: true
+          VALIDATE_OPENAPI: true
           VALIDATE_YAML: true

.github/workflows/net.yml

@@ -1,28 +0,0 @@
-name: Net
-# Net creates a 4 node test network with docker-compose
-# This workflow is run on every pull request, if a *{.go, .mod, .sum} file has been modified, and push to master and release/** branches
-on:
-  pull_request:
-    paths:
-      - "**.go"
-      - "**.mod"
-      - "**.sum"
-  push:
-    branches:
-      - master
-      - release/**
-
-jobs:
-  net-short:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - uses: actions/checkout@v2
-      - name: 10 Blocks
-        run: |
-          set -x
-          docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint golang make build-linux
-          make localnet-start &
-          ./scripts/localnet-blocks-test.sh 40 5 10 localhost
-
-# Decide if we want to run longer lived testnets

.github/workflows/proto-lint.yml

@@ -0,0 +1,21 @@
+name: Protobuf Lint
+on:
+  pull_request:
+    paths:
+      - 'proto/**'
+  push:
+    branches:
+      - v0.34.x
+    paths:
+      - 'proto/**'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v3
+      - uses: bufbuild/buf-setup-action@v1.8.0
+      - uses: bufbuild/buf-lint-action@v1
+        with:
+          input: 'proto'

.github/workflows/proto.yml

@@ -1,22 +0,0 @@
-name: Protobuf
-# Protobuf runs buf (https://buf.build/) lint and check-breakage
-# This workflow is only run when a .proto file has been modified
-on:
-  pull_request:
-    paths:
-      - "**.proto"
-jobs:
-  proto-lint:
-    runs-on: ubuntu-latest
-    timeout-minutes: 4
-    steps:
-      - uses: actions/checkout@master
-      - name: lint
-        run: make proto-lint
-  proto-breakage:
-    runs-on: ubuntu-latest
-    timeout-minutes: 4
-    steps:
-      - uses: actions/checkout@master
-      - name: check-breakage
-        run: make proto-check-breaking-ci

.github/workflows/release.yml

@@ -0,0 +1,29 @@
+name: "Release"
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+" # Push events to matching v*, i.e. v1.0, v20.15.10
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.18'
+
+      - run: echo https://github.com/tendermint/tendermint/blob/${GITHUB_REF#refs/tags/}/CHANGELOG.md#${GITHUB_REF#refs/tags/} > ../release_notes.md 
+
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v3
+        with:
+          version: latest
+          args: release --rm-dist --release-notes=../release_notes.md
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v6
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           stale-pr-message: "This pull request has been automatically marked as stale because it has not had

.github/workflows/tests.yml

@@ -23,132 +23,124 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/setup-go@v3
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - name: Set GOBIN
-        run: |
-          echo "::add-path::$(go env GOPATH)/bin"
+          go-version: "1.18"
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
       - name: install
         run: make install install_abci
         if: "env.GIT_DIFF != ''"
-      - uses: actions/cache@v2.1.1
+      - uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF
       # Cache binaries for use by other jobs
-      - uses: actions/cache@v2.1.1
+      - uses: actions/cache@v3
         with:
           path: ~/go/bin
           key: ${{ runner.os }}-${{ github.sha }}-tm-binary
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF
 
   test_abci_apps:
     runs-on: ubuntu-latest
-    needs: Build
+    needs: build
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/setup-go@v3
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - name: Set GOBIN
-        run: |
-          echo "::add-path::$(go env GOPATH)/bin"
-      - uses: actions/cache@v2.1.1
+          go-version: "^1.18"
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-        if: "env.GIT_DIFF != ''"
-      - uses: actions/cache@v2.1.1
+        if: env.GIT_DIFF
+      - uses: actions/cache@v3
         with:
           path: ~/go/bin
           key: ${{ runner.os }}-${{ github.sha }}-tm-binary
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF
       - name: test_abci_apps
         run: abci/tests/test_app/test.sh
         shell: bash
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF
 
   test_abci_cli:
     runs-on: ubuntu-latest
-    needs: Build
+    needs: build
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/setup-go@v3
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - name: Set GOBIN
-        run: |
-          echo "::add-path::$(go env GOPATH)/bin"
-      - uses: actions/cache@v2.1.1
+          go-version: "^1.18"
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-        if: "env.GIT_DIFF != ''"
-      - uses: actions/cache@v2.1.1
+        if: env.GIT_DIFF
+      - uses: actions/cache@v3
         with:
           path: ~/go/bin
           key: ${{ runner.os }}-${{ github.sha }}-tm-binary
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF
       - run: abci/tests/test_cli/test.sh
         shell: bash
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF
 
   test_apps:
     runs-on: ubuntu-latest
-    needs: Build
+    needs: build
     timeout-minutes: 5
     steps:
-      - uses: actions/checkout@v2
-      - uses: technote-space/get-diff-action@v3
+      - uses: actions/setup-go@v3
         with:
-          SUFFIX_FILTER: |
-            .go
-            .mod
-            .sum
-          SET_ENV_NAME_INSERTIONS: 1
-          SET_ENV_NAME_LINES: 1
-      - name: Set GOBIN
-        run: |
-          echo "::add-path::$(go env GOPATH)/bin"
-      - uses: actions/cache@v2.1.1
+          go-version: "1.18"
+      - uses: actions/checkout@v3
+      - uses: technote-space/get-diff-action@v6
+        with:
+          PATTERNS: |
+            **/**.go
+            go.mod
+            go.sum
+      - uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-        if: "env.GIT_DIFF != ''"
-      - uses: actions/cache@v2.1.1
+        if: env.GIT_DIFF
+      - uses: actions/cache@v3
         with:
           path: ~/go/bin
           key: ${{ runner.os }}-${{ github.sha }}-tm-binary
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF
       - name: test_apps
         run: test/app/test.sh
         shell: bash
-        if: "env.GIT_DIFF != ''"
+        if: env.GIT_DIFF

.gitignore

@@ -10,6 +10,9 @@ remote_dump
 .revision
 vendor
 .vagrant
+test/e2e/build
+test/maverick/maverick
+test/e2e/networks/*/
 test/p2p/data/
 test/logs
 coverage.txt
@@ -19,6 +22,7 @@ docs/.vuepress/dist
 *.log
 abci-cli
 docs/node_modules/
+docs/.vuepress/public/rpc
 index.html.md
 
 scripts/wal2json/wal2json
@@ -42,5 +46,24 @@ addrbook.json
 terraform.tfstate
 terraform.tfstate.backup
 terraform.tfstate.d
-
 profile\.out
+test/app/grpc_client
+test/loadtime/build
+test/e2e/build
+test/e2e/networks/*/
+test/logs
+test/maverick/maverick
+test/p2p/data/
+vendor
+test/fuzz/**/corpus
+test/fuzz/**/crashers
+test/fuzz/**/suppressions
+test/fuzz/**/*.zip
+*.aux
+*.bbl
+*.blg
+*.pdf
+*.gz
+*.dvi
+# Python virtual environments
+.venv

.golangci.yml

@@ -1,43 +1,48 @@
 linters:
   enable:
+    - asciicheck
     - bodyclose
-    - deadcode
+    # - deadcode
     - depguard
     - dogsled
     - dupl
     - errcheck
+    - exportloopref
     # - funlen
     # - gochecknoglobals
     # - gochecknoinits
+    # - gocognit
     - goconst
-    - gocritic
+    # - gocritic
     # - gocyclo
     # - godox
     - gofmt
     - goimports
-    - golint
+    - revive
     - gosec
     - gosimple
     - govet
     - ineffassign
     # - interfacer
     - lll
-    - misspell
     # - maligned
+    # - misspell
     - nakedret
+    # - nolintlint
     - prealloc
-    - scopelint
     - staticcheck
-    - structcheck
+    # - structcheck // to be fixed by golangci-lint
     - stylecheck
-    - typecheck
+    # - typecheck
     - unconvert
     # - unparam
-    - unused
+    # - unused
     - varcheck
     # - whitespace
     # - wsl
-    # - gocognit
+  disable:
+    - unused
+    - deadcode
     - nolintlint
 
 issues:
@@ -57,5 +62,11 @@ linters-settings:
     suggest-new: true
   # govet:
   #   check-shadowing: true
-  golint:
+  revive:
     min-confidence: 0
+  misspell:
+    locale: US
+    ignore-words:
+      - behaviour
+
+

.goreleaser.yml

@@ -0,0 +1,37 @@
+project_name: tendermint
+
+env:
+  # Require use of Go modules.
+  - GO111MODULE=on
+
+builds:
+  - id: "Tendermint"
+    main: ./cmd/tendermint/main.go
+    ldflags:
+      - -s -w -X github.com/tendermint/tendermint/version.TMCoreSemVer={{ .Version }}
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - darwin
+      - linux
+      - windows
+    goarch:
+      - amd64
+      - arm
+      - arm64
+
+checksum:
+  name_template: SHA256SUMS-{{.Version}}.txt
+  algorithm: sha256
+
+release:
+  prerelease: auto
+  name_template: "{{.Version}}"
+
+archives:
+  - files:
+      - LICENSE
+      - README.md
+      - UPGRADING.md
+      - SECURITY.md
+      - CHANGELOG.md

.markdownlint.yml

@@ -1,8 +1,11 @@
-default: true,
-MD007: { "indent": 4 }
+default: true
+MD001: false
+MD007: {indent: 4}
 MD013: false
-MD024: { siblings_only: true }
+MD024: {siblings_only: true}
 MD025: false
-MD033: { no-inline-html: false }
-no-hard-tabs: false
-whitespace: false
+MD033: false
+MD036: false
+MD010: false
+MD012: false
+MD028: false

.markdownlintignore

@@ -1,5 +1,6 @@
 docs/node_modules
 CHANGELOG.md
 docs/architecture/*
+crypto/secp256k1/**
 scripts/*
 .github

.vscode/settings.json

@@ -5,4 +5,5 @@
       "--proto_path=${workspaceRoot}/third_party/proto"
     ]
   }
+  
 }

CHANGELOG_PENDING.md

@@ -1,10 +1,6 @@
 # Unreleased Changes
 
-## v0.34.0-rc5
-
-Special thanks to external contributors on this release:
-
-Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint).
+## v0.34.23
 
 ### BREAKING CHANGES
 
@@ -23,4 +19,3 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 ### IMPROVEMENTS
 
 ### BUG FIXES
-

CONTRIBUTING.md

@@ -26,7 +26,7 @@ will indicate their support with a heartfelt emoji.
 
 If the issue would benefit from thorough discussion, maintainers may
 request that you create a [Request For
-Comment](https://github.com/tendermint/spec/tree/master/rfc). Discussion
+Comment](https://github.com/tendermint/tendermint/tree/main/rfc). Discussion
 at the RFC stage will build collective understanding of the dimensions
 of the problems and help structure conversations around trade-offs.
 
@@ -108,24 +108,7 @@ We use [Protocol Buffers](https://developers.google.com/protocol-buffers) along
 
 For linting and checking breaking changes, we use [buf](https://buf.build/). If you would like to run linting and check if the changes you have made are breaking then you will need to have docker running locally. Then the linting cmd will be `make proto-lint` and the breaking changes check will be `make proto-check-breaking`.
 
-There are two ways to generate your proto stubs.
-
-1. Use Docker, pull an image that will generate your proto stubs with no need to install anything. `make proto-gen-docker`
-2. Run `make proto-gen` after installing `protoc` and gogoproto, you can do this by running `make protobuf`.
-
-### Installation Instructions
-
-To install `protoc`, download an appropriate release (<https://github.com/protocolbuffers/protobuf>) and then move the provided binaries into your PATH (follow instructions in README included with the download).
-
-To install `gogoproto`, do the following:
-
-```sh
-go get github.com/gogo/protobuf/gogoproto
-cd $GOPATH/pkg/mod/github.com/gogo/protobuf@v1.3.1 # or wherever go get installs things
-make install
-```
-
-You should now be able to run `make proto-gen` from inside the root Tendermint directory to generate new files from proto files.
+We use [Docker](https://www.docker.com/) to generate the protobuf stubs. To generate the stubs yourself, make sure docker is running then run `make proto-gen`.
 
 ## Vagrant
 
@@ -246,22 +229,27 @@ Each PR should have one commit once it lands on `master`; this can be accomplish
 
 #### Major Release
 
-1. start on `master`
-2. run integration tests (see `test_integrations` in Makefile)
-3. prepare release in a pull request against `master` (to be squash merged):
-   - copy `CHANGELOG_PENDING.md` to top of `CHANGELOG.md`
-   - run `python ./scripts/linkify_changelog.py CHANGELOG.md` to add links for
+1. Start on `master`
+2. Run integration tests (see `test_integrations` in Makefile)
+3. Prepare release in a pull request against `master` (to be squash merged):
+   - Copy `CHANGELOG_PENDING.md` to top of `CHANGELOG.md`; if this release
+      had release candidates, squash all the RC updates into one
+   - Run `python ./scripts/linkify_changelog.py CHANGELOG.md` to add links for
      all issues
    - run `bash ./scripts/authors.sh` to get a list of authors since the latest
      release, and add the github aliases of external contributors to the top of
      the changelog. To lookup an alias from an email, try `bash ./scripts/authors.sh <email>`
-   - reset the `CHANGELOG_PENDING.md`
-   - bump Tendermint version in `version.go`
-   - bump P2P and block protocol versions in  `version.go`, if necessary
-   - bump ABCI protocol version in `version.go`, if necessary
-   - make sure all significant breaking changes are covered in `UPGRADING.md`
-4. push your changes with prepared release details to `vX.X` (this will trigger the release `vX.X.0`)
-5. merge back to master (don't squash merge!)
+   - Reset the `CHANGELOG_PENDING.md`
+   - Bump TMVersionDefault version in  `version.go`
+   - Bump P2P and block protocol versions in  `version.go`, if necessary
+   - Bump ABCI protocol version in `version.go`, if necessary
+   - Make sure all significant breaking changes are covered in `UPGRADING.md`
+   - Add any release notes you would like to be added to the body of the release to `release_notes.md`.
+4. Push a tag with prepared release details (this will trigger the release `vX.X.0`)
+   - `git tag -a vX.X.x -m 'Release vX.X.x'`
+   - `git push origin vX.X.x`
+5. Update the changelog.md file on master with the releases changelog.
+6. Delete any RC branches and tags for this release (if applicable)
 
 #### Minor Release
 
@@ -274,15 +262,17 @@ Minor releases are done differently from major releases: They are built off of l
    - run `python ./scripts/linkify_changelog.py CHANGELOG.md` to add links for all issues
    - run `bash ./scripts/authors.sh` to get a list of authors since the latest release, and add the GitHub aliases of external contributors to the top of the CHANGELOG. To lookup an alias from an email, try `bash ./scripts/authors.sh <email>`
    - reset the `CHANGELOG_PENDING.md`
-   - bump Tendermint version in `version.go`
    - bump P2P and block protocol versions in  `version.go`, if necessary
    - bump ABCI protocol version in `version.go`, if necessary
    - make sure all significant breaking changes are covered in `UPGRADING.md`
+   - Add any release notes you would like to be added to the body of the release to `release_notes.md`.
 4. Create a release branch `release/vX.X.x` off the release candidate branch:
    - `git checkout -b release/vX.X.x`
    - `git push -u origin release/vX.X.x`
    - Note that all branches prefixed with `release` are protected once pushed. You will need admin help to make any changes to the branch.
-5. Open a pull request of the new minor release branch onto the latest major release branch `vX.X` and then rebase to merge. This will start the release process.
+5. Once the release branch has been approved, make sure to pull it locally, then push a tag.
+   - `git tag -a vX.X.x -m 'Release vX.X.x'`
+   - `git push origin vX.X.x`
 6. Create a pull request back to master with the CHANGELOG & version changes from the latest release.
    - Remove all `R:minor` labels from the pull requests that were included in the release.
    - Do not merge the release branch into master.
@@ -293,19 +283,113 @@ Minor releases are done differently from major releases: They are built off of l
 
 1. start from the existing release branch you want to backport changes to (e.g. v0.30)
    Branch to a release/vX.X.X branch locally (e.g. release/v0.30.7)
-2. cherry pick the commit(s) that contain the changes you want to backport (usually these commits are from squash-merged PRs which were already reviewed)
-3. steps 2 and 3 from [Major Release](#major-release)
-4. push changes to release/vX.X.X branch
-5. open a PR against the existing vX.X branch
+2. Cherry pick the commit(s) that contain the changes you want to backport (usually these commits are from squash-merged PRs which were already reviewed)
+3. Follow steps 2 and 3 from [Major Release](#major-release)
+4. Push changes to release/vX.X.X branch
+5. Open a PR against the existing vX.X branch
+
+#### Release Candidates
+
+Before creating an official release, especially a major release, we may want to create a
+release candidate (RC) for our friends and partners to test out. We use git tags to
+create RCs, and we build them off of RC branches. RC branches typically have names formatted
+like `RCX/vX.X.X` (or, concretely, `RC0/v0.34.0`), while the tags themselves follow
+the "standard" release naming conventions, with `-rcX` at the end (`vX.X.X-rcX`).
+
+(Note that branches and tags _cannot_ have the same names, so it's important that these branches
+have distinct names from the tags/release names.)
+
+1. Start from the RC branch (e.g. `RC0/v0.34.0`).
+2. Create the new tag, specifying a name and a tag "message":  
+   `git tag -a v0.34.0-rc0 -m "Release Candidate v0.34.0-rc0` 
+3. Push the tag back up to origin:  
+   `git push origin v0.34.0-rc4`  
+   Now the tag should be available on the repo's releases page. 
+4. Create a new release candidate branch for any possible updates to the RC:  
+   `git checkout -b RC1/v0.34.0; git push origin RC1/v0.34.0`
 
 ## Testing
 
-All repos should be hooked up to [CircleCI](https://circleci.com/).
+### Unit tests
 
-If they have `.go` files in the root directory, they will be automatically
-tested by circle using `go test -v -race ./...`. If not, they will need a
-`circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and
-includes its continuous integration status using a badge in the `README.md`.
+Unit tests are located in `_test.go` files as directed by [the Go testing
+package](https://golang.org/pkg/testing/). If you're adding or removing a
+function, please check there's a `TestType_Method` test for it.
+
+Run: `make test`
+
+### Integration tests
+
+Integration tests are also located in `_test.go` files. What differentiates
+them is a more complicated setup, which usually involves setting up two or more
+components.
+
+Run: `make test_integrations`
+
+### End-to-end tests
+
+End-to-end tests are used to verify a fully integrated Tendermint network.
+
+See [README](./test/e2e/README.md) for details.
+
+Run:
+
+```sh
+cd test/e2e && \
+  make && \
+  ./build/runner -f networks/ci.toml
+```
+
+### Maverick
+
+**If you're changing the code in `consensus` package, please make sure to
+replicate all the changes in `./test/maverick/consensus`**. Maverick is a
+byzantine node used to assert that the validator gets punished for malicious
+behavior.
+
+See [README](./test/maverick/README.md) for details.
+
+### Model-based tests (ADVANCED)
+
+*NOTE: if you're just submitting your first PR, you won't need to touch these
+most probably (99.9%)*.
+
+For components, that have been [formally
+verified](https://en.wikipedia.org/wiki/Formal_verification) using
+[TLA+](https://en.wikipedia.org/wiki/TLA%2B), it may be possible to generate
+tests using a combination of the [Apalache Model
+Checker](https://apalache.informal.systems/) and [tendermint-rs testgen
+util](https://github.com/informalsystems/tendermint-rs/tree/master/testgen).
+
+Now, I know there's a lot to take in. If you want to learn more, check out [
+this video](https://www.youtube.com/watch?v=aveoIMphzW8) by Andrey Kupriyanov
+& Igor Konnov.
+
+At the moment, we have model-based tests for the light client, located in the
+`./light/mbt` directory.
+
+Run: `cd light/mbt && go test`
+
+### Fuzz tests (ADVANCED)
+
+*NOTE: if you're just submitting your first PR, you won't need to touch these
+most probably (99.9%)*.
+
+[Fuzz tests](https://en.wikipedia.org/wiki/Fuzzing) can be found inside the
+`./test/fuzz` directory. See [README.md](./test/fuzz/README.md) for details.
+
+Run: `cd test/fuzz && make fuzz-{PACKAGE-COMPONENT}`
+
+### Jepsen tests (ADVANCED)
+
+*NOTE: if you're just submitting your first PR, you won't need to touch these
+most probably (99.9%)*.
+
+[Jepsen](http://jepsen.io/) tests are used to verify the
+[linearizability](https://jepsen.io/consistency/models/linearizable) property
+of the Tendermint consensus. They are located in a separate repository
+-> <https://github.com/tendermint/jepsen>. Please refer to its README for more
+information.
 
 ### RPC Testing
 
@@ -318,4 +402,8 @@ make build-linux build-contract-tests-hooks
 make contract-tests
 ```
 
-This command will popup a network and check every endpoint against what has been documented
+**WARNING: these are currently broken due to <https://github.com/apiaryio/dredd>
+not supporting complete OpenAPI 3**.
+
+This command will popup a network and check every endpoint against what has
+been documented.

DOCKER/Dockerfile

@@ -1,4 +1,14 @@
-FROM alpine:3.9
+# stage 1 Generate Tendermint Binary
+FROM golang:1.18-alpine as builder
+RUN apk update && \
+    apk upgrade && \
+    apk --no-cache add make
+COPY / /tendermint
+WORKDIR /tendermint
+RUN make build-linux
+
+# stage 2
+FROM golang:1.18-alpine
 LABEL maintainer="hello@tendermint.com"
 
 # Tendermint will be looking for the genesis file in /tendermint/config/genesis.json
@@ -29,15 +39,14 @@ EXPOSE 26656 26657 26660
 
 STOPSIGNAL SIGTERM
 
-ARG BINARY=tendermint
-COPY $BINARY /usr/bin/tendermint
+COPY --from=builder /tendermint/build/tendermint /usr/bin/tendermint
 
 # You can overwrite these before the first run to influence
 # config.json and genesis.json. Additionally, you can override
 # CMD to add parameters to `tendermint node`.
 ENV PROXY_APP=kvstore MONIKER=dockernode CHAIN_ID=dockerchain
 
-COPY ./docker-entrypoint.sh /usr/local/bin/
+COPY ./DOCKER/docker-entrypoint.sh /usr/local/bin/
 
 ENTRYPOINT ["docker-entrypoint.sh"]
 CMD ["node"]

DOCKER/README.md

@@ -6,7 +6,7 @@ DockerHub tags for official releases are [here](https://hub.docker.com/r/tenderm
 
 Official releases can be found [here](https://github.com/tendermint/tendermint/releases).
 
-The Dockerfile for tendermint is not expected to change in the near future. The master file used for all builds can be found [here](https://raw.githubusercontent.com/tendermint/tendermint/master/DOCKER/Dockerfile).
+The Dockerfile for tendermint is not expected to change in the near future. The master file used for all builds can be found [here](https://raw.githubusercontent.com/tendermint/tendermint/main/DOCKER/Dockerfile).
 
 Respective versioned files can be found <https://raw.githubusercontent.com/tendermint/tendermint/vX.XX.XX/DOCKER/Dockerfile> (replace the Xs with the version number).
 
@@ -20,9 +20,9 @@ Respective versioned files can be found <https://raw.githubusercontent.com/tende
 
 Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, and securely replicates it on many machines.
 
-For more background, see the [the docs](https://docs.tendermint.com/master/introduction/#quick-start).
+For more background, see the [the docs](https://docs.tendermint.com/v0.34/introduction/#quick-start).
 
-To get started developing applications, see the [application developers guide](https://docs.tendermint.com/master/introduction/quick-start.html).
+To get started developing applications, see the [application developers guide](https://docs.tendermint.com/v0.34/introduction/quick-start.html).
 
 ## How to use this image
 
@@ -37,7 +37,7 @@ docker run -it --rm -v "/tmp:/tendermint" tendermint/tendermint node --proxy_app
 
 ## Local cluster
 
-To run a 4-node network, see the `Makefile` in the root of [the repo](https://github.com/tendermint/tendermint/blob/master/Makefile) and run:
+To run a 4-node network, see the `Makefile` in the root of [the repo](https://github.com/tendermint/tendermint/blob/v0.34.x/Makefile) and run:
 
 ```sh
 make build-linux
@@ -49,8 +49,8 @@ Note that this will build and use a different image than the ones provided here.
 
 ## License
 
-- Tendermint's license is [Apache 2.0](https://github.com/tendermint/tendermint/blob/master/LICENSE).
+- Tendermint's license is [Apache 2.0](https://github.com/tendermint/tendermint/blob/main/LICENSE).
 
 ## Contributing
 
-Contributions are most welcome! See the [contributing file](https://github.com/tendermint/tendermint/blob/master/CONTRIBUTING.md) for more information.
+Contributions are most welcome! See the [contributing file](https://github.com/tendermint/tendermint/blob/main/CONTRIBUTING.md) for more information.

LICENSE

@@ -1,5 +1,3 @@
-Tendermint Core
-License: Apache2.0
 
                                  Apache License
                            Version 2.0, January 2004
@@ -181,7 +179,7 @@ License: Apache2.0
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -189,7 +187,7 @@ License: Apache2.0
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2016 All in Bits, Inc
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -2,10 +2,17 @@ PACKAGES=$(shell go list ./...)
 OUTPUT?=build/tendermint
 
 BUILD_TAGS?=tendermint
-LD_FLAGS = -X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`
+
+# If building a release, please checkout the version tag to get the correct version setting
+ifneq ($(shell git symbolic-ref -q --short HEAD),)
+VERSION := unreleased-$(shell git symbolic-ref -q --short HEAD)-$(shell git rev-parse HEAD)
+else
+VERSION := $(shell git describe)
+endif
+
+LD_FLAGS = -X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION)
 BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
 HTTPS_GIT := https://github.com/tendermint/tendermint.git
-DOCKER_BUF := docker run -v $(shell pwd):/workspace --workdir /workspace bufbuild/buf
 CGO_ENABLED ?= 0
 
 # handle nostrip
@@ -48,8 +55,6 @@ LD_FLAGS += $(LDFLAGS)
 all: check build test install
 .PHONY: all
 
-# The below include contains the tools.
-include tools.mk
 include tests.mk
 
 ###############################################################################
@@ -64,42 +69,59 @@ install:
 	CGO_ENABLED=$(CGO_ENABLED) go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/tendermint
 .PHONY: install
 
+
+###############################################################################
+###                                Mocks                                    ###
+###############################################################################
+
+mockery:
+	go generate -run="./scripts/mockery_generate.sh" ./...
+.PHONY: mockery
+
 ###############################################################################
 ###                                Protobuf                                 ###
 ###############################################################################
 
-proto-all: proto-gen proto-lint proto-check-breaking
-.PHONY: proto-all
+check-proto-deps:
+ifeq (,$(shell which protoc-gen-gogofaster))
+	@go install github.com/gogo/protobuf/protoc-gen-gogofaster@latest
+endif
+.PHONY: check-proto-deps
 
-proto-gen:
-	## If you get the following error,
-	## "error while loading shared libraries: libprotobuf.so.14: cannot open shared object file: No such file or directory"
-	## See https://stackoverflow.com/a/25518702
-	## Note the $< here is substituted for the %.proto
-	## Note the $@ here is substituted for the %.pb.go
-	@sh scripts/protocgen.sh
+check-proto-format-deps:
+ifeq (,$(shell which clang-format))
+	$(error "clang-format is required for Protobuf formatting. See instructions for your platform on how to install it.")
+endif
+.PHONY: check-proto-format-deps
+
+proto-gen: check-proto-deps
+	@echo "Generating Protobuf files"
+	@go run github.com/bufbuild/buf/cmd/buf generate
+	@mv ./proto/tendermint/abci/types.pb.go ./abci/types/
 .PHONY: proto-gen
 
-proto-gen-docker:
-	@echo "Generating Protobuf files"
-	@docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto sh ./scripts/protocgen.sh
-.PHONY: proto-gen-docker
-
-proto-lint:
-	@$(DOCKER_BUF) check lint --error-format=json
+# These targets are provided for convenience and are intended for local
+# execution only.
+proto-lint: check-proto-deps
+	@echo "Linting Protobuf files"
+	@go run github.com/bufbuild/buf/cmd/buf lint
 .PHONY: proto-lint
 
-proto-format:
+proto-format: check-proto-format-deps
 	@echo "Formatting Protobuf files"
-	docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
+	@find . -name '*.proto' -path "./proto/*" -exec clang-format -i {} \;
 .PHONY: proto-format
 
-proto-check-breaking:
-	@$(DOCKER_BUF) check breaking --against-input .git#branch=master
+proto-check-breaking: check-proto-deps
+	@echo "Checking for breaking changes in Protobuf files against local branch"
+	@echo "Note: This is only useful if your changes have not yet been committed."
+	@echo "      Otherwise read up on buf's \"breaking\" command usage:"
+	@echo "      https://docs.buf.build/breaking/usage"
+	@go run github.com/bufbuild/buf/cmd/buf breaking --against ".git"
 .PHONY: proto-check-breaking
 
 proto-check-breaking-ci:
-	@$(DOCKER_BUF) check breaking --against-input $(HTTPS_GIT)#branch=master
+	@go run github.com/bufbuild/buf/cmd/buf breaking --against $(HTTPS_GIT)#branch=v0.34.x
 .PHONY: proto-check-breaking-ci
 
 ###############################################################################
@@ -188,12 +210,12 @@ DESTINATION = ./index.html.md
 ###############################################################################
 
 build-docs:
-	cd docs && \
-	while read p; do \
-		(git checkout $${p} . && npm install && VUEPRESS_BASE="/$${p}/" npm run build) ; \
-		mkdir -p ~/output/$${p} ; \
-		cp -r .vuepress/dist/* ~/output/$${p}/ ; \
-		cp ~/output/$${p}/index.html ~/output ; \
+	@cd docs && \
+	while read -r branch path_prefix; do \
+		(git checkout $${branch} && npm ci && VUEPRESS_BASE="/$${path_prefix}/" npm run build) ; \
+		mkdir -p ~/output/$${path_prefix} ; \
+		cp -r .vuepress/dist/* ~/output/$${path_prefix}/ ; \
+		cp ~/output/$${path_prefix}/index.html ~/output ; \
 	done < versions ;
 .PHONY: build-docs
 
@@ -220,7 +242,7 @@ build-docker: build-linux
 ###############################################################################
 
 # Build linux binary on other platforms
-build-linux: tools
+build-linux:
 	GOOS=linux GOARCH=amd64 $(MAKE) build
 .PHONY: build-linux
 

README.md

@@ -2,157 +2,174 @@
 
 ![banner](docs/tendermint-core-image.jpg)
 
-[Byzantine-Fault Tolerant](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance)
-[State Machines](https://en.wikipedia.org/wiki/State_machine_replication).
-Or [Blockchain](<https://en.wikipedia.org/wiki/Blockchain_(database)>), for short.
+[Byzantine-Fault Tolerant][bft] [State Machine Replication][smr]. Or
+[Blockchain], for short.
 
-[![version](https://img.shields.io/github/tag/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/releases/latest)
-[![API Reference](https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667)](https://pkg.go.dev/github.com/tendermint/tendermint)
-[![Go version](https://img.shields.io/badge/go-1.14-blue.svg)](https://github.com/moovweb/gvm)
-[![Discord chat](https://img.shields.io/discord/669268347736686612.svg)](https://discord.gg/AzefAFd)
-[![license](https://img.shields.io/github/license/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/blob/master/LICENSE)
-[![tendermint/tendermint](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint)
-[![Sourcegraph](https://sourcegraph.com/github.com/tendermint/tendermint/-/badge.svg)](https://sourcegraph.com/github.com/tendermint/tendermint?badge)
+[![Version][version-badge]][version-url]
+[![API Reference][api-badge]][api-url]
+[![Go version][go-badge]][go-url]
+[![Discord chat][discord-badge]][discord-url]
+[![License][license-badge]][license-url]
+[![Sourcegraph][sg-badge]][sg-url]
 
-| Branch | Tests                                                                                                                                                                                                                                                  | Coverage                                                                                                                             | Linting                                                                    |
-| ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------- |
-| master | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) </br> ![Tests](https://github.com/tendermint/tendermint/workflows/Tests/badge.svg?branch=master) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) | ![Lint](https://github.com/tendermint/tendermint/workflows/Lint/badge.svg) |
+| Branch | Tests                              | Linting                         |
+|--------|------------------------------------|---------------------------------|
+| main   | [![Tests][tests-badge]][tests-url] | [![Lint][lint-badge]][lint-url] |
 
-Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine - written in any programming language -
-and securely replicates it on many machines.
+Tendermint Core is a Byzantine Fault Tolerant (BFT) middleware that takes a
+state transition machine - written in any programming language - and securely
+replicates it on many machines.
 
-For protocol details, see [the specification](https://github.com/tendermint/spec).
+For protocol details, refer to the [Tendermint Specification](./spec/README.md).
 
-For detailed analysis of the consensus protocol, including safety and liveness proofs,
-see our recent paper, "[The latest gossip on BFT consensus](https://arxiv.org/abs/1807.04938)".
+For detailed analysis of the consensus protocol, including safety and liveness
+proofs, read our paper, "[The latest gossip on BFT
+consensus](https://arxiv.org/abs/1807.04938)".
+
+## Documentation
+
+Complete documentation can be found on the
+[website](https://docs.tendermint.com/).
 
 ## Releases
 
-Please do not depend on master as your production branch. Use [releases](https://github.com/tendermint/tendermint/releases) instead.
+Please do not depend on `main` as your production branch. Use
+[releases](https://github.com/tendermint/tendermint/releases) instead.
 
-Tendermint is being used in production in both private and public environments,
-most notably the blockchains of the [Cosmos Network](https://cosmos.network/).
-However, we are still making breaking changes to the protocol and the APIs and have not yet released v1.0.
-See below for more details about [versioning](#versioning).
+Tendermint has been in the production of private and public environments, most
+notably the blockchains of the Cosmos Network. we haven't released v1.0 yet
+since we are making breaking changes to the protocol and the APIs. See below for
+more details about [versioning](#versioning).
 
-In any case, if you intend to run Tendermint in production, we're happy to help. You can
-contact us [over email](mailto:hello@interchain.berlin) or [join the chat](https://discord.gg/AzefAFd).
+In any case, if you intend to run Tendermint in production, we're happy to help.
+You can contact us [over email](mailto:hello@interchain.io) or [join the
+chat](https://discord.gg/cosmosnetwork).
+
+More on how releases are conducted can be found [here](./RELEASES.md).
 
 ## Security
 
 To report a security vulnerability, see our [bug bounty
-program](https://hackerone.com/tendermint). 
-For examples of the kinds of bugs we're looking for, see [our security policy](SECURITY.md)
+program](https://hackerone.com/cosmos). For examples of the kinds of bugs we're
+looking for, see [our security policy](SECURITY.md).
+
+We also maintain a dedicated mailing list for security updates. We will only
+ever use this mailing list to notify you of vulnerabilities and fixes in
+Tendermint Core. You can subscribe [here](http://eepurl.com/gZ5hQD).
 
 ## Minimum requirements
 
-| Requirement | Notes            |
-| ----------- | ---------------- |
-| Go version  | Go1.14 or higher |
-
-## Documentation
-
-Complete documentation can be found on the [website](https://docs.tendermint.com/master/).
+| Requirement | Notes             |
+|-------------|-------------------|
+| Go version  | Go 1.18 or higher |
 
 ### Install
 
-See the [install instructions](/docs/introduction/install.md).
+See the [install instructions](./docs/introduction/install.md).
 
 ### Quick Start
 
-- [Single node](/docs/introduction/quick-start.md)
-- [Local cluster using docker-compose](/docs/networks/docker-compose.md)
-- [Remote cluster using Terraform and Ansible](/docs/networks/terraform-and-ansible.md)
-- [Join the Cosmos testnet](https://cosmos.network/testnet)
+- [Single node](./docs/introduction/quick-start.md)
+- [Local cluster using docker-compose](./docs/tools/docker-compose.md)
+- [Remote cluster using Terraform and Ansible](./docs/tools/terraform-and-ansible.md)
 
 ## Contributing
 
 Please abide by the [Code of Conduct](CODE_OF_CONDUCT.md) in all interactions.
 
-Before contributing to the project, please take a look at the [contributing guidelines](CONTRIBUTING.md)
-and the [style guide](STYLE_GUIDE.md). You may also find it helpful to read the
-[specifications](https://github.com/tendermint/spec), watch the [Developer Sessions](/docs/DEV_SESSIONS.md), 
-and familiarize yourself with our
-[Architectural Decision Records](https://github.com/tendermint/tendermint/tree/master/docs/architecture).
+Before contributing to the project, please take a look at the [contributing
+guidelines](CONTRIBUTING.md) and the [style guide](STYLE_GUIDE.md). You may also
+find it helpful to read the [specifications](./spec/README.md), and familiarize
+yourself with our [Architectural Decision Records
+(ADRs)](./docs/architecture/README.md) and
+[Request For Comments (RFCs)](./docs/rfc/README.md).
 
 ## Versioning
 
 ### Semantic Versioning
 
-Tendermint uses [Semantic Versioning](http://semver.org/) to determine when and how the version changes.
-According to SemVer, anything in the public API can change at any time before version 1.0.0
+Tendermint uses [Semantic Versioning](http://semver.org/) to determine when and
+how the version changes. According to SemVer, anything in the public API can
+change at any time before version 1.0.0
 
-To provide some stability to Tendermint users in these 0.X.X days, the MINOR version is used
-to signal breaking changes across a subset of the total public API. This subset includes all
-interfaces exposed to other processes (cli, rpc, p2p, etc.), but does not
-include the Go APIs.
+To provide some stability to users of 0.X.X versions of Tendermint, the MINOR
+version is used to signal breaking changes across Tendermint's API. This API
+includes all publicly exposed types, functions, and methods in non-internal Go
+packages as well as the types and methods accessible via the Tendermint RPC
+interface.
 
-That said, breaking changes in the following packages will be documented in the
-CHANGELOG even if they don't lead to MINOR version bumps:
-
-- crypto
-- config
-- libs
-    - bech32
-    - bits
-    - bytes
-    - json
-    - log
-    - math
-    - net
-    - os
-    - protoio
-    - rand
-    - sync
-    - strings
-    - service
-- node
-- rpc/client
-- types
+Breaking changes to these public APIs will be documented in the CHANGELOG.
 
 ### Upgrades
 
-In an effort to avoid accumulating technical debt prior to 1.0.0,
-we do not guarantee that breaking changes (ie. bumps in the MINOR version)
-will work with existing Tendermint blockchains. In these cases you will
-have to start a new blockchain, or write something custom to get the old
-data into the new chain. However, any bump in the PATCH version should be 
-compatible with existing blockchain histories.
-
+In an effort to avoid accumulating technical debt prior to 1.0.0, we do not
+guarantee that breaking changes (ie. bumps in the MINOR version) will work with
+existing Tendermint blockchains. In these cases you will have to start a new
+blockchain, or write something custom to get the old data into the new chain.
+However, any bump in the PATCH version should be compatible with existing
+blockchain histories.
 
 For more information on upgrading, see [UPGRADING.md](./UPGRADING.md).
 
 ### Supported Versions
 
-Because we are a small core team, we only ship patch updates, including security updates,
-to the most recent minor release and the second-most recent minor release. Consequently,
-we strongly recommend keeping Tendermint up-to-date. Upgrading instructions can be found
-in [UPGRADING.md](./UPGRADING.md).
+Because we are a small core team, we only ship patch updates, including security
+updates, to the most recent minor release and the second-most recent minor
+release. Consequently, we strongly recommend keeping Tendermint up-to-date.
+Upgrading instructions can be found in [UPGRADING.md](./UPGRADING.md).
 
 ## Resources
 
-### Tendermint Core
+### Libraries
 
-For details about the blockchain data structures and the p2p protocols, see the
-[Tendermint specification](https://docs.tendermint.com/master/spec/).
-
-For details on using the software, see the [documentation](/docs/) which is also
-hosted at: <https://docs.tendermint.com/master/>
-
-### Tools
-
-Benchmarking is provided by [`tm-load-test`](https://github.com/informalsystems/tm-load-test).
-Additional tooling can be found in [/docs/tools](/docs/tools).
+- [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); A framework for building
+  applications in Golang
+- [Tendermint in Rust](https://github.com/informalsystems/tendermint-rs)
+- [ABCI Tower](https://github.com/penumbra-zone/tower-abci)
 
 ### Applications
 
-- [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework
-- [Ethermint](http://github.com/cosmos/ethermint); Ethereum on Tendermint
-- [Many more](https://tendermint.com/ecosystem)
+- [Cosmos Hub](https://hub.cosmos.network/)
+- [Terra](https://www.terra.money/)
+- [Celestia](https://celestia.org/)
+- [Anoma](https://anoma.network/)
+- [Vocdoni](https://docs.vocdoni.io/)
 
 ### Research
 
 - [The latest gossip on BFT consensus](https://arxiv.org/abs/1807.04938)
 - [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769)
 - [Original Whitepaper: "Tendermint: Consensus Without Mining"](https://tendermint.com/static/docs/tendermint.pdf)
-- [Blog](https://blog.cosmos.network/tendermint/home)
+- [Tendermint Core Blog](https://medium.com/tendermint/tagged/tendermint-core)
+- [Cosmos Blog](https://blog.cosmos.network/tendermint/home)
+
+## Join us!
+
+Tendermint Core is maintained by [Interchain GmbH](https://interchain.berlin).
+If you'd like to work full-time on Tendermint Core,
+[we're hiring](https://interchain-gmbh.breezy.hr/)!
+
+Funding for Tendermint Core development comes primarily from the
+[Interchain Foundation](https://interchain.io), a Swiss non-profit. The
+Tendermint trademark is owned by [Tendermint Inc.](https://tendermint.com), the
+for-profit entity that also maintains [tendermint.com](https://tendermint.com).
+
+[bft]: https://en.wikipedia.org/wiki/Byzantine_fault_tolerance
+[smr]: https://en.wikipedia.org/wiki/State_machine_replication
+[Blockchain]: https://en.wikipedia.org/wiki/Blockchain
+[version-badge]: https://img.shields.io/github/tag/tendermint/tendermint.svg
+[version-url]: https://github.com/tendermint/tendermint/releases/latest
+[api-badge]: https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
+[api-url]: https://pkg.go.dev/github.com/tendermint/tendermint
+[go-badge]: https://img.shields.io/badge/go-1.18-blue.svg
+[go-url]: https://github.com/moovweb/gvm
+[discord-badge]: https://img.shields.io/discord/669268347736686612.svg
+[discord-url]: https://discord.gg/cosmosnetwork
+[license-badge]: https://img.shields.io/github/license/tendermint/tendermint.svg
+[license-url]: https://github.com/tendermint/tendermint/blob/main/LICENSE
+[sg-badge]: https://sourcegraph.com/github.com/tendermint/tendermint/-/badge.svg
+[sg-url]: https://sourcegraph.com/github.com/tendermint/tendermint?badge
+[tests-url]: https://github.com/tendermint/tendermint/actions/workflows/tests.yml
+[tests-badge]: https://github.com/tendermint/tendermint/actions/workflows/tests.yml/badge.svg?branch=main
+[lint-badge]: https://github.com/tendermint/tendermint/actions/workflows/lint.yml/badge.svg
+[lint-url]: https://github.com/tendermint/tendermint/actions/workflows/lint.yml

SECURITY.md

@@ -2,98 +2,146 @@
 
 ## Reporting a Bug
 
-As part of our [Coordinated Vulnerability Disclosure
-Policy](https://tendermint.com/security), we operate a [bug
-bounty](https://hackerone.com/tendermint).
-See the policy for more details on submissions and rewards, and see "Example Vulnerabilities" (below) for examples of the kinds of bugs we're most interested in.
+As part of our [Coordinated Vulnerability Disclosure Policy](https://tendermint.com/security),
+we operate a [bug bounty][hackerone]. See the policy for more
+details on submissions and rewards, and see "Example Vulnerabilities" (below)
+for examples of the kinds of bugs we're most interested in.
 
-### Guidelines 
+### Guidelines
 
 We require that all researchers:
 
-* Use the bug bounty to disclose all vulnerabilities, and avoid posting vulnerability information in public places, including Github Issues, Discord channels, and Telegram groups
-* Make every effort to avoid privacy violations, degradation of user experience, disruption to production systems (including but not limited to the Cosmos Hub), and destruction of data
-* Keep any information about vulnerabilities that you’ve discovered confidential between yourself and the Tendermint Core engineering team until the issue has been resolved and disclosed 
+* Use the bug bounty to disclose all vulnerabilities, and avoid posting
+  vulnerability information in public places, including Github Issues, Discord
+  channels, and Telegram groups
+* Make every effort to avoid privacy violations, degradation of user experience,
+  disruption to production systems (including but not limited to the Cosmos
+  Hub), and destruction of data
+* Keep any information about vulnerabilities that you’ve discovered confidential
+  between yourself and the Tendermint Core engineering team until the issue has
+  been resolved and disclosed
 * Avoid posting personally identifiable information, privately or publicly
 
 If you follow these guidelines when reporting an issue to us, we commit to:
 
-* Not pursue or support any legal action related to your research on this vulnerability
-* Work with you to understand, resolve and ultimately disclose the issue in a timely fashion 
+* Not pursue or support any legal action related to your research on this
+  vulnerability
+* Work with you to understand, resolve and ultimately disclose the issue in a
+  timely fashion
 
-## Disclosure Process 
+## Disclosure Process
 
 Tendermint Core uses the following disclosure process:
 
-1. Once a security report is received, the Tendermint Core team works to verify the issue and confirm its severity level using CVSS. 
-2. The Tendermint Core team collaborates with the Gaia team to determine the vulnerability’s potential impact on the Cosmos Hub. 
-3. Patches are prepared for eligible releases of Tendermint in private repositories. See “Supported Releases” below for more information on which releases are considered eligible. 
-4. If it is determined that a CVE-ID is required, we request a CVE through a CVE Numbering Authority. 
-5. We notify the community that a security release is coming, to give users time to prepare their systems for the update. Notifications can include forum posts, tweets, and emails to partners and validators, including emails sent to the [Tendermint Security Mailing List](https://berlin.us4.list-manage.com/subscribe?u=431b35421ff7edcc77df5df10&id=3fe93307bc).
-6. 24 hours following this notification, the fixes are applied publicly and new releases are issued. 
-7. Cosmos SDK and Gaia update their Tendermint Core dependencies to use these releases, and then themselves issue new releases. 
-8. Once releases are available for Tendermint Core, Cosmos SDK and Gaia, we notify the community, again, through the same channels as above. We also publish a Security Advisory on Github and publish the CVE, as long as neither the Security Advisory nor the CVE include any information on how to exploit these vulnerabilities beyond what information is already available in the patch itself. 
-9. Once the community is notified, we will pay out any relevant bug bounties to submitters. 
-10. One week after the releases go out, we will publish a post with further details on the vulnerability as well as our response to it. 
+1. Once a security report is received, the Tendermint Core team works to verify
+   the issue and confirm its severity level using CVSS.
+2. The Tendermint Core team collaborates with the Gaia team to determine the
+   vulnerability’s potential impact on the Cosmos Hub.
+3. Patches are prepared for eligible releases of Tendermint in private
+   repositories. See “Supported Releases” below for more information on which
+   releases are considered eligible.
+4. If it is determined that a CVE-ID is required, we request a CVE through a CVE
+   Numbering Authority.
+5. We notify the community that a security release is coming, to give users time
+   to prepare their systems for the update. Notifications can include forum
+   posts, tweets, and emails to partners and validators, including emails sent
+   to the [Tendermint Security Mailing List][tmsec-mailing].
+6. 24 hours following this notification, the fixes are applied publicly and new
+   releases are issued.
+7. Cosmos SDK and Gaia update their Tendermint Core dependencies to use these
+   releases, and then themselves issue new releases.
+8. Once releases are available for Tendermint Core, Cosmos SDK and Gaia, we
+   notify the community, again, through the same channels as above. We also
+   publish a Security Advisory on Github and publish the CVE, as long as neither
+   the Security Advisory nor the CVE include any information on how to exploit
+   these vulnerabilities beyond what information is already available in the
+   patch itself.
+9. Once the community is notified, we will pay out any relevant bug bounties to
+   submitters.
+10. One week after the releases go out, we will publish a post with further
+    details on the vulnerability as well as our response to it.
 
-This process can take some time. Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we follow the process described above to ensure that disclosures are handled consistently and to keep Tendermint Core and its downstream dependent projects--including but not limited to Gaia and the Cosmos Hub--as secure as possible. 
+This process can take some time. Every effort will be made to handle the bug in
+as timely a manner as possible, however it's important that we follow the
+process described above to ensure that disclosures are handled consistently and
+to keep Tendermint Core and its downstream dependent projects--including but not
+limited to Gaia and the Cosmos Hub--as secure as possible.
 
-### Example Timeline 
+### Example Timeline
 
-The following is an example timeline for the triage and response. The required roles and team members are described in parentheses after each task; however, multiple people can play each role and each person may play multiple roles. 
+The following is an example timeline for the triage and response. The required
+roles and team members are described in parentheses after each task; however,
+multiple people can play each role and each person may play multiple roles.
 
-#### > 24 Hours Before Release Time
+#### 24+ Hours Before Release Time
 
-1. Request CVE number (ADMIN) 
-2. Gather emails and other contact info for validators (COMMS LEAD) 
-3. Test fixes on a testnet  (TENDERMINT ENG, COSMOS ENG) 
-4. Write “Security Advisory” for forum (TENDERMINT LEAD) 
+1. Request CVE number (ADMIN)
+2. Gather emails and other contact info for validators (COMMS LEAD)
+3. Create patches in a private security repo, and ensure that PRs are open
+   targeting all relevant release branches (TENDERMINT ENG, TENDERMINT LEAD)
+4. Test fixes on a testnet  (TENDERMINT ENG, COSMOS SDK ENG)
+5. Write “Security Advisory” for forum (TENDERMINT LEAD)
 
 #### 24 Hours Before Release Time
 
-1. Post “Security Advisory” pre-notification on forum (TENDERMINT LEAD) 
-2. Post Tweet linking to forum post (COMMS LEAD) 
-3. Announce security advisory/link to post in various other social channels (Telegram, Discord) (COMMS LEAD) 
-4. Send emails to validators or other users (PARTNERSHIPS LEAD) 
+1. Post “Security Advisory” pre-notification on forum (TENDERMINT LEAD)
+2. Post Tweet linking to forum post (COMMS LEAD)
+3. Announce security advisory/link to post in various other social channels
+   (Telegram, Discord) (COMMS LEAD)
+4. Send emails to validators or other users (PARTNERSHIPS LEAD)
 
 #### Release Time
 
-1. Cut Tendermint releases for eligible versions (TENDERMINT ENG, TENDERMINT LEAD)
+1. Cut Tendermint releases for eligible versions (TENDERMINT ENG, TENDERMINT
+   LEAD)
 2. Cut Cosmos SDK release for eligible versions (COSMOS ENG)
 3. Cut Gaia release for eligible versions (GAIA ENG)
 4. Post “Security releases” on forum (TENDERMINT LEAD)
 5. Post new Tweet linking to forum post (COMMS LEAD)
-6. Remind everyone via social channels (Telegram, Discord)  that the release is out (COMMS LEAD)
-7. Send emails to validators or other users (COMMS LEAD) 
-8. Publish Security Advisory and CVE, if CVE has no sensitive information (ADMIN) 
+6. Remind everyone via social channels (Telegram, Discord)  that the release is
+   out (COMMS LEAD)
+7. Send emails to validators or other users (COMMS LEAD)
+8. Publish Security Advisory and CVE, if CVE has no sensitive information
+   (ADMIN)
 
 #### After Release Time
 
 1. Write forum post with exploit details (TENDERMINT LEAD)
-2. Approve pay-out on HackerOne for submitter (ADMIN) 
+2. Approve pay-out on HackerOne for submitter (ADMIN)
 
 #### 7 Days After Release Time
 
-1. Publish CVE if it has not yet been published (ADMIN) 
+1. Publish CVE if it has not yet been published (ADMIN)
 2. Publish forum post with exploit details (TENDERMINT ENG, TENDERMINT LEAD)
 
 ## Supported Releases
 
-The Tendermint Core team commits to releasing security patch releases for both the latest minor release as well for the major/minor release that the Cosmos Hub is running. 
+The Tendermint Core team commits to releasing security patch releases for both
+the latest minor release as well for the major/minor release that the Cosmos Hub
+is running.
 
-If you are running older versions of Tendermint Core, we encourage you to upgrade at your earliest opportunity so that you can receive security patches directly from the Tendermint repo. While you are welcome to backport security patches to older versions for your own use, we will not publish or promote these backports. 
+If you are running older versions of Tendermint Core, we encourage you to
+upgrade at your earliest opportunity so that you can receive security patches
+directly from the Tendermint repo. While you are welcome to backport security
+patches to older versions for your own use, we will not publish or promote these
+backports.
 
 ## Scope
 
-The full scope of our bug bounty program is outlined on our [Hacker One program page](https://hackerone.com/tendermint). Please also note that, in the interest of the safety of our users and staff, a few things are explicitly excluded from scope:
+The full scope of our bug bounty program is outlined on our
+[Hacker One program page][hackerone]. Please also note that, in the interest of
+the safety of our users and staff, a few things are explicitly excluded from
+scope:
 
-* Any third-party services 
-* Findings from physical testing, such as office access 
+* Any third-party services
+* Findings from physical testing, such as office access
 * Findings derived from social engineering (e.g., phishing)
 
-## Example Vulnerabilities 
+## Example Vulnerabilities
 
-The following is a list of examples of the kinds of vulnerabilities that we’re most interested in. It is not exhaustive: there are other kinds of issues we may also be interested in! 
+The following is a list of examples of the kinds of vulnerabilities that we’re
+most interested in. It is not exhaustive: there are other kinds of issues we may
+also be interested in!
 
 ### Specification
 
@@ -105,7 +153,8 @@ The following is a list of examples of the kinds of vulnerabilities that we’re
 
 Assuming less than 1/3 of the voting power is Byzantine (malicious):
 
-* Validation of blockchain data structures, including blocks, block parts, votes, and so on
+* Validation of blockchain data structures, including blocks, block parts,
+  votes, and so on
 * Execution of blocks
 * Validator set changes
 * Proposer round robin
@@ -114,6 +163,9 @@ Assuming less than 1/3 of the voting power is Byzantine (malicious):
 * A node halting (liveness failure)
 * Syncing new and old nodes
 
+Assuming more than 1/3 the voting power is Byzantine:
+
+* Attacks that go unpunished (unhandled evidence)
 
 ### Networking
 
@@ -139,7 +191,7 @@ Attacks may come through the P2P network or the RPC layer:
 
 ### Libraries
 
-* Serialization (Amino)
+* Serialization
 * Reading/Writing files and databases
 
 ### Cryptography
@@ -150,5 +202,8 @@ Attacks may come through the P2P network or the RPC layer:
 
 ### Light Client
 
-* Core verification 
+* Core verification
 * Bisection/sequential algorithms
+
+[hackerone]: https://hackerone.com/cosmos
+[tmsec-mailing]: https://berlin.us4.list-manage.com/subscribe?u=431b35421ff7edcc77df5df10&id=3fe93307bc

UPGRADING.md

@@ -1,6 +1,22 @@
 # Upgrading Tendermint Core
 
-This guide provides instructions for upgrading to specific versions of Tendermint Core.
+This guide provides instructions for upgrading to specific versions of
+Tendermint Core.
+
+## v0.34.20
+
+### Feature: Priority Mempool
+
+This release backports an implementation of the Priority Mempool from the v0.35
+branch. This implementation of the mempool permits the application to set a
+priority on each transaction during CheckTx, and during block selection the
+highest-priority transactions are chosen (subject to the constraints on size
+and gas cost).
+
+Operators can enable the priority mempool by setting `mempool.version` to
+`"v1"` in the `config.toml`. For more technical details about the priority
+mempool, see [ADR 067: Mempool
+Refactor](https://github.com/tendermint/tendermint/blob/main/docs/architecture/adr-067-mempool-refactor.md).
 
 ## v0.34.0
 
@@ -8,65 +24,66 @@ This guide provides instructions for upgrading to specific versions of Tendermin
 This release is not compatible with previous blockchains due to changes to
 the encoding format (see "Protocol Buffers," below) and the block header (see "Blockchain Protocol").
 
+Note also that Tendermint 0.34 also requires Go 1.16 or higher.
+
 ### ABCI Changes
 
-* New ABCI methods (`ListSnapshots`, `LoadSnapshotChunk`, `OfferSnapshot`, and `ApplySnapshotChunk`) 
-  were added to support the new State Sync feature. 
-  Previously, syncing a new node to a preexisting network could take days; but with State Sync, 
-  new nodes are able to join a network in a matter of seconds. 
-  Read [the spec](https://docs.tendermint.com/master/spec/abci/apps.html#state-sync) 
-  if you want to learn more about State Sync, or if you'd like your application to use it. 
-  (If you don't want to support State Sync in your application, you can just implement these new 
-  ABCI methods as no-ops, leaving them empty.) 
+* The `ABCIVersion` is now `0.17.0`.
 
-* `KV.Pair` has been replaced with `abci.EventAttribute`. The `EventAttribute.Index` field 
+* New ABCI methods (`ListSnapshots`, `LoadSnapshotChunk`, `OfferSnapshot`, and `ApplySnapshotChunk`)
+  were added to support the new State Sync feature.
+  Previously, syncing a new node to a preexisting network could take days; but with State Sync,
+  new nodes are able to join a network in a matter of seconds.
+  Read [the spec](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/abci/apps.md#state-sync)
+  if you want to learn more about State Sync, or if you'd like your application to use it.
+  (If you don't want to support State Sync in your application, you can just implement these new
+  ABCI methods as no-ops, leaving them empty.)
+
+* `KV.Pair` has been replaced with `abci.EventAttribute`. The `EventAttribute.Index` field
   allows ABCI applications to dictate which events should be indexed.
 
-* The blockchain can now start from an arbitrary initial height, 
+* The blockchain can now start from an arbitrary initial height,
   provided to the application via `RequestInitChain.InitialHeight`.
 
-* ABCI evidence type is now an enum with two recognized types of evidence: 
-  `DUPLICATE_VOTE` and `LIGHT_CLIENT_ATTACK`. 
-  Applications should be able to handle these evidence types 
+* ABCI evidence type is now an enum with two recognized types of evidence:
+  `DUPLICATE_VOTE` and `LIGHT_CLIENT_ATTACK`.
+  Applications should be able to handle these evidence types
   (i.e., through slashing or other accountability measures).
 
-* The [`PublicKey` type](https://github.com/tendermint/tendermint/blob/master/proto/tendermint/crypto/keys.proto#L13-L15) 
-  (used in ABCI as part of `ValidatorUpdate`) now uses a `oneof` protobuf type. 
-  Note that since Tendermint only supports ed25519 validator keys, there's only one 
+* The [`PublicKey` type](https://github.com/tendermint/tendermint/blob/v0.34.x/proto/tendermint/crypto/keys.proto#L13-L15)
+  (used in ABCI as part of `ValidatorUpdate`) now uses a `oneof` protobuf type.
+  Note that since Tendermint only supports ed25519 validator keys, there's only one
   option in the `oneof`.  For more, see "Protocol Buffers," below.
 
-* The field `Proof`, on the ABCI type `ResponseQuery`, is now named `ProofOps`. 
-  For more, see "Crypto," below. 
+* The field `Proof`, on the ABCI type `ResponseQuery`, is now named `ProofOps`.
+  For more, see "Crypto," below.
 
 ### P2P Protocol
 
 The default codec is now proto3, not amino. The schema files can be found in the `/proto`
-directory. For more, see "Protobuf," below. 
+directory. For more, see "Protobuf," below.
 
 ### Blockchain Protocol
 
-* `Header#LastResultsHash` previously was the root hash of a Merkle tree built from `ResponseDeliverTx(Code, Data)` responses. 
-  As of 0.34,`Header#LastResultsHash` is now the root hash of a Merkle tree built from:
-    * `BeginBlock#Events`
-    * Root hash of a Merkle tree built from `ResponseDeliverTx(Code, Data,
-      GasWanted, GasUsed, Events)` responses
-    * `BeginBlock#Events`
+* `Header#LastResultsHash`, which is the root hash of a Merkle tree built from
+  `ResponseDeliverTx(Code, Data)` as of v0.34 also includes `GasWanted` and `GasUsed`
+  fields.
 
 * Merkle hashes of empty trees previously returned nothing, but now return the hash of an empty input,
-  to conform with [RFC-6962](https://tools.ietf.org/html/rfc6962). 
+  to conform with [RFC-6962](https://tools.ietf.org/html/rfc6962).
   This mainly affects `Header#DataHash`, `Header#LastResultsHash`, and
   `Header#EvidenceHash`, which are often empty. Non-empty hashes can also be affected, e.g. if their
   inputs depend on other (empty) Merkle hashes, giving different results.
 
 ### Transaction Indexing
 
-Tendermint now relies on the application to tell it which transactions to index. This means that 
-in the `config.toml`, generated by Tendermint, there is no longer a way to specify which 
-transactions to index. `tx.height` & `tx.hash` will always be indexed when using the `kv` indexer.
+Tendermint now relies on the application to tell it which transactions to index. This means that
+in the `config.toml`, generated by Tendermint, there is no longer a way to specify which
+transactions to index. `tx.height` and `tx.hash` will always be indexed when using the `kv` indexer.
 
-Applications must now choose to either a) enable indexing for all transactions, or 
+Applications must now choose to either a) enable indexing for all transactions, or
 b) allow node operators to decide which transactions to index.
-Applications can notify Tendermint to index a specific transaction by setting 
+Applications can notify Tendermint to index a specific transaction by setting
 `Index: bool` to `true` in the Event Attribute:
 
 ```go
@@ -82,19 +99,19 @@ Applications can notify Tendermint to index a specific transaction by setting
 
 ### Protocol Buffers
 
-Tendermint 0.34 replaces Amino with Protocol Buffers for encoding. 
-This migration is extensive and results in a number of changes, however, 
+Tendermint 0.34 replaces Amino with Protocol Buffers for encoding.
+This migration is extensive and results in a number of changes, however,
 Tendermint only uses the types generated from Protocol Buffers for disk and
-wire serialization. 
+wire serialization.
 **This means that these changes should not affect you as a Tendermint user.**
 
 However, Tendermint users and contributors may note the following changes:
 
-* Directory layout changes: All proto files have been moved under one directory, `/proto`. 
-  This is in line with the recommended file layout by [Buf](https://buf.build).  
+* Directory layout changes: All proto files have been moved under one directory, `/proto`.
+  This is in line with the recommended file layout by [Buf](https://buf.build).
   For more, see the [Buf documentation](https://buf.build/docs/lint-checkers#file_layout).
-* ABCI Changes: As noted in the "ABCI Changes" section above, the `PublicKey` type now uses 
-  a `oneof` type. 
+* ABCI Changes: As noted in the "ABCI Changes" section above, the `PublicKey` type now uses
+  a `oneof` type.
 
 For more on the Protobuf changes, please see our [blog post on this migration](https://medium.com/tendermint/tendermint-0-34-protocol-buffers-and-you-8c40558939ae).
 
@@ -108,79 +125,92 @@ Tendermint 0.34 includes new and updated consensus parameters.
 
 #### Evidence Parameters
 
-* `MaxNum`, which caps the total amount of evidence by a absolute number. The default is 50.
+* `MaxBytes`, which caps the total amount of evidence. The default is 1048576 (1 MB).
 
 ### Crypto
 
 #### Keys
 
-* Keys no longer include a type prefix. For example, ed25519 pubkeys have been renamed from 
-  `PubKeyEd25519` to `PubKey`. This reduces stutter (e.g., `ed25519.PubKey`). 
+* Keys no longer include a type prefix. For example, ed25519 pubkeys have been renamed from
+  `PubKeyEd25519` to `PubKey`. This reduces stutter (e.g., `ed25519.PubKey`).
 * Keys are now byte slices (`[]byte`) instead of byte arrays (`[<size>]byte`).
-* The multisig functionality that was previously in Tendermint now has 
-  a new home within the Cosmos SDK: 
+* The multisig functionality that was previously in Tendermint now has
+  a new home within the Cosmos SDK:
   [`cosmos/cosmos-sdk/types/multisig`](https://github.com/cosmos/cosmos-sdk/blob/master/crypto/types/multisig/multisignature.go).
-* Similarly, secp256k1 has been removed from the Tendermint repo. 
-  There is still [a secp256k1 implementation in the Cosmos SDK](https://github.com/cosmos/cosmos-sdk/tree/443e0c1f89bd3730a731aea30453bd732f7efa35/crypto/keys/secp256k1), 
-  and we recommend you use that package for all your secp256k1 needs.
 
 #### `merkle` Package
 
 * `SimpleHashFromMap()` and `SimpleProofsFromMap()` were removed.
-* The prefix `Simple` has been removed. (For example, `SimpleProof` is now called `Proof`.) 
-* All protobuf messages have been moved to the `/proto` directory. 
-* The protobuf message `Proof` that contained multiple ProofOp's has been renamed to `ProofOps`. 
-  As noted above, this affects the ABCI type `ResponseQuery`: 
+* The prefix `Simple` has been removed. (For example, `SimpleProof` is now called `Proof`.)
+* All protobuf messages have been moved to the `/proto` directory.
+* The protobuf message `Proof` that contained multiple ProofOp's has been renamed to `ProofOps`.
+  As noted above, this affects the ABCI type `ResponseQuery`:
   The field that was named Proof is now named `ProofOps`.
 * `HashFromByteSlices` and `ProofsFromByteSlices` now return a hash for empty inputs, to conform with
   [RFC-6962](https://tools.ietf.org/html/rfc6962).
 
 ### `libs` Package
 
-The `bech32` package has moved to the Cosmos SDK: 
+The `bech32` package has moved to the Cosmos SDK:
 [`cosmos/cosmos-sdk/types/bech32`](https://github.com/cosmos/cosmos-sdk/tree/4173ea5ebad906dd9b45325bed69b9c655504867/types/bech32).
 
 ### CLI
 
 The `tendermint lite` command has been renamed to `tendermint light` and has a slightly different API.
-See [the docs](https://docs.tendermint.com/master/tendermint-core/light-client-protocol.html#http-proxy) for details.
+See [the docs](https://docs.tendermint.com/v0.33/tendermint-core/light-client-protocol.html#http-proxy) for details.
 
 ### Light Client
 
-We have a new, rewritten light client! You can 
+We have a new, rewritten light client! You can
 [read more](https://medium.com/tendermint/everything-you-need-to-know-about-the-tendermint-light-client-f80d03856f98)
-about the justifications and details behind this change. 
+about the justifications and details behind this change.
 
 Other user-relevant changes include:
 
 * The old `lite` package was removed; the new light client uses the `light` package.
-* The `Verifier` was broken up into two pieces: 
-    * Core verification logic (pure `VerifyX` functions) 
+* The `Verifier` was broken up into two pieces:
+    * Core verification logic (pure `VerifyX` functions)
     * `Client` object, which represents the complete light client
-* The RPC client can be found in the `/rpc` directory. 
+* The new light client stores headers and validator sets as `LightBlock`s
+* The RPC client can be found in the `/rpc` directory.
 * The HTTP(S) proxy is located in the `/proxy` directory.
 
 ### `state` Package
 
 * A new field `State.InitialHeight` has been added to record the initial chain height, which must be `1`
   (not `0`) if starting from height `1`. This can be configured via the genesis field `initial_height`.
-* The `state` package now has a `Store` interface. All functions in 
-  [state/store.go](https://github.com/tendermint/tendermint/blob/56911ee35298191c95ef1c7d3d5ec508237aaff4/state/store.go#L42-L42) 
+* The `state` package now has a `Store` interface. All functions in
+  [state/store.go](https://github.com/tendermint/tendermint/blob/56911ee35298191c95ef1c7d3d5ec508237aaff4/state/store.go#L42-L42)
   are now part of the interface. The interface returns errors on all methods and can be used by calling `state.NewStore(dbm.DB)`.
 
 ### `privval` Package
 
 All requests are now accompanied by the chain ID from the network.
-This is a optional field and can be ignored by key management systems. 
-It is recommended to check the chain ID if using the same key management system for multiple chains.
+This is a optional field and can be ignored by key management systems;
+however, if you are using the same key management system for multiple different
+blockchains, we recommend that you check the chain ID.
+
 
 ### RPC
 
-`/unsafe_start_cpu_profiler`, `/unsafe_stop_cpu_profiler` and
-`/unsafe_write_heap_profile` were removed. 
-For profiling, please use the pprof server, which can
-be enabled through `--rpc.pprof_laddr=X` flag or `pprof_laddr=X` config setting
-in the rpc section.
+* `/unsafe_start_cpu_profiler`, `/unsafe_stop_cpu_profiler` and
+  `/unsafe_write_heap_profile` were removed.
+   For profiling, please use the pprof server, which can
+  be enabled through `--rpc.pprof_laddr=X` flag or `pprof_laddr=X` config setting
+  in the rpc section.
+* The `Content-Type` header returned on RPC calls is now (correctly) set as `application/json`.
+
+### Version
+
+Version is now set through Go linker flags `ld_flags`. Applications that are using tendermint as a library should set this at compile time.
+
+Example:
+
+```sh
+go install -mod=readonly -ldflags "-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(go list -m github.com/tendermint/tendermint | sed  's/ /\@/g') -s -w " -trimpath ./cmd
+```
+
+Additionally, the exported constant `version.Version` is now `version.TMCoreSemVer`.
 
 ## v0.33.4
 
@@ -283,7 +313,7 @@ Evidence Params has been changed to include duration.
 ### RPC Changes
 
 * `/validators` is now paginated (default: 30 vals per page)
-* `/block_results` response format updated [see RPC docs for details](https://docs.tendermint.com/master/rpc/#/Info/block_results)
+* `/block_results` response format updated [see RPC docs for details](https://docs.tendermint.com/v0.33/rpc/#/Info/block_results)
 * Event suffix has been removed from the ID in event responses
 * IDs are now integers not `json-client-XYZ`
 
@@ -402,11 +432,11 @@ the compilation tag:
 
 Use `cleveldb` tag instead of `gcc` to compile Tendermint with CLevelDB or
 use `make build_c` / `make install_c` (full instructions can be found at
-<https://tendermint.com/docs/introduction/install.html#compile-with-cleveldb-support>)
+<https://docs.tendermint.com/v0.33/introduction/install.html#compile-with-cleveldb-support>)
 
 ## v0.31.0
 
-This release contains a breaking change to the behaviour of the pubsub system.
+This release contains a breaking change to the behavior of the pubsub system.
 It also contains some minor breaking changes in the Go API and ABCI.
 There are no changes to the block or p2p protocols, so v0.31.0 should work fine
 with blockchains created from the v0.30 series.
@@ -424,7 +454,7 @@ In this case, the WS client will receive an error with description:
   "error": {
     "code": -32000,
     "msg": "Server error",
-    "data": "subscription was cancelled (reason: client is not pulling messages fast enough)" // or "subscription was cancelled (reason: Tendermint exited)"
+    "data": "subscription was canceled (reason: client is not pulling messages fast enough)" // or "subscription was canceled (reason: Tendermint exited)"
   }
 }
 
@@ -477,14 +507,14 @@ due to changes in how various data structures are hashed.
 Any implementations of Tendermint blockchain verification, including lite clients,
 will need to be updated. For specific details:
 
-* [Merkle tree](https://github.com/tendermint/spec/blob/master/spec/blockchain/encoding.md#merkle-trees)
-* [ConsensusParams](https://github.com/tendermint/spec/blob/master/spec/blockchain/state.md#consensusparams)
+* [Merkle tree](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/blockchain/encoding.md#merkle-trees)
+* [ConsensusParams](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/blockchain/state.md#consensusparams)
 
 There was also a small change to field ordering in the vote struct. Any
 implementations of an out-of-process validator (like a Key-Management Server)
 will need to be updated. For specific details:
 
-* [Vote](https://github.com/tendermint/spec/blob/master/spec/consensus/signing.md#votes)
+* [Vote](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/consensus/signing.md#votes)
 
 Finally, the proposer selection algorithm continues to evolve. See the
 [work-in-progress
@@ -605,7 +635,7 @@ to `timeout_propose = "3s"`.
 
 ### RPC Changes
 
-The default behaviour of `/abci_query` has been changed to not return a proof,
+The default behavior of `/abci_query` has been changed to not return a proof,
 and the name of the parameter that controls this has been changed from `trusted`
 to `prove`. To get proofs with your queries, ensure you set `prove=true`.
 

Vagrantfile

@@ -33,10 +33,10 @@ Vagrant.configure("2") do |config|
     usermod -aG docker vagrant
 
     # install go
-    wget -q https://dl.google.com/go/go1.14.linux-amd64.tar.gz
-    tar -xvf go1.14.linux-amd64.tar.gz
+    wget -q https://dl.google.com/go/go1.15.linux-amd64.tar.gz
+    tar -xvf go1.15.linux-amd64.tar.gz
     mv go /usr/local
-    rm -f go1.14.linux-amd64.tar.gz
+    rm -f go1.15.linux-amd64.tar.gz
 
     # install nodejs (for docs)
     curl -sL https://deb.nodesource.com/setup_11.x | bash -

abci/README.md

@@ -19,7 +19,7 @@ To get up and running quickly, see the [getting started guide](../docs/app-dev/g
 
 A detailed description of the ABCI methods and message types is contained in:
 
-- [The main spec](https://github.com/tendermint/spec/blob/master/spec/abci/abci.md)
+- [The main spec](https://github.com/tendermint/tendermint/blob/v0.34.x/spec/abci/abci.md)
 - [A protobuf file](./types/types.proto)
 - [A Go interface](./types/application.go)
 

abci/client/client.go

@@ -74,20 +74,22 @@ func NewClient(addr, transport string, mustConnect bool) (client Client, err err
 	return
 }
 
-//----------------------------------------
-
 type Callback func(*types.Request, *types.Response)
 
-//----------------------------------------
-
 type ReqRes struct {
 	*types.Request
 	*sync.WaitGroup
 	*types.Response // Not set atomically, so be sure to use WaitGroup.
 
-	mtx  tmsync.Mutex
-	done bool                  // Gets set to true once *after* WaitGroup.Done().
-	cb   func(*types.Response) // A single callback that may be set.
+	mtx tmsync.Mutex
+
+	// callbackInvoked as a variable to track if the callback was already
+	// invoked during the regular execution of the request. This variable
+	// allows clients to set the callback simultaneously without potentially
+	// invoking the callback twice by accident, once when 'SetCallback' is
+	// called and once during the normal request.
+	callbackInvoked bool
+	cb              func(*types.Response) // A single callback that may be set.
 }
 
 func NewReqRes(req *types.Request) *ReqRes {
@@ -96,39 +98,49 @@ func NewReqRes(req *types.Request) *ReqRes {
 		WaitGroup: waitGroup1(),
 		Response:  nil,
 
-		done: false,
-		cb:   nil,
+		callbackInvoked: false,
+		cb:              nil,
 	}
 }
 
-// Sets the callback for this ReqRes atomically.
-// If reqRes is already done, calls cb immediately.
-// NOTE: reqRes.cb should not change if reqRes.done.
-// NOTE: only one callback is supported.
-func (reqRes *ReqRes) SetCallback(cb func(res *types.Response)) {
-	reqRes.mtx.Lock()
+// Sets sets the callback. If reqRes is already done, it will call the cb
+// immediately. Note, reqRes.cb should not change if reqRes.done and only one
+// callback is supported.
+func (r *ReqRes) SetCallback(cb func(res *types.Response)) {
+	r.mtx.Lock()
 
-	if reqRes.done {
-		reqRes.mtx.Unlock()
-		cb(reqRes.Response)
+	if r.callbackInvoked {
+		r.mtx.Unlock()
+		cb(r.Response)
 		return
 	}
 
-	reqRes.cb = cb
-	reqRes.mtx.Unlock()
+	r.cb = cb
+	r.mtx.Unlock()
 }
 
-func (reqRes *ReqRes) GetCallback() func(*types.Response) {
-	reqRes.mtx.Lock()
-	defer reqRes.mtx.Unlock()
-	return reqRes.cb
+// InvokeCallback invokes a thread-safe execution of the configured callback
+// if non-nil.
+func (r *ReqRes) InvokeCallback() {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	if r.cb != nil {
+		r.cb(r.Response)
+	}
+	r.callbackInvoked = true
 }
 
-// NOTE: it should be safe to read reqRes.cb without locks after this.
-func (reqRes *ReqRes) SetDone() {
-	reqRes.mtx.Lock()
-	reqRes.done = true
-	reqRes.mtx.Unlock()
+// GetCallback returns the configured callback of the ReqRes object which may be
+// nil. Note, it is not safe to concurrently call this in cases where it is
+// marked done and SetCallback is called before calling GetCallback as that
+// will invoke the callback twice and create a potential race condition.
+//
+// ref: https://github.com/tendermint/tendermint/issues/5439
+func (r *ReqRes) GetCallback() func(*types.Response) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	return r.cb
 }
 
 func waitGroup1() (wg *sync.WaitGroup) {

abci/client/grpc_client.go

@@ -3,6 +3,7 @@ package abcicli
 import (
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
 	"golang.org/x/net/context"
@@ -22,8 +23,9 @@ type grpcClient struct {
 	service.BaseService
 	mustConnect bool
 
-	client types.ABCIApplicationClient
-	conn   *grpc.ClientConn
+	client   types.ABCIApplicationClient
+	conn     *grpc.ClientConn
+	chReqRes chan *ReqRes // dispatches "async" responses to callbacks *in order*, needed by mempool
 
 	mtx   tmsync.Mutex
 	addr  string
@@ -35,6 +37,13 @@ func NewGRPCClient(addr string, mustConnect bool) Client {
 	cli := &grpcClient{
 		addr:        addr,
 		mustConnect: mustConnect,
+		// Buffering the channel is needed to make calls appear asynchronous,
+		// which is required when the caller makes multiple async calls before
+		// processing callbacks (e.g. due to holding locks). 64 means that a
+		// caller can make up to 64 async calls before a callback must be
+		// processed (otherwise it deadlocks). It also means that we can make 64
+		// gRPC calls while processing a slow callback at the channel head.
+		chReqRes: make(chan *ReqRes, 64),
 	}
 	cli.BaseService = *service.NewBaseService(nil, "grpcClient", cli)
 	return cli
@@ -48,8 +57,37 @@ func (cli *grpcClient) OnStart() error {
 	if err := cli.BaseService.OnStart(); err != nil {
 		return err
 	}
+
+	// This processes asynchronous request/response messages and dispatches
+	// them to callbacks.
+	go func() {
+		// Use a separate function to use defer for mutex unlocks (this handles panics)
+		callCb := func(reqres *ReqRes) {
+			cli.mtx.Lock()
+			defer cli.mtx.Unlock()
+
+			reqres.Done()
+
+			// Notify client listener if set
+			if cli.resCb != nil {
+				cli.resCb(reqres.Request, reqres.Response)
+			}
+
+			// Notify reqRes listener if set
+			reqres.InvokeCallback()
+		}
+		for reqres := range cli.chReqRes {
+			if reqres != nil {
+				callCb(reqres)
+			} else {
+				cli.Logger.Error("Received nil reqres")
+			}
+		}
+	}()
+
 RETRY_LOOP:
 	for {
+		//nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option.
 		conn, err := grpc.Dial(cli.addr, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc))
 		if err != nil {
 			if cli.mustConnect {
@@ -85,14 +123,15 @@ func (cli *grpcClient) OnStop() {
 	if cli.conn != nil {
 		cli.conn.Close()
 	}
+	close(cli.chReqRes)
 }
 
 func (cli *grpcClient) StopForError(err error) {
-	cli.mtx.Lock()
 	if !cli.IsRunning() {
 		return
 	}
 
+	cli.mtx.Lock()
 	if cli.err == nil {
 		cli.err = err
 	}
@@ -261,46 +300,60 @@ func (cli *grpcClient) ApplySnapshotChunkAsync(params types.RequestApplySnapshot
 	return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_ApplySnapshotChunk{ApplySnapshotChunk: res}})
 }
 
+// finishAsyncCall creates a ReqRes for an async call, and immediately populates it
+// with the response. We don't complete it until it's been ordered via the channel.
 func (cli *grpcClient) finishAsyncCall(req *types.Request, res *types.Response) *ReqRes {
 	reqres := NewReqRes(req)
-	reqres.Response = res // Set response
-	reqres.Done()         // Release waiters
-	reqres.SetDone()      // so reqRes.SetCallback will run the callback
-
-	// goroutine for callbacks
-	go func() {
-		cli.mtx.Lock()
-		defer cli.mtx.Unlock()
-
-		// Notify client listener if set
-		if cli.resCb != nil {
-			cli.resCb(reqres.Request, res)
-		}
-
-		// Notify reqRes listener if set
-		if cb := reqres.GetCallback(); cb != nil {
-			cb(res)
-		}
-	}()
-
+	reqres.Response = res
+	cli.chReqRes <- reqres // use channel for async responses, since they must be ordered
 	return reqres
 }
 
+// finishSyncCall waits for an async call to complete. It is necessary to call all
+// sync calls asynchronously as well, to maintain call and response ordering via
+// the channel, and this method will wait until the async call completes.
+func (cli *grpcClient) finishSyncCall(reqres *ReqRes) *types.Response {
+	// It's possible that the callback is called twice, since the callback can
+	// be called immediately on SetCallback() in addition to after it has been
+	// set. This is because completing the ReqRes happens in a separate critical
+	// section from the one where the callback is called: there is a race where
+	// SetCallback() is called between completing the ReqRes and dispatching the
+	// callback.
+	//
+	// We also buffer the channel with 1 response, since SetCallback() will be
+	// called synchronously if the reqres is already completed, in which case
+	// it will block on sending to the channel since it hasn't gotten around to
+	// receiving from it yet.
+	//
+	// ReqRes should really handle callback dispatch internally, to guarantee
+	// that it's only called once and avoid the above race conditions.
+	var once sync.Once
+	ch := make(chan *types.Response, 1)
+	reqres.SetCallback(func(res *types.Response) {
+		once.Do(func() {
+			ch <- res
+		})
+	})
+	return <-ch
+}
+
 //----------------------------------------
 
 func (cli *grpcClient) FlushSync() error {
-	return nil
+	reqres := cli.FlushAsync()
+	cli.finishSyncCall(reqres).GetFlush()
+	return cli.Error()
 }
 
 func (cli *grpcClient) EchoSync(msg string) (*types.ResponseEcho, error) {
 	reqres := cli.EchoAsync(msg)
 	// StopForError should already have been called if error is set
-	return reqres.Response.GetEcho(), cli.Error()
+	return cli.finishSyncCall(reqres).GetEcho(), cli.Error()
 }
 
 func (cli *grpcClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) {
 	reqres := cli.InfoAsync(req)
-	return reqres.Response.GetInfo(), cli.Error()
+	return cli.finishSyncCall(reqres).GetInfo(), cli.Error()
 }
 
 func (cli *grpcClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) {
@@ -310,57 +363,57 @@ func (cli *grpcClient) SetOptionSync(req types.RequestSetOption) (*types.Respons
 
 func (cli *grpcClient) DeliverTxSync(params types.RequestDeliverTx) (*types.ResponseDeliverTx, error) {
 	reqres := cli.DeliverTxAsync(params)
-	return reqres.Response.GetDeliverTx(), cli.Error()
+	return cli.finishSyncCall(reqres).GetDeliverTx(), cli.Error()
 }
 
 func (cli *grpcClient) CheckTxSync(params types.RequestCheckTx) (*types.ResponseCheckTx, error) {
 	reqres := cli.CheckTxAsync(params)
-	return reqres.Response.GetCheckTx(), cli.Error()
+	return cli.finishSyncCall(reqres).GetCheckTx(), cli.Error()
 }
 
 func (cli *grpcClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) {
 	reqres := cli.QueryAsync(req)
-	return reqres.Response.GetQuery(), cli.Error()
+	return cli.finishSyncCall(reqres).GetQuery(), cli.Error()
 }
 
 func (cli *grpcClient) CommitSync() (*types.ResponseCommit, error) {
 	reqres := cli.CommitAsync()
-	return reqres.Response.GetCommit(), cli.Error()
+	return cli.finishSyncCall(reqres).GetCommit(), cli.Error()
 }
 
 func (cli *grpcClient) InitChainSync(params types.RequestInitChain) (*types.ResponseInitChain, error) {
 	reqres := cli.InitChainAsync(params)
-	return reqres.Response.GetInitChain(), cli.Error()
+	return cli.finishSyncCall(reqres).GetInitChain(), cli.Error()
 }
 
 func (cli *grpcClient) BeginBlockSync(params types.RequestBeginBlock) (*types.ResponseBeginBlock, error) {
 	reqres := cli.BeginBlockAsync(params)
-	return reqres.Response.GetBeginBlock(), cli.Error()
+	return cli.finishSyncCall(reqres).GetBeginBlock(), cli.Error()
 }
 
 func (cli *grpcClient) EndBlockSync(params types.RequestEndBlock) (*types.ResponseEndBlock, error) {
 	reqres := cli.EndBlockAsync(params)
-	return reqres.Response.GetEndBlock(), cli.Error()
+	return cli.finishSyncCall(reqres).GetEndBlock(), cli.Error()
 }
 
 func (cli *grpcClient) ListSnapshotsSync(params types.RequestListSnapshots) (*types.ResponseListSnapshots, error) {
 	reqres := cli.ListSnapshotsAsync(params)
-	return reqres.Response.GetListSnapshots(), cli.Error()
+	return cli.finishSyncCall(reqres).GetListSnapshots(), cli.Error()
 }
 
 func (cli *grpcClient) OfferSnapshotSync(params types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) {
 	reqres := cli.OfferSnapshotAsync(params)
-	return reqres.Response.GetOfferSnapshot(), cli.Error()
+	return cli.finishSyncCall(reqres).GetOfferSnapshot(), cli.Error()
 }
 
 func (cli *grpcClient) LoadSnapshotChunkSync(
 	params types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
 	reqres := cli.LoadSnapshotChunkAsync(params)
-	return reqres.Response.GetLoadSnapshotChunk(), cli.Error()
+	return cli.finishSyncCall(reqres).GetLoadSnapshotChunk(), cli.Error()
 }
 
 func (cli *grpcClient) ApplySnapshotChunkSync(
 	params types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
 	reqres := cli.ApplySnapshotChunkAsync(params)
-	return reqres.Response.GetApplySnapshotChunk(), cli.Error()
+	return cli.finishSyncCall(reqres).GetApplySnapshotChunk(), cli.Error()
 }

abci/client/local_client.go

@@ -20,6 +20,12 @@ type localClient struct {
 	Callback
 }
 
+var _ Client = (*localClient)(nil)
+
+// NewLocalClient creates a local client, which will be directly calling the
+// methods of the given app.
+//
+// Both Async and Sync methods ignore the given context.Context parameter.
 func NewLocalClient(mtx *tmsync.Mutex, app types.Application) Client {
 	if mtx == nil {
 		mtx = new(tmsync.Mutex)
@@ -321,12 +327,13 @@ func (app *localClient) ApplySnapshotChunkSync(
 
 func (app *localClient) callback(req *types.Request, res *types.Response) *ReqRes {
 	app.Callback(req, res)
-	return newLocalReqRes(req, res)
+	rr := newLocalReqRes(req, res)
+	rr.callbackInvoked = true
+	return rr
 }
 
 func newLocalReqRes(req *types.Request, res *types.Response) *ReqRes {
 	reqRes := NewReqRes(req)
 	reqRes.Response = res
-	reqRes.SetDone()
 	return reqRes
 }

abci/client/socket_client.go

@@ -212,9 +212,7 @@ func (cli *socketClient) didRecvResponse(res *types.Response) error {
 	//
 	// NOTE: It is possible this callback isn't set on the reqres object. At this
 	// point, in which case it will be called after, when it is set.
-	if cb := reqres.GetCallback(); cb != nil {
-		cb(res)
-	}
+	reqres.InvokeCallback()
 
 	return nil
 }

abci/client/socket_client_test.go

@@ -2,6 +2,7 @@ package abcicli_test
 
 import (
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
@@ -118,3 +119,71 @@ func (slowApp) BeginBlock(req types.RequestBeginBlock) types.ResponseBeginBlock
 	time.Sleep(200 * time.Millisecond)
 	return types.ResponseBeginBlock{}
 }
+
+// TestCallbackInvokedWhenSetLaet ensures that the callback is invoked when
+// set after the client completes the call into the app. Currently this
+// test relies on the callback being allowed to be invoked twice if set multiple
+// times, once when set early and once when set late.
+func TestCallbackInvokedWhenSetLate(t *testing.T) {
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	app := blockedABCIApplication{
+		wg: wg,
+	}
+	_, c := setupClientServer(t, app)
+	reqRes := c.CheckTxAsync(types.RequestCheckTx{})
+
+	done := make(chan struct{})
+	cb := func(_ *types.Response) {
+		close(done)
+	}
+	reqRes.SetCallback(cb)
+	app.wg.Done()
+	<-done
+
+	var called bool
+	cb = func(_ *types.Response) {
+		called = true
+	}
+	reqRes.SetCallback(cb)
+	require.True(t, called)
+}
+
+type blockedABCIApplication struct {
+	wg *sync.WaitGroup
+	types.BaseApplication
+}
+
+func (b blockedABCIApplication) CheckTx(r types.RequestCheckTx) types.ResponseCheckTx {
+	b.wg.Wait()
+	return b.BaseApplication.CheckTx(r)
+}
+
+// TestCallbackInvokedWhenSetEarly ensures that the callback is invoked when
+// set before the client completes the call into the app.
+func TestCallbackInvokedWhenSetEarly(t *testing.T) {
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+	app := blockedABCIApplication{
+		wg: wg,
+	}
+	_, c := setupClientServer(t, app)
+	reqRes := c.CheckTxAsync(types.RequestCheckTx{})
+
+	done := make(chan struct{})
+	cb := func(_ *types.Response) {
+		close(done)
+	}
+	reqRes.SetCallback(cb)
+	app.wg.Done()
+
+	called := func() bool {
+		select {
+		case <-done:
+			return true
+		default:
+			return false
+		}
+	}
+	require.Eventually(t, called, time.Second, time.Millisecond*25)
+}

abci/example/example_test.go

@@ -130,7 +130,7 @@ func dialerFunc(ctx context.Context, addr string) (net.Conn, error) {
 
 func testGRPCSync(t *testing.T, app types.ABCIApplicationServer) {
 	numDeliverTxs := 2000
-	socketFile := fmt.Sprintf("test-%08x.sock", rand.Int31n(1<<30))
+	socketFile := fmt.Sprintf("/tmp/test-%08x.sock", rand.Int31n(1<<30))
 	defer os.Remove(socketFile)
 	socket := fmt.Sprintf("unix://%v", socketFile)
 
@@ -148,6 +148,7 @@ func testGRPCSync(t *testing.T, app types.ABCIApplicationServer) {
 	})
 
 	// Connect to the socket
+	//nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option.
 	conn, err := grpc.Dial(socket, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc))
 	if err != nil {
 		t.Fatalf("Error dialing GRPC server: %v", err.Error())

abci/example/kvstore/helpers.go

@@ -10,7 +10,7 @@ import (
 func RandVal(i int) types.ValidatorUpdate {
 	pubkey := tmrand.Bytes(32)
 	power := tmrand.Uint16() + 1
-	v := types.Ed25519ValidatorUpdate(pubkey, int64(power))
+	v := types.UpdateValidator(pubkey, int64(power), "")
 	return v
 }
 

abci/example/kvstore/kvstore_test.go

@@ -129,7 +129,7 @@ func TestValUpdates(t *testing.T) {
 	total := 10
 	nInit := 5
 	vals := RandVals(total)
-	// iniitalize with the first nInit
+	// initialize with the first nInit
 	kvstore.InitChain(types.RequestInitChain{
 		Validators: vals[:nInit],
 	})
@@ -162,7 +162,7 @@ func TestValUpdates(t *testing.T) {
 
 	makeApplyBlock(t, kvstore, 2, diff, tx1, tx2, tx3)
 
-	vals1 = append(vals[:nInit-2], vals[nInit+1]) // nolint: gocritic
+	vals1 = append(vals[:nInit-2], vals[nInit+1]) //nolint: gocritic
 	vals2 = kvstore.Validators()
 	valsEqual(t, vals1, vals2)
 
@@ -293,7 +293,7 @@ func TestClientServer(t *testing.T) {
 
 	// set up grpc app
 	kvstore = NewApplication()
-	gclient, gserver, err := makeGRPCClientServer(kvstore, "kvstore-grpc")
+	gclient, gserver, err := makeGRPCClientServer(kvstore, "/tmp/kvstore-grpc")
 	require.NoError(t, err)
 
 	t.Cleanup(func() {

abci/example/kvstore/persistent_kvstore.go

@@ -212,7 +212,7 @@ func isValidatorTx(tx []byte) bool {
 func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.ResponseDeliverTx {
 	tx = tx[len(ValidatorSetChangePrefix):]
 
-	//get the pubkey and power
+	//  get the pubkey and power
 	pubKeyAndPower := strings.Split(string(tx), "!")
 	if len(pubKeyAndPower) != 2 {
 		return types.ResponseDeliverTx{
@@ -238,16 +238,16 @@ func (app *PersistentKVStoreApplication) execValidatorTx(tx []byte) types.Respon
 	}
 
 	// update
-	return app.updateValidator(types.Ed25519ValidatorUpdate(pubkey, power))
+	return app.updateValidator(types.UpdateValidator(pubkey, power, ""))
 }
 
 // add, update, or remove a validator
 func (app *PersistentKVStoreApplication) updateValidator(v types.ValidatorUpdate) types.ResponseDeliverTx {
-	key := []byte("val:" + string(v.PubKey.GetEd25519()))
 	pubkey, err := cryptoenc.PubKeyFromProto(v.PubKey)
 	if err != nil {
 		panic(fmt.Errorf("can't decode public key: %w", err))
 	}
+	key := []byte("val:" + string(pubkey.Bytes()))
 
 	if v.Power == 0 {
 		// remove validator

abci/server/server.go

@@ -2,9 +2,8 @@
 Package server is used to start a new ABCI server.
 
 It contains two server implementation:
- * gRPC server
- * socket server
-
+  - gRPC server
+  - socket server
 */
 package server
 

abci/tests/server/client.go

@@ -16,7 +16,7 @@ func InitChain(client abcicli.Client) error {
 	for i := 0; i < total; i++ {
 		pubkey := tmrand.Bytes(33)
 		power := tmrand.Int()
-		vals[i] = types.Ed25519ValidatorUpdate(pubkey, int64(power))
+		vals[i] = types.UpdateValidator(pubkey, int64(power), "")
 	}
 	_, err := client.InitChainSync(types.RequestInitChain{
 		Validators: vals,

abci/tests/test_app/main.go

@@ -69,7 +69,7 @@ func testCounter() {
 	}()
 
 	if err := ensureABCIIsUp(abciType, maxABCIConnectTries); err != nil {
-		log.Fatalf("echo failed: %v", err)
+		log.Fatalf("echo failed: %v", err) //nolint:gocritic
 	}
 
 	client := startClient(abciType)

abci/types/pubkey.go

@@ -1,16 +1,16 @@
 package types
 
 import (
+	fmt "fmt"
+
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
-)
-
-const (
-	PubKeyEd25519 = "ed25519"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
 )
 
 func Ed25519ValidatorUpdate(pk []byte, power int64) ValidatorUpdate {
 	pke := ed25519.PubKey(pk)
+
 	pkp, err := cryptoenc.PubKeyToProto(pke)
 	if err != nil {
 		panic(err)
@@ -22,3 +22,23 @@ func Ed25519ValidatorUpdate(pk []byte, power int64) ValidatorUpdate {
 		Power:  power,
 	}
 }
+
+func UpdateValidator(pk []byte, power int64, keyType string) ValidatorUpdate {
+	switch keyType {
+	case "", ed25519.KeyType:
+		return Ed25519ValidatorUpdate(pk, power)
+	case secp256k1.KeyType:
+		pke := secp256k1.PubKey(pk)
+		pkp, err := cryptoenc.PubKeyToProto(pke)
+		if err != nil {
+			panic(err)
+		}
+		return ValidatorUpdate{
+			// Address:
+			PubKey: pkp,
+			Power:  power,
+		}
+	default:
+		panic(fmt.Sprintf("key type %s not supported", keyType))
+	}
+}

abci/types/result.go

@@ -42,7 +42,7 @@ func (r ResponseQuery) IsErr() bool {
 }
 
 //---------------------------------------------------------------------------
-// override JSON marshalling so we emit defaults (ie. disable omitempty)
+// override JSON marshaling so we emit defaults (ie. disable omitempty)
 
 var (
 	jsonpbMarshaller = jsonpb.Marshaler{

abci/types/util.go

@@ -27,7 +27,5 @@ func (v ValidatorUpdates) Less(i, j int) bool {
 }
 
 func (v ValidatorUpdates) Swap(i, j int) {
-	v1 := v[i]
-	v[i] = v[j]
-	v[j] = v1
+	v[i], v[j] = v[j], v[i]
 }

behaviour/doc.go

@@ -8,35 +8,34 @@ There are four different behaviours a reactor can report.
 
 1. bad message
 
-type badMessage struct {
-	explanation string
-}
+	type badMessage struct {
+		explanation string
+	}
 
-This message will request the peer be stopped for an error
+# This message will request the peer be stopped for an error
 
 2. message out of order
 
-type messageOutOfOrder struct {
-	explanation string
-}
+	type messageOutOfOrder struct {
+		explanation string
+	}
 
-This message will request the peer be stopped for an error
+# This message will request the peer be stopped for an error
 
 3. consesnsus Vote
 
-type consensusVote struct {
-	explanation string
-}
+	type consensusVote struct {
+		explanation string
+	}
 
-This message will request the peer be marked as good
+# This message will request the peer be marked as good
 
 4. block part
 
-type blockPart struct {
-	explanation string
-}
+	type blockPart struct {
+		explanation string
+	}
 
 This message will request the peer be marked as good
-
 */
 package behaviour

blockchain/msgs_test.go

@@ -78,7 +78,7 @@ func TestBcStatusResponseMessageValidateBasic(t *testing.T) {
 	}
 }
 
-// nolint:lll // ignore line length in tests
+//nolint:lll // ignore line length in tests
 func TestBlockchainMessageVectors(t *testing.T) {
 	block := types.MakeBlock(int64(3), []types.Tx{types.Tx("Hello World")}, nil, nil)
 	block.Version.Block = 11 // overwrite updated protocol version
@@ -97,7 +97,7 @@ func TestBlockchainMessageVectors(t *testing.T) {
 			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
 			"0a0a08ffffffffffffffff7f"},
 		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1ab5010ab2010a5b0a02080b1803220b088092b8c398feffffff012a0212003a20c4da88e876062aa1543400d50d0eaa0dac88096057949cfb7bca7f3a48c04bf96a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855122f0a0b48656c6c6f20576f726c641220c4da88e876062aa1543400d50d0eaa0dac88096057949cfb7bca7f3a48c04bf91a221220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1a700a6e0a5b0a02080b1803220b088092b8c398feffffff012a0212003a20c4da88e876062aa1543400d50d0eaa0dac88096057949cfb7bca7f3a48c04bf96a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855120d0a0b48656c6c6f20576f726c641a00"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
 			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
 		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{

blockchain/v0/pool.go

@@ -32,6 +32,7 @@ const (
 	maxTotalRequesters        = 600
 	maxPendingRequests        = maxTotalRequesters
 	maxPendingRequestsPerPeer = 20
+	requestRetrySeconds       = 30
 
 	// Minimum recv rate to ensure we're receiving blocks from a peer fast
 	// enough. If a peer is not sending us data at at least that rate, we
@@ -410,6 +411,7 @@ func (pool *BlockPool) sendError(err error, peerID p2p.ID) {
 }
 
 // for debugging purposes
+//
 //nolint:unused
 func (pool *BlockPool) debug() string {
 	pool.mtx.Lock()
@@ -509,7 +511,7 @@ type bpRequester struct {
 	pool       *BlockPool
 	height     int64
 	gotBlockCh chan struct{}
-	redoCh     chan p2p.ID //redo may send multitime, add peerId to identify repeat
+	redoCh     chan p2p.ID // redo may send multitime, add peerId to identify repeat
 
 	mtx    tmsync.Mutex
 	peerID p2p.ID
@@ -601,7 +603,7 @@ OUTER_LOOP:
 			}
 			peer = bpr.pool.pickIncrAvailablePeer(bpr.height)
 			if peer == nil {
-				//log.Info("No peers available", "height", height)
+				bpr.Logger.Debug("No peers currently available; will retry shortly", "height", bpr.height)
 				time.Sleep(requestIntervalMS * time.Millisecond)
 				continue PICK_PEER_LOOP
 			}
@@ -611,6 +613,7 @@ OUTER_LOOP:
 		bpr.peerID = peer.id
 		bpr.mtx.Unlock()
 
+		to := time.NewTimer(requestRetrySeconds * time.Second)
 		// Send request and wait.
 		bpr.pool.sendRequest(bpr.height, peer.id)
 	WAIT_LOOP:
@@ -623,6 +626,11 @@ OUTER_LOOP:
 				return
 			case <-bpr.Quit():
 				return
+			case <-to.C:
+				bpr.Logger.Debug("Retrying block request after timeout", "height", bpr.height, "peer", bpr.peerID)
+				// Simulate a redo
+				bpr.reset()
+				continue OUTER_LOOP
 			case peerID := <-bpr.redoCh:
 				if peerID == bpr.peerID {
 					bpr.reset()

blockchain/v0/pool_test.go

@@ -22,7 +22,7 @@ type testPeer struct {
 	id        p2p.ID
 	base      int64
 	height    int64
-	inputChan chan inputData //make sure each peer's data is sequential
+	inputChan chan inputData // make sure each peer's data is sequential
 }
 
 type inputData struct {

blockchain/v0/reactor.go

@@ -140,7 +140,7 @@ func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	return []*p2p.ChannelDescriptor{
 		{
 			ID:                  BlockchainChannel,
-			Priority:            10,
+			Priority:            5,
 			SendQueueCapacity:   1000,
 			RecvBufferCapacity:  50 * 4096,
 			RecvMessageCapacity: bc.MaxMsgSize,
@@ -207,7 +207,7 @@ func (bcR *BlockchainReactor) respondToPeer(msg *bcproto.BlockRequest,
 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
 	msg, err := bc.DecodeMsg(msgBytes)
 	if err != nil {
-		bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
+		bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "err", err)
 		bcR.Switch.StopPeerForError(src, err)
 		return
 	}
@@ -256,8 +256,13 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 func (bcR *BlockchainReactor) poolRoutine(stateSynced bool) {
 
 	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
+	defer trySyncTicker.Stop()
+
 	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
+	defer statusUpdateTicker.Stop()
+
 	switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)
+	defer switchToConsensusTicker.Stop()
 
 	blocksSynced := uint64(0)
 
@@ -299,7 +304,7 @@ func (bcR *BlockchainReactor) poolRoutine(stateSynced bool) {
 
 			case <-statusUpdateTicker.C:
 				// ask for status updates
-				go bcR.BroadcastStatusRequest() // nolint: errcheck
+				go bcR.BroadcastStatusRequest() //nolint: errcheck
 
 			}
 		}
@@ -346,7 +351,7 @@ FOR_LOOP:
 
 			// See if there are any blocks to sync.
 			first, second := bcR.pool.PeekTwoBlocks()
-			//bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
+			// bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
 			if first == nil || second == nil {
 				// We need both to sync the first block.
 				continue FOR_LOOP
@@ -364,6 +369,12 @@ FOR_LOOP:
 			// currently necessary.
 			err := state.Validators.VerifyCommitLight(
 				chainID, firstID, first.Height, second.LastCommit)
+
+			if err == nil {
+				// validate the block before we persist it
+				err = bcR.blockExec.ValidateBlock(state, first)
+			}
+
 			if err != nil {
 				bcR.Logger.Error("Error in validation", "err", err)
 				peerID := bcR.pool.RedoRequest(first.Height)
@@ -381,29 +392,29 @@ FOR_LOOP:
 					bcR.Switch.StopPeerForError(peer2, fmt.Errorf("blockchainReactor validation error: %v", err))
 				}
 				continue FOR_LOOP
-			} else {
-				bcR.pool.PopRequest()
-
-				// TODO: batch saves so we dont persist to disk every block
-				bcR.store.SaveBlock(first, firstParts, second.LastCommit)
-
-				// TODO: same thing for app - but we would need a way to
-				// get the hash without persisting the state
-				var err error
-				state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first)
-				if err != nil {
-					// TODO This is bad, are we zombie?
-					panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
-				}
-				blocksSynced++
-
-				if blocksSynced%100 == 0 {
-					lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
-					bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
-						"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
-					lastHundred = time.Now()
-				}
 			}
+
+			bcR.pool.PopRequest()
+
+			// TODO: batch saves so we dont persist to disk every block
+			bcR.store.SaveBlock(first, firstParts, second.LastCommit)
+
+			// TODO: same thing for app - but we would need a way to
+			// get the hash without persisting the state
+			state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first)
+			if err != nil {
+				// TODO This is bad, are we zombie?
+				panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+			}
+			blocksSynced++
+
+			if blocksSynced%100 == 0 {
+				lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
+				bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
+					"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
+				lastHundred = time.Now()
+			}
+
 			continue FOR_LOOP
 
 		case <-bcR.Quit():

blockchain/v0/reactor_test.go

@@ -70,7 +70,9 @@ func newBlockchainReactor(
 
 	blockDB := dbm.NewMemDB()
 	stateDB := dbm.NewMemDB()
-	stateStore := sm.NewStore(stateDB)
+	stateStore := sm.NewStore(stateDB, sm.StoreOptions{
+		DiscardABCIResponses: false,
+	})
 	blockStore := store.NewBlockStore(blockDB)
 
 	state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc)
@@ -83,7 +85,9 @@ func newBlockchainReactor(
 	// pool.height is determined from the store.
 	fastSync := true
 	db := dbm.NewMemDB()
-	stateStore = sm.NewStore(db)
+	stateStore = sm.NewStore(db, sm.StoreOptions{
+		DiscardABCIResponses: false,
+	})
 	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
 		mock.Mempool{}, sm.EmptyEvidencePool{})
 	if err = stateStore.Save(state); err != nil {
@@ -247,7 +251,7 @@ func TestBadBlockStopsPeer(t *testing.T) {
 		}
 	}
 
-	//at this time, reactors[0-3] is the newest
+	// at this time, reactors[0-3] is the newest
 	assert.Equal(t, 3, reactorPairs[1].reactor.Switch.Peers().Size())
 
 	// Mark reactorPairs[3] as an invalid peer. Fiddling with .store without a mutex is a data

blockchain/v1/reactor.go

@@ -100,7 +100,7 @@ func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *st
 	fsm := NewFSM(startHeight, bcR)
 	bcR.fsm = fsm
 	bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
-	//bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch)
+	// bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch)
 
 	return bcR
 }
@@ -253,8 +253,7 @@ func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
 	msg, err := bc.DecodeMsg(msgBytes)
 	if err != nil {
-		bcR.Logger.Error("error decoding message",
-			"src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
+		bcR.Logger.Error("error decoding message", "src", src, "chId", chID, "err", err)
 		_ = bcR.swReporter.Report(behaviour.BadMessage(src.ID(), err.Error()))
 		return
 	}
@@ -298,6 +297,16 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte)
 		}
 		bcR.Logger.Info("Received", "src", src, "height", bi.Height)
 		bcR.messagesForFSMCh <- msgForFSM
+	case *bcproto.NoBlockResponse:
+		msgForFSM := bcReactorMessage{
+			event: noBlockResponseEv,
+			data: bReactorEventData{
+				peerID: src.ID(),
+				height: msg.Height,
+			},
+		}
+		bcR.Logger.Debug("Peer does not have requested block", "peer", src, "height", msg.Height)
+		bcR.messagesForFSMCh <- msgForFSM
 
 	case *bcproto.StatusResponse:
 		// Got a peer status. Unverified.
@@ -525,8 +534,8 @@ func (bcR *BlockchainReactor) switchToConsensus() {
 // Called by FSM and pool:
 // - pool calls when it detects slow peer or when peer times out
 // - FSM calls when:
-//    - adding a block (addBlock) fails
-//    - reactor processing of a block reports failure and FSM sends back the peers of first and second blocks
+//   - adding a block (addBlock) fails
+//   - reactor processing of a block reports failure and FSM sends back the peers of first and second blocks
 func (bcR *BlockchainReactor) sendPeerError(err error, peerID p2p.ID) {
 	bcR.Logger.Info("sendPeerError:", "peer", peerID, "error", err)
 	msgData := bcFsmMessage{

blockchain/v1/reactor_fsm.go

@@ -74,6 +74,7 @@ const (
 	startFSMEv = iota + 1
 	statusResponseEv
 	blockResponseEv
+	noBlockResponseEv
 	processedBlockEv
 	makeRequestsEv
 	stopFSMEv
@@ -94,6 +95,9 @@ func (msg *bcReactorMessage) String() string {
 	case blockResponseEv:
 		dataStr = fmt.Sprintf("peer=%v block.height=%v length=%v",
 			msg.data.peerID, msg.data.block.Height, msg.data.length)
+	case noBlockResponseEv:
+		dataStr = fmt.Sprintf("peer=%v requested height=%v",
+			msg.data.peerID, msg.data.height)
 	case processedBlockEv:
 		dataStr = fmt.Sprintf("error=%v", msg.data.err)
 	case makeRequestsEv:
@@ -119,6 +123,8 @@ func (ev bReactorEvent) String() string {
 		return "statusResponseEv"
 	case blockResponseEv:
 		return "blockResponseEv"
+	case noBlockResponseEv:
+		return "noBlockResponseEv"
 	case processedBlockEv:
 		return "processedBlockEv"
 	case makeRequestsEv:
@@ -269,7 +275,10 @@ func init() {
 					return waitForPeer, err
 				}
 				return waitForBlock, err
+			case noBlockResponseEv:
+				fsm.logger.Error("peer does not have requested block", "peer", data.peerID)
 
+				return waitForBlock, nil
 			case processedBlockEv:
 				if data.err != nil {
 					first, second, _ := fsm.pool.FirstTwoBlocksAndPeers()

blockchain/v1/reactor_fsm_test.go

@@ -822,7 +822,7 @@ const (
 	maxRequestsPerPeerTest      = 20
 	maxTotalPendingRequestsTest = 600
 	maxNumPeersTest             = 1000
-	maxNumBlocksInChainTest     = 10000 //should be smaller than 9999999
+	maxNumBlocksInChainTest     = 10000 // should be smaller than 9999999
 )
 
 func makeCorrectTransitionSequenceWithRandomParameters() testFields {

blockchain/v1/reactor_test.go

@@ -102,7 +102,9 @@ func newBlockchainReactor(
 
 	blockDB := dbm.NewMemDB()
 	stateDB := dbm.NewMemDB()
-	stateStore := sm.NewStore(stateDB)
+	stateStore := sm.NewStore(stateDB, sm.StoreOptions{
+		DiscardABCIResponses: false,
+	})
 	blockStore := store.NewBlockStore(blockDB)
 
 	state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc)
@@ -115,7 +117,9 @@ func newBlockchainReactor(
 	// pool.height is determined from the store.
 	fastSync := true
 	db := dbm.NewMemDB()
-	stateStore = sm.NewStore(db)
+	stateStore = sm.NewStore(db, sm.StoreOptions{
+		DiscardABCIResponses: false,
+	})
 	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
 		mock.Mempool{}, sm.EmptyEvidencePool{})
 	if err = stateStore.Save(state); err != nil {
@@ -305,10 +309,10 @@ outerFor:
 		break
 	}
 
-	//at this time, reactors[0-3] is the newest
+	// at this time, reactors[0-3] is the newest
 	assert.Equal(t, numNodes-1, reactorPairs[1].bcR.Switch.Peers().Size())
 
-	//mark last reactorPair as an invalid peer
+	// mark last reactorPair as an invalid peer
 	reactorPairs[numNodes-1].bcR.store = otherChain.bcR.store
 
 	lastLogger := log.TestingLogger()

blockchain/v2/processor.go

@@ -17,6 +17,11 @@ type pcBlockVerificationFailure struct {
 	secondPeerID p2p.ID
 }
 
+func (e pcBlockVerificationFailure) String() string {
+	return fmt.Sprintf("pcBlockVerificationFailure{%d 1st peer: %v, 2nd peer: %v}",
+		e.height, e.firstPeerID, e.secondPeerID)
+}
+
 // successful block execution
 type pcBlockProcessed struct {
 	priorityNormal
@@ -24,6 +29,10 @@ type pcBlockProcessed struct {
 	peerID p2p.ID
 }
 
+func (e pcBlockProcessed) String() string {
+	return fmt.Sprintf("pcBlockProcessed{%d peer: %v}", e.height, e.peerID)
+}
+
 // processor has finished
 type pcFinished struct {
 	priorityNormal
@@ -87,9 +96,12 @@ func (state *pcState) synced() bool {
 }
 
 func (state *pcState) enqueue(peerID p2p.ID, block *types.Block, height int64) {
-	if _, ok := state.queue[height]; ok {
-		panic("duplicate block enqueued by processor")
+	if item, ok := state.queue[height]; ok {
+		panic(fmt.Sprintf(
+			"duplicate block %d (%X) enqueued by processor (sent by %v; existing block %X from %v)",
+			height, block.Hash(), peerID, item.block.Hash(), item.peerID))
 	}
+
 	state.queue[height] = queueItem{block: block, peerID: peerID}
 }
 
@@ -145,16 +157,20 @@ func (state *pcState) handle(event Event) (Event, error) {
 			}
 			return noOp, nil
 		}
-		first, second := firstItem.block, secondItem.block
 
-		firstParts := first.MakePartSet(types.BlockPartSizeBytes)
-		firstPartSetHeader := firstParts.Header()
-		firstID := types.BlockID{Hash: first.Hash(), PartSetHeader: firstPartSetHeader}
+		var (
+			first, second = firstItem.block, secondItem.block
+			firstParts    = first.MakePartSet(types.BlockPartSizeBytes)
+			firstID       = types.BlockID{Hash: first.Hash(), PartSetHeader: firstParts.Header()}
+		)
 
+		// verify if +second+ last commit "confirms" +first+ block
 		err = state.context.verifyCommit(tmState.ChainID, firstID, first.Height, second.LastCommit)
 		if err != nil {
 			state.purgePeer(firstItem.peerID)
-			state.purgePeer(secondItem.peerID)
+			if firstItem.peerID != secondItem.peerID {
+				state.purgePeer(secondItem.peerID)
+			}
 			return pcBlockVerificationFailure{
 					height: first.Height, firstPeerID: firstItem.peerID, secondPeerID: secondItem.peerID},
 				nil
@@ -170,7 +186,6 @@ func (state *pcState) handle(event Event) (Event, error) {
 		state.blocksSynced++
 
 		return pcBlockProcessed{height: first.Height, peerID: firstItem.peerID}, nil
-
 	}
 
 	return noOp, nil

blockchain/v2/reactor.go

@@ -187,7 +187,7 @@ type rTryPrunePeer struct {
 }
 
 func (e rTryPrunePeer) String() string {
-	return fmt.Sprintf(": %v", e.time)
+	return fmt.Sprintf("rTryPrunePeer{%v}", e.time)
 }
 
 // ticker event for scheduling block requests
@@ -197,7 +197,7 @@ type rTrySchedule struct {
 }
 
 func (e rTrySchedule) String() string {
-	return fmt.Sprintf(": %v", e.time)
+	return fmt.Sprintf("rTrySchedule{%v}", e.time)
 }
 
 // ticker for block processing
@@ -205,6 +205,10 @@ type rProcessBlock struct {
 	priorityNormal
 }
 
+func (e rProcessBlock) String() string {
+	return "rProcessBlock"
+}
+
 // reactor generated events based on blockchain related messages from peers:
 // blockResponse message received from a peer
 type bcBlockResponse struct {
@@ -215,6 +219,11 @@ type bcBlockResponse struct {
 	block  *types.Block
 }
 
+func (resp bcBlockResponse) String() string {
+	return fmt.Sprintf("bcBlockResponse{%d#%X (size: %d bytes) from %v at %v}",
+		resp.block.Height, resp.block.Hash(), resp.size, resp.peerID, resp.time)
+}
+
 // blockNoResponse message received from a peer
 type bcNoBlockResponse struct {
 	priorityNormal
@@ -223,6 +232,11 @@ type bcNoBlockResponse struct {
 	height int64
 }
 
+func (resp bcNoBlockResponse) String() string {
+	return fmt.Sprintf("bcNoBlockResponse{%v has no block at height %d at %v}",
+		resp.peerID, resp.height, resp.time)
+}
+
 // statusResponse message received from a peer
 type bcStatusResponse struct {
 	priorityNormal
@@ -232,12 +246,21 @@ type bcStatusResponse struct {
 	height int64
 }
 
+func (resp bcStatusResponse) String() string {
+	return fmt.Sprintf("bcStatusResponse{%v is at height %d (base: %d) at %v}",
+		resp.peerID, resp.height, resp.base, resp.time)
+}
+
 // new peer is connected
 type bcAddNewPeer struct {
 	priorityNormal
 	peerID p2p.ID
 }
 
+func (resp bcAddNewPeer) String() string {
+	return fmt.Sprintf("bcAddNewPeer{%v}", resp.peerID)
+}
+
 // existing peer is removed
 type bcRemovePeer struct {
 	priorityHigh
@@ -245,12 +268,20 @@ type bcRemovePeer struct {
 	reason interface{}
 }
 
+func (resp bcRemovePeer) String() string {
+	return fmt.Sprintf("bcRemovePeer{%v due to %v}", resp.peerID, resp.reason)
+}
+
 // resets the scheduler and processor state, e.g. following a switch from state syncing
 type bcResetState struct {
 	priorityHigh
 	state state.State
 }
 
+func (e bcResetState) String() string {
+	return fmt.Sprintf("bcResetState{%v}", e.state)
+}
+
 // Takes the channel as a parameter to avoid race conditions on r.events.
 func (r *BlockchainReactor) demux(events <-chan Event) {
 	var lastRate = 0.0
@@ -358,6 +389,10 @@ func (r *BlockchainReactor) demux(events <-chan Event) {
 			case scSchedulerFail:
 				r.logger.Error("Scheduler failure", "err", event.reason.Error())
 			case scPeersPruned:
+				// Remove peers from the processor.
+				for _, peerID := range event.peers {
+					r.processor.send(scPeerError{peerID: peerID, reason: errors.New("peer was pruned")})
+				}
 				r.logger.Debug("Pruned peers", "count", len(event.peers))
 			case noOpEvent:
 			default:
@@ -424,7 +459,7 @@ func (r *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
 	msg, err := bc.DecodeMsg(msgBytes)
 	if err != nil {
 		r.logger.Error("error decoding message",
-			"src", src.ID(), "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
+			"src", src.ID(), "chId", chID, "msg", msg, "err", err)
 		_ = r.reporter.Report(behaviour.BadMessage(src.ID(), err.Error()))
 		return
 	}
@@ -465,12 +500,12 @@ func (r *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
 		r.mtx.RUnlock()
 
 	case *bcproto.BlockResponse:
-		r.mtx.RLock()
 		bi, err := types.BlockFromProto(msg.Block)
 		if err != nil {
 			r.logger.Error("error transitioning block from protobuf", "err", err)
 			return
 		}
+		r.mtx.RLock()
 		if r.events != nil {
 			r.events <- bcBlockResponse{
 				peerID: src.ID(),
@@ -520,7 +555,7 @@ func (r *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
 	return []*p2p.ChannelDescriptor{
 		{
 			ID:                  BlockchainChannel,
-			Priority:            10,
+			Priority:            5,
 			SendQueueCapacity:   2000,
 			RecvBufferCapacity:  50 * 4096,
 			RecvMessageCapacity: bc.MaxMsgSize,

blockchain/v2/reactor_test.go

@@ -59,7 +59,9 @@ func (mp mockPeer) TrySend(byte, []byte) bool { return true }
 func (mp mockPeer) Set(string, interface{}) {}
 func (mp mockPeer) Get(string) interface{}  { return struct{}{} }
 
-//nolint:unused
+func (mp mockPeer) SetRemovalFailed()      {}
+func (mp mockPeer) GetRemovalFailed() bool { return false }
+
 type mockBlockStore struct {
 	blocks map[int64]*types.Block
 }
@@ -156,7 +158,9 @@ func newTestReactor(p testReactorParams) *BlockchainReactor {
 			panic(fmt.Errorf("error start app: %w", err))
 		}
 		db := dbm.NewMemDB()
-		stateStore := sm.NewStore(db)
+		stateStore := sm.NewStore(db, sm.StoreOptions{
+			DiscardABCIResponses: false,
+		})
 		appl = sm.NewBlockExecutor(stateStore, p.logger, proxyApp.Consensus(), mock.Mempool{}, sm.EmptyEvidencePool{})
 		if err = stateStore.Save(state); err != nil {
 			panic(err)
@@ -501,14 +505,18 @@ func newReactorStore(
 
 	stateDB := dbm.NewMemDB()
 	blockStore := store.NewBlockStore(dbm.NewMemDB())
-	stateStore := sm.NewStore(stateDB)
+	stateStore := sm.NewStore(stateDB, sm.StoreOptions{
+		DiscardABCIResponses: false,
+	})
 	state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc)
 	if err != nil {
 		panic(fmt.Errorf("error constructing state from genesis file: %w", err))
 	}
 
 	db := dbm.NewMemDB()
-	stateStore = sm.NewStore(db)
+	stateStore = sm.NewStore(db, sm.StoreOptions{
+		DiscardABCIResponses: false},
+	)
 	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
 		mock.Mempool{}, sm.EmptyEvidencePool{})
 	if err = stateStore.Save(state); err != nil {

blockchain/v2/routine.go

@@ -2,6 +2,7 @@ package v2
 
 import (
 	"fmt"
+	"strings"
 	"sync/atomic"
 
 	"github.com/Workiva/go-datastructures/queue"
@@ -11,6 +12,8 @@ import (
 
 type handleFunc = func(event Event) (Event, error)
 
+const historySize = 25
+
 // Routine is a structure that models a finite state machine as serialized
 // stream of events processed by a handle function. This Routine structure
 // handles the concurrency and messaging guarantees. Events are sent via
@@ -21,6 +24,7 @@ type Routine struct {
 	name    string
 	handle  handleFunc
 	queue   *queue.PriorityQueue
+	history []Event
 	out     chan Event
 	fin     chan error
 	rdy     chan struct{}
@@ -34,6 +38,7 @@ func newRoutine(name string, handleFunc handleFunc, bufferSize int) *Routine {
 		name:    name,
 		handle:  handleFunc,
 		queue:   queue.NewPriorityQueue(bufferSize, true),
+		history: make([]Event, 0, historySize),
 		out:     make(chan Event, bufferSize),
 		rdy:     make(chan struct{}, 1),
 		fin:     make(chan error, 1),
@@ -47,19 +52,29 @@ func (rt *Routine) setLogger(logger log.Logger) {
 	rt.logger = logger
 }
 
-// nolint:unused
 func (rt *Routine) setMetrics(metrics *Metrics) {
 	rt.metrics = metrics
 }
 
 func (rt *Routine) start() {
-	rt.logger.Info(fmt.Sprintf("%s: run\n", rt.name))
+	rt.logger.Info("routine start", "msg", log.NewLazySprintf("%s: run", rt.name))
 	running := atomic.CompareAndSwapUint32(rt.running, uint32(0), uint32(1))
 	if !running {
 		panic(fmt.Sprintf("%s is already running", rt.name))
 	}
 	close(rt.rdy)
 	defer func() {
+		if r := recover(); r != nil {
+			var (
+				b strings.Builder
+				j int
+			)
+			for i := len(rt.history) - 1; i >= 0; i-- {
+				fmt.Fprintf(&b, "%d: %+v\n", j, rt.history[i])
+				j++
+			}
+			panic(fmt.Sprintf("%v\nlast events:\n%v", r, b.String()))
+		}
 		stopped := atomic.CompareAndSwapUint32(rt.running, uint32(1), uint32(0))
 		if !stopped {
 			panic(fmt.Sprintf("%s is failed to stop", rt.name))
@@ -82,7 +97,19 @@ func (rt *Routine) start() {
 			return
 		}
 		rt.metrics.EventsOut.With("routine", rt.name).Add(1)
-		rt.logger.Debug(fmt.Sprintf("%s: produced %T %+v\n", rt.name, oEvent, oEvent))
+		rt.logger.Debug("routine start", "msg", log.NewLazySprintf("%s: produced %T %+v", rt.name, oEvent, oEvent))
+
+		// Skip rTrySchedule and rProcessBlock events as they clutter the history
+		// due to their frequency.
+		switch events[0].(type) {
+		case rTrySchedule:
+		case rProcessBlock:
+		default:
+			rt.history = append(rt.history, events[0].(Event))
+			if len(rt.history) > historySize {
+				rt.history = rt.history[1:]
+			}
+		}
 
 		rt.out <- oEvent
 	}
@@ -90,14 +117,14 @@ func (rt *Routine) start() {
 
 // XXX: look into returning OpError in the net package
 func (rt *Routine) send(event Event) bool {
-	rt.logger.Debug(fmt.Sprintf("%s: received %T %+v", rt.name, event, event))
+	rt.logger.Debug("routine send", "msg", log.NewLazySprintf("%s: received %T %+v", rt.name, event, event))
 	if !rt.isRunning() {
 		return false
 	}
 	err := rt.queue.Put(event)
 	if err != nil {
 		rt.metrics.EventsShed.With("routine", rt.name).Add(1)
-		rt.logger.Info(fmt.Sprintf("%s: send failed, queue was full/stopped \n", rt.name))
+		rt.logger.Error(fmt.Sprintf("%s: send failed, queue was full/stopped", rt.name))
 		return false
 	}
 
@@ -122,7 +149,7 @@ func (rt *Routine) stop() {
 		return
 	}
 
-	rt.logger.Info(fmt.Sprintf("%s: stop\n", rt.name))
+	rt.logger.Info("routine stop", "msg", log.NewLazySprintf("%s: stop", rt.name))
 	rt.queue.Dispose() // this should block until all queue items are free?
 }
 

blockchain/v2/scheduler.go

@@ -2,6 +2,7 @@ package v2
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"math"
 	"sort"
@@ -18,6 +19,10 @@ type scFinishedEv struct {
 	reason string
 }
 
+func (e scFinishedEv) String() string {
+	return fmt.Sprintf("scFinishedEv{%v}", e.reason)
+}
+
 // send a blockRequest message
 type scBlockRequest struct {
 	priorityNormal
@@ -25,6 +30,10 @@ type scBlockRequest struct {
 	height int64
 }
 
+func (e scBlockRequest) String() string {
+	return fmt.Sprintf("scBlockRequest{%d from %v}", e.height, e.peerID)
+}
+
 // a block has been received and validated by the scheduler
 type scBlockReceived struct {
 	priorityNormal
@@ -32,6 +41,10 @@ type scBlockReceived struct {
 	block  *types.Block
 }
 
+func (e scBlockReceived) String() string {
+	return fmt.Sprintf("scBlockReceived{%d#%X from %v}", e.block.Height, e.block.Hash(), e.peerID)
+}
+
 // scheduler detected a peer error
 type scPeerError struct {
 	priorityHigh
@@ -40,7 +53,7 @@ type scPeerError struct {
 }
 
 func (e scPeerError) String() string {
-	return fmt.Sprintf("scPeerError - peerID %s, err %s", e.peerID, e.reason)
+	return fmt.Sprintf("scPeerError{%v errored with %v}", e.peerID, e.reason)
 }
 
 // scheduler removed a set of peers (timed out or slow peer)
@@ -49,6 +62,10 @@ type scPeersPruned struct {
 	peers []p2p.ID
 }
 
+func (e scPeersPruned) String() string {
+	return fmt.Sprintf("scPeersPruned{%v}", e.peers)
+}
+
 // XXX: make this fatal?
 // scheduler encountered a fatal error
 type scSchedulerFail struct {
@@ -56,6 +73,10 @@ type scSchedulerFail struct {
 	reason error
 }
 
+func (e scSchedulerFail) String() string {
+	return fmt.Sprintf("scSchedulerFail{%v}", e.reason)
+}
+
 type blockState int
 
 const (
@@ -189,7 +210,7 @@ func newScheduler(initHeight int64, startTime time.Time) *scheduler {
 		receivedBlocks: make(map[int64]p2p.ID),
 		targetPending:  10,               // TODO - pass as param
 		peerTimeout:    15 * time.Second, // TODO - pass as param
-		minRecvRate:    0,                //int64(7680), TODO - pass as param
+		minRecvRate:    0,                // int64(7680), TODO - pass as param
 	}
 
 	return &sc
@@ -217,14 +238,13 @@ func (sc *scheduler) touchPeer(peerID p2p.ID, time time.Time) error {
 	return nil
 }
 
-func (sc *scheduler) removePeer(peerID p2p.ID) error {
+func (sc *scheduler) removePeer(peerID p2p.ID) {
 	peer, ok := sc.peers[peerID]
 	if !ok {
-		return nil
+		return
 	}
-
 	if peer.state == peerStateRemoved {
-		return fmt.Errorf("tried to remove peer %s in peerStateRemoved", peerID)
+		return
 	}
 
 	for height, pendingPeerID := range sc.pendingBlocks {
@@ -258,8 +278,6 @@ func (sc *scheduler) removePeer(peerID p2p.ID) error {
 			delete(sc.blockStates, h)
 		}
 	}
-
-	return nil
 }
 
 // check if the blockPool is running low and add new blocks in New state to be requested.
@@ -284,17 +302,16 @@ func (sc *scheduler) setPeerRange(peerID p2p.ID, base int64, height int64) error
 	peer := sc.ensurePeer(peerID)
 
 	if peer.state == peerStateRemoved {
-		return fmt.Errorf("cannot set peer height for a peer in peerStateRemoved")
+		return nil // noop
 	}
 
 	if height < peer.height {
-		if err := sc.removePeer(peerID); err != nil {
-			return err
-		}
+		sc.removePeer(peerID)
 		return fmt.Errorf("cannot move peer height lower. from %d to %d", peer.height, height)
 	}
 
 	if base > height {
+		sc.removePeer(peerID)
 		return fmt.Errorf("cannot set peer base higher than its height")
 	}
 
@@ -348,15 +365,9 @@ func (sc *scheduler) setStateAtHeight(height int64, state blockState) {
 	sc.blockStates[height] = state
 }
 
+// CONTRACT: peer exists and in Ready state.
 func (sc *scheduler) markReceived(peerID p2p.ID, height int64, size int64, now time.Time) error {
-	peer, ok := sc.peers[peerID]
-	if !ok {
-		return fmt.Errorf("received block from unknown peer %s", peerID)
-	}
-
-	if peer.state != peerStateReady {
-		return fmt.Errorf("cannot receive blocks from not ready peer %s", peerID)
-	}
+	peer := sc.peers[peerID]
 
 	if state := sc.getStateAtHeight(height); state != blockStatePending || sc.pendingBlocks[height] != peerID {
 		return fmt.Errorf("received block %d from peer %s without being requested", height, peerID)
@@ -412,17 +423,17 @@ func (sc *scheduler) markPending(peerID p2p.ID, height int64, time time.Time) er
 }
 
 func (sc *scheduler) markProcessed(height int64) error {
+	// It is possible that a peer error or timeout is handled after the processor
+	// has processed the block but before the scheduler received this event, so
+	// when pcBlockProcessed event is received, the block had been requested
+	// again => don't check the block state.
 	sc.lastAdvance = time.Now()
-	state := sc.getStateAtHeight(height)
-	if state != blockStateReceived {
-		return fmt.Errorf("cannot mark height %d received from block state %s", height, state)
-	}
-
-	sc.height++
+	sc.height = height + 1
+	delete(sc.pendingBlocks, height)
+	delete(sc.pendingTime, height)
 	delete(sc.receivedBlocks, height)
 	delete(sc.blockStates, height)
 	sc.addNewBlocks()
-
 	return nil
 }
 
@@ -508,9 +519,7 @@ func (peers PeerByID) Less(i, j int) bool {
 }
 
 func (peers PeerByID) Swap(i, j int) {
-	it := peers[i]
-	peers[i] = peers[j]
-	peers[j] = it
+	peers[i], peers[j] = peers[j], peers[i]
 }
 
 // Handlers
@@ -519,12 +528,13 @@ func (peers PeerByID) Swap(i, j int) {
 func (sc *scheduler) handleBlockResponse(event bcBlockResponse) (Event, error) {
 	err := sc.touchPeer(event.peerID, event.time)
 	if err != nil {
-		return scPeerError{peerID: event.peerID, reason: err}, nil
+		// peer does not exist OR not ready
+		return noOp, nil
 	}
 
 	err = sc.markReceived(event.peerID, event.block.Height, event.size, event.time)
 	if err != nil {
-		_ = sc.removePeer(event.peerID)
+		sc.removePeer(event.peerID)
 		return scPeerError{peerID: event.peerID, reason: err}, nil
 	}
 
@@ -532,16 +542,14 @@ func (sc *scheduler) handleBlockResponse(event bcBlockResponse) (Event, error) {
 }
 
 func (sc *scheduler) handleNoBlockResponse(event bcNoBlockResponse) (Event, error) {
-	if len(sc.peers) == 0 {
-		return noOp, nil
-	}
-
+	// No such peer or peer was removed.
 	peer, ok := sc.peers[event.peerID]
 	if !ok || peer.state == peerStateRemoved {
 		return noOp, nil
 	}
+
 	// The peer may have been just removed due to errors, low speed or timeouts.
-	_ = sc.removePeer(event.peerID)
+	sc.removePeer(event.peerID)
 
 	return scPeerError{peerID: event.peerID,
 		reason: fmt.Errorf("peer %v with base %d height %d claims no block for %d",
@@ -550,13 +558,11 @@ func (sc *scheduler) handleNoBlockResponse(event bcNoBlockResponse) (Event, erro
 
 func (sc *scheduler) handleBlockProcessed(event pcBlockProcessed) (Event, error) {
 	if event.height != sc.height {
-		panic(fmt.Sprintf("processed height %d but expected height %d", event.height, sc.height))
+		panic(fmt.Sprintf("processed height %d, but expected height %d", event.height, sc.height))
 	}
+
 	err := sc.markProcessed(event.height)
 	if err != nil {
-		// It is possible that a peer error or timeout is handled after the processor
-		// has processed the block but before the scheduler received this event,
-		// so when pcBlockProcessed event is received the block had been requested again.
 		return scSchedulerFail{reason: err}, nil
 	}
 
@@ -570,13 +576,10 @@ func (sc *scheduler) handleBlockProcessed(event pcBlockProcessed) (Event, error)
 // Handles an error from the processor. The processor had already cleaned the blocks from
 // the peers included in this event. Just attempt to remove the peers.
 func (sc *scheduler) handleBlockProcessError(event pcBlockVerificationFailure) (Event, error) {
-	if len(sc.peers) == 0 {
-		return noOp, nil
-	}
 	// The peers may have been just removed due to errors, low speed or timeouts.
-	_ = sc.removePeer(event.firstPeerID)
+	sc.removePeer(event.firstPeerID)
 	if event.firstPeerID != event.secondPeerID {
-		_ = sc.removePeer(event.secondPeerID)
+		sc.removePeer(event.secondPeerID)
 	}
 
 	if sc.allBlocksProcessed() {
@@ -592,20 +595,18 @@ func (sc *scheduler) handleAddNewPeer(event bcAddNewPeer) (Event, error) {
 }
 
 func (sc *scheduler) handleRemovePeer(event bcRemovePeer) (Event, error) {
-	err := sc.removePeer(event.peerID)
-	if err != nil {
-		// XXX - It is possible that the removePeer fails here for legitimate reasons
-		// for example if a peer timeout or error was handled just before this.
-		return scSchedulerFail{reason: err}, nil
-	}
+	sc.removePeer(event.peerID)
+
 	if sc.allBlocksProcessed() {
 		return scFinishedEv{reason: "removed peer"}, nil
 	}
-	return noOp, nil
+
+	// Return scPeerError so the peer (and all associated blocks) is removed from
+	// the processor.
+	return scPeerError{peerID: event.peerID, reason: errors.New("peer was stopped")}, nil
 }
 
 func (sc *scheduler) handleTryPrunePeer(event rTryPrunePeer) (Event, error) {
-
 	// Check behavior of peer responsible to deliver block at sc.height.
 	timeHeightAsked, ok := sc.pendingTime[sc.height]
 	if ok && time.Since(timeHeightAsked) > sc.peerTimeout {
@@ -613,9 +614,7 @@ func (sc *scheduler) handleTryPrunePeer(event rTryPrunePeer) (Event, error) {
 		// from that peer within sc.peerTimeout. Remove the peer. This is to ensure that a peer
 		// will be timed out even if it sends blocks at higher heights but prevents progress by
 		// not sending the block at current height.
-		if err := sc.removePeer(sc.pendingBlocks[sc.height]); err != nil {
-			return nil, err
-		}
+		sc.removePeer(sc.pendingBlocks[sc.height])
 	}
 
 	prunablePeers := sc.prunablePeers(sc.peerTimeout, sc.minRecvRate, event.time)
@@ -623,11 +622,7 @@ func (sc *scheduler) handleTryPrunePeer(event rTryPrunePeer) (Event, error) {
 		return noOp, nil
 	}
 	for _, peerID := range prunablePeers {
-		err := sc.removePeer(peerID)
-		if err != nil {
-			// Should never happen as prunablePeers() returns only existing peers in Ready state.
-			panic("scheduler data corruption")
-		}
+		sc.removePeer(peerID)
 	}
 
 	// If all blocks are processed we should finish.
@@ -636,7 +631,6 @@ func (sc *scheduler) handleTryPrunePeer(event rTryPrunePeer) (Event, error) {
 	}
 
 	return scPeersPruned{peers: prunablePeers}, nil
-
 }
 
 func (sc *scheduler) handleResetState(event bcResetState) (Event, error) {

blockchain/v2/scheduler_test.go

@@ -418,7 +418,6 @@ func TestScRemovePeer(t *testing.T) {
 					"P1": {height: 10, state: peerStateRemoved},
 					"P2": {height: 11, state: peerStateReady}},
 				allB: []int64{8, 9, 10, 11}},
-			wantErr: true,
 		},
 		{
 			name: "remove Ready peer with blocks requested",
@@ -492,9 +491,7 @@ func TestScRemovePeer(t *testing.T) {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			sc := newTestScheduler(tt.fields)
-			if err := sc.removePeer(tt.args.peerID); (err != nil) != tt.wantErr {
-				t.Errorf("removePeer() wantErr %v, error = %v", tt.wantErr, err)
-			}
+			sc.removePeer(tt.args.peerID)
 			wantSc := newTestScheduler(tt.wantFields)
 			assert.Equal(t, wantSc, sc, "wanted peers %v, got %v", wantSc.peers, sc.peers)
 		})
@@ -534,7 +531,6 @@ func TestScSetPeerRange(t *testing.T) {
 				peers: map[string]*scPeer{"P1": {height: 2, state: peerStateRemoved}}},
 			args:       args{peerID: "P1", height: 4},
 			wantFields: scTestParams{peers: map[string]*scPeer{"P1": {height: 2, state: peerStateRemoved}}},
-			wantErr:    true,
 		},
 		{
 			name: "decrease height of single peer",
@@ -586,8 +582,7 @@ func TestScSetPeerRange(t *testing.T) {
 				allB:  []int64{1, 2, 3, 4}},
 			args: args{peerID: "P1", base: 6, height: 5},
 			wantFields: scTestParams{
-				peers: map[string]*scPeer{"P1": {height: 4, state: peerStateReady}},
-				allB:  []int64{1, 2, 3, 4}},
+				peers: map[string]*scPeer{"P1": {height: 4, state: peerStateRemoved}}},
 			wantErr: true,
 		},
 		{
@@ -993,19 +988,20 @@ func TestScMarkProcessed(t *testing.T) {
 		{
 			name: "processed an unreceived block",
 			fields: scTestParams{
-				peers:       map[string]*scPeer{"P1": {height: 2, state: peerStateReady}},
-				allB:        []int64{1, 2},
-				pending:     map[int64]p2p.ID{2: "P1"},
-				pendingTime: map[int64]time.Time{2: now},
-				received:    map[int64]p2p.ID{1: "P1"}},
+				height:        2,
+				peers:         map[string]*scPeer{"P1": {height: 4, state: peerStateReady}},
+				allB:          []int64{2},
+				pending:       map[int64]p2p.ID{2: "P1"},
+				pendingTime:   map[int64]time.Time{2: now},
+				targetPending: 1,
+			},
 			args: args{height: 2},
 			wantFields: scTestParams{
-				peers:       map[string]*scPeer{"P1": {height: 2, state: peerStateReady}},
-				allB:        []int64{1, 2},
-				pending:     map[int64]p2p.ID{2: "P1"},
-				pendingTime: map[int64]time.Time{2: now},
-				received:    map[int64]p2p.ID{1: "P1"}},
-			wantErr: true,
+				height:        3,
+				peers:         map[string]*scPeer{"P1": {height: 4, state: peerStateReady}},
+				allB:          []int64{3},
+				targetPending: 1,
+			},
 		},
 		{
 			name: "mark processed success",
@@ -1413,13 +1409,13 @@ func TestScHandleBlockResponse(t *testing.T) {
 			name:      "empty scheduler",
 			fields:    scTestParams{},
 			args:      args{event: block6FromP1},
-			wantEvent: scPeerError{peerID: "P1", reason: fmt.Errorf("some error")},
+			wantEvent: noOpEvent{},
 		},
 		{
 			name:      "block from removed peer",
 			fields:    scTestParams{peers: map[string]*scPeer{"P1": {height: 8, state: peerStateRemoved}}},
 			args:      args{event: block6FromP1},
-			wantEvent: scPeerError{peerID: "P1", reason: fmt.Errorf("some error")},
+			wantEvent: noOpEvent{},
 		},
 		{
 			name: "block we haven't asked for",
@@ -1438,7 +1434,7 @@ func TestScHandleBlockResponse(t *testing.T) {
 				pendingTime: map[int64]time.Time{6: now},
 			},
 			args:      args{event: block6FromP1},
-			wantEvent: scPeerError{peerID: "P1", reason: fmt.Errorf("some error")},
+			wantEvent: noOpEvent{},
 		},
 		{
 			name: "block with bad timestamp",
@@ -1572,7 +1568,7 @@ func TestScHandleBlockProcessed(t *testing.T) {
 			name:      "empty scheduler",
 			fields:    scTestParams{height: 6},
 			args:      args{event: processed6FromP1},
-			wantEvent: scSchedulerFail{reason: fmt.Errorf("some error")},
+			wantEvent: noOpEvent{},
 		},
 		{
 			name: "processed block we don't have",
@@ -1584,7 +1580,7 @@ func TestScHandleBlockProcessed(t *testing.T) {
 				pendingTime: map[int64]time.Time{6: now},
 			},
 			args:      args{event: processed6FromP1},
-			wantEvent: scSchedulerFail{reason: fmt.Errorf("some error")},
+			wantEvent: noOpEvent{},
 		},
 		{
 			name: "processed block ok, we processed all blocks",
@@ -1998,7 +1994,7 @@ func TestScHandleStatusResponse(t *testing.T) {
 			name:      "increase height of removed peer",
 			fields:    scTestParams{peers: map[string]*scPeer{"P1": {height: 2, state: peerStateRemoved}}},
 			args:      args{event: statusRespP1Ev},
-			wantEvent: scPeerError{peerID: "P1", reason: fmt.Errorf("some error")},
+			wantEvent: noOpEvent{},
 		},
 
 		{

buf.gen.yaml

@@ -0,0 +1,9 @@
+version: v1
+plugins:
+  - name: gogofaster
+    out: ./proto/
+    opt:
+      - Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types
+      - Mgoogle/protobuf/duration.proto=github.com/golang/protobuf/ptypes/duration
+      - plugins=grpc
+      - paths=source_relative

buf.work.yaml

@@ -0,0 +1,3 @@
+version: v1
+directories:
+  - proto

cmd/tendermint/commands/compact.go

@@ -0,0 +1,66 @@
+package commands
+
+import (
+	"errors"
+	"path/filepath"
+	"sync"
+
+	"github.com/spf13/cobra"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/tendermint/tendermint/libs/log"
+)
+
+var CompactGoLevelDBCmd = &cobra.Command{
+	Use:   "experimental-compact-goleveldb",
+	Short: "force compacts the tendermint storage engine (only GoLevelDB supported)",
+	Long: `
+This is a temporary utility command that performs a force compaction on the state 
+and blockstores to reduce disk space for a pruning node. This should only be run 
+once the node has stopped. This command will likely be omitted in the future after
+the planned refactor to the storage engine.
+
+Currently, only GoLevelDB is supported.
+	`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if config.DBBackend != "goleveldb" {
+			return errors.New("compaction is currently only supported with goleveldb")
+		}
+
+		compactGoLevelDBs(config.RootDir, logger)
+		return nil
+	},
+}
+
+func compactGoLevelDBs(rootDir string, logger log.Logger) {
+	dbNames := []string{"state", "blockstore"}
+	o := &opt.Options{
+		DisableSeeksCompaction: true,
+	}
+	wg := sync.WaitGroup{}
+
+	for _, dbName := range dbNames {
+		dbName := dbName
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			dbPath := filepath.Join(rootDir, "data", dbName+".db")
+			store, err := leveldb.OpenFile(dbPath, o)
+			if err != nil {
+				logger.Error("failed to initialize tendermint db", "path", dbPath, "err", err)
+				return
+			}
+			defer store.Close()
+
+			logger.Info("starting compaction...", "db", dbPath)
+
+			err = store.CompactRange(util.Range{Start: nil, Limit: nil})
+			if err != nil {
+				logger.Error("failed to compact tendermint db", "path", dbPath, "err", err)
+			}
+		}()
+	}
+	wg.Wait()
+}

cmd/tendermint/commands/debug/debug.go

@@ -33,7 +33,7 @@ func init() {
 		&nodeRPCAddr,
 		flagNodeRPCAddr,
 		"tcp://localhost:26657",
-		"The Tendermint node's RPC address (<host>:<port>)",
+		"the Tendermint node's RPC address (<host>:<port>)",
 	)
 
 	DebugCmd.AddCommand(killCmd)

cmd/tendermint/commands/debug/dump.go

@@ -32,14 +32,14 @@ func init() {
 		&frequency,
 		flagFrequency,
 		30,
-		"The frequency (seconds) in which to poll, aggregate and dump Tendermint debug data",
+		"the frequency (seconds) in which to poll, aggregate and dump Tendermint debug data",
 	)
 
 	dumpCmd.Flags().StringVar(
 		&profAddr,
 		flagProfAddr,
 		"",
-		"The profiling server address (<host>:<port>)",
+		"the profiling server address (<host>:<port>)",
 	)
 }
 

cmd/tendermint/commands/debug/kill.go

@@ -105,7 +105,7 @@ func killProc(pid uint64, dir string) error {
 	// pipe STDERR output from tailing the Tendermint process to a file
 	//
 	// NOTE: This will only work on UNIX systems.
-	cmd := exec.Command("tail", "-f", fmt.Sprintf("/proc/%d/fd/2", pid)) // nolint: gosec
+	cmd := exec.Command("tail", "-f", fmt.Sprintf("/proc/%d/fd/2", pid)) //nolint: gosec
 
 	outFile, err := os.Create(filepath.Join(dir, "stacktrace.out"))
 	if err != nil {

cmd/tendermint/commands/debug/util.go

@@ -67,7 +67,7 @@ func copyConfig(home, dir string) error {
 func dumpProfile(dir, addr, profile string, debug int) error {
 	endpoint := fmt.Sprintf("%s/debug/pprof/%s?debug=%d", addr, profile, debug)
 
-	resp, err := http.Get(endpoint) // nolint: gosec
+	resp, err := http.Get(endpoint) //nolint: gosec
 	if err != nil {
 		return fmt.Errorf("failed to query for %s profile: %w", profile, err)
 	}

cmd/tendermint/commands/gen_node_key.go

@@ -12,9 +12,11 @@ import (
 // GenNodeKeyCmd allows the generation of a node key. It prints node's ID to
 // the standard output.
 var GenNodeKeyCmd = &cobra.Command{
-	Use:   "gen_node_key",
-	Short: "Generate a node key for this node and print its ID",
-	RunE:  genNodeKey,
+	Use:     "gen-node-key",
+	Aliases: []string{"gen_node_key"},
+	Short:   "Generate a node key for this node and print its ID",
+	PreRun:  deprecateSnakeCase,
+	RunE:    genNodeKey,
 }
 
 func genNodeKey(cmd *cobra.Command, args []string) error {

cmd/tendermint/commands/gen_validator.go

@@ -12,9 +12,11 @@ import (
 // GenValidatorCmd allows the generation of a keypair for a
 // validator.
 var GenValidatorCmd = &cobra.Command{
-	Use:   "gen_validator",
-	Short: "Generate new validator keypair",
-	Run:   genValidator,
+	Use:     "gen-validator",
+	Aliases: []string{"gen_validator"},
+	Short:   "Generate new validator keypair",
+	PreRun:  deprecateSnakeCase,
+	Run:     genValidator,
 }
 
 func genValidator(cmd *cobra.Command, args []string) {

cmd/tendermint/commands/light.go

@@ -1,11 +1,13 @@
 package commands
 
 import (
+	"bufio"
 	"context"
 	"errors"
 	"fmt"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -20,7 +22,6 @@ import (
 	lproxy "github.com/tendermint/tendermint/light/proxy"
 	lrpc "github.com/tendermint/tendermint/light/rpc"
 	dbs "github.com/tendermint/tendermint/light/store/db"
-	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 	rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
 )
 
@@ -39,6 +40,12 @@ need a primary RPC address, a trusted hash and height and witness RPC addresses
 (if not using sequential verification). To restart the node, thereafter
 only the chainID is required.
 
+When /abci_query is called, the Merkle key path format is:
+
+	/{store name}/{key}
+
+Please verify with your application that this Merkle key format is used (true
+for applications built w/ Cosmos SDK).
 `,
 	RunE: runProxy,
 	Args: cobra.ExactArgs(1),
@@ -68,27 +75,28 @@ var (
 
 func init() {
 	LightCmd.Flags().StringVar(&listenAddr, "laddr", "tcp://localhost:8888",
-		"Serve the proxy on the given address")
+		"serve the proxy on the given address")
 	LightCmd.Flags().StringVarP(&primaryAddr, "primary", "p", "",
-		"Connect to a Tendermint node at this address")
+		"connect to a Tendermint node at this address")
 	LightCmd.Flags().StringVarP(&witnessAddrsJoined, "witnesses", "w", "",
-		"Tendermint nodes to cross-check the primary node, comma-separated")
-	LightCmd.Flags().StringVar(&home, "home-dir", ".tendermint-light", "Specify the home directory")
+		"tendermint nodes to cross-check the primary node, comma-separated")
+	LightCmd.Flags().StringVar(&home, "home-dir", os.ExpandEnv(filepath.Join("$HOME", ".tendermint-light")),
+		"specify the home directory")
 	LightCmd.Flags().IntVar(
 		&maxOpenConnections,
 		"max-open-connections",
 		900,
-		"Maximum number of simultaneous connections (including WebSocket).")
+		"maximum number of simultaneous connections (including WebSocket).")
 	LightCmd.Flags().DurationVar(&trustingPeriod, "trusting-period", 168*time.Hour,
-		"Trusting period that headers can be verified within. Should be significantly less than the unbonding period")
+		"trusting period that headers can be verified within. Should be significantly less than the unbonding period")
 	LightCmd.Flags().Int64Var(&trustedHeight, "height", 1, "Trusted header's height")
 	LightCmd.Flags().BytesHexVar(&trustedHash, "hash", []byte{}, "Trusted header's hash")
 	LightCmd.Flags().BoolVar(&verbose, "verbose", false, "Verbose output")
 	LightCmd.Flags().StringVar(&trustLevelStr, "trust-level", "1/3",
-		"Trust level. Must be between 1/3 and 3/3",
+		"trust level. Must be between 1/3 and 3/3",
 	)
 	LightCmd.Flags().BoolVar(&sequential, "sequential", false,
-		"Sequential Verification. Verify all headers sequentially as opposed to using skipping verification",
+		"sequential verification. Verify all headers sequentially as opposed to using skipping verification",
 	)
 }
 
@@ -138,7 +146,25 @@ func runProxy(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("can't parse trust level: %w", err)
 	}
 
-	options := []light.Option{light.Logger(logger)}
+	options := []light.Option{
+		light.Logger(logger),
+		light.ConfirmationFunction(func(action string) bool {
+			fmt.Println(action)
+			scanner := bufio.NewScanner(os.Stdin)
+			for {
+				scanner.Scan()
+				response := scanner.Text()
+				switch response {
+				case "y", "Y":
+					return true
+				case "n", "N":
+					return false
+				default:
+					fmt.Println("please input 'Y' or 'n' and press ENTER")
+				}
+			}
+		}),
+	}
 
 	if sequential {
 		options = append(options, light.SequentialVerification())
@@ -175,11 +201,6 @@ func runProxy(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	rpcClient, err := rpchttp.New(primaryAddr, "/websocket")
-	if err != nil {
-		return fmt.Errorf("http client for %s: %w", primaryAddr, err)
-	}
-
 	cfg := rpcserver.DefaultConfig()
 	cfg.MaxBodyBytes = config.RPC.MaxBodyBytes
 	cfg.MaxHeaderBytes = config.RPC.MaxHeaderBytes
@@ -191,12 +212,11 @@ func runProxy(cmd *cobra.Command, args []string) error {
 		cfg.WriteTimeout = config.RPC.TimeoutBroadcastTxCommit + 1*time.Second
 	}
 
-	p := lproxy.Proxy{
-		Addr:   listenAddr,
-		Config: cfg,
-		Client: lrpc.NewClient(rpcClient, c),
-		Logger: logger,
+	p, err := lproxy.NewProxy(c, listenAddr, primaryAddr, cfg, logger, lrpc.KeyPathFn(lrpc.DefaultMerkleKeyPathFn()))
+	if err != nil {
+		return err
 	}
+
 	// Stop upon receiving SIGTERM or CTRL-C.
 	tmos.TrapSignal(logger, func() {
 		p.Listener.Close()

cmd/tendermint/commands/probe_upnp.go

@@ -11,9 +11,11 @@ import (
 
 // ProbeUpnpCmd adds capabilities to test the UPnP functionality.
 var ProbeUpnpCmd = &cobra.Command{
-	Use:   "probe_upnp",
-	Short: "Test UPnP functionality",
-	RunE:  probeUpnp,
+	Use:     "probe-upnp",
+	Aliases: []string{"probe_upnp"},
+	Short:   "Test UPnP functionality",
+	RunE:    probeUpnp,
+	PreRun:  deprecateSnakeCase,
 }
 
 func probeUpnp(cmd *cobra.Command, args []string) error {

cmd/tendermint/commands/reindex_event.go

@@ -0,0 +1,232 @@
+package commands
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+	dbm "github.com/tendermint/tm-db"
+
+	abcitypes "github.com/tendermint/tendermint/abci/types"
+	tmcfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/progressbar"
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/state/indexer"
+	blockidxkv "github.com/tendermint/tendermint/state/indexer/block/kv"
+	"github.com/tendermint/tendermint/state/indexer/sink/psql"
+	"github.com/tendermint/tendermint/state/txindex"
+	"github.com/tendermint/tendermint/state/txindex/kv"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	reindexFailed = "event re-index failed: "
+)
+
+var (
+	ErrHeightNotAvailable = errors.New("height is not available")
+	ErrInvalidRequest     = errors.New("invalid request")
+)
+
+// ReIndexEventCmd constructs a command to re-index events in a block height interval.
+var ReIndexEventCmd = &cobra.Command{
+	Use:   "reindex-event",
+	Short: "Re-index events to the event store backends",
+	Long: `
+reindex-event is an offline tooling to re-index block and tx events to the eventsinks.
+You can run this command when the event store backend dropped/disconnected or you want to
+replace the backend. The default start-height is 0, meaning the tooling will start 
+reindex from the base block height(inclusive); and the default end-height is 0, meaning 
+the tooling will reindex until the latest block height(inclusive). User can omit
+either or both arguments.
+
+Note: This operation requires ABCIResponses. Do not set DiscardABCIResponses to true if you
+want to use this command.
+	`,
+	Example: `
+	tendermint reindex-event
+	tendermint reindex-event --start-height 2
+	tendermint reindex-event --end-height 10
+	tendermint reindex-event --start-height 2 --end-height 10
+	`,
+	Run: func(cmd *cobra.Command, args []string) {
+		bs, ss, err := loadStateAndBlockStore(config)
+		if err != nil {
+			fmt.Println(reindexFailed, err)
+			return
+		}
+
+		if err := checkValidHeight(bs); err != nil {
+			fmt.Println(reindexFailed, err)
+			return
+		}
+
+		bi, ti, err := loadEventSinks(config)
+		if err != nil {
+			fmt.Println(reindexFailed, err)
+			return
+		}
+
+		riArgs := eventReIndexArgs{
+			startHeight:  startHeight,
+			endHeight:    endHeight,
+			blockIndexer: bi,
+			txIndexer:    ti,
+			blockStore:   bs,
+			stateStore:   ss,
+		}
+		if err := eventReIndex(cmd, riArgs); err != nil {
+			panic(fmt.Errorf("%s: %w", reindexFailed, err))
+		}
+
+		fmt.Println("event re-index finished")
+	},
+}
+
+var (
+	startHeight int64
+	endHeight   int64
+)
+
+func init() {
+	ReIndexEventCmd.Flags().Int64Var(&startHeight, "start-height", 0, "the block height would like to start for re-index")
+	ReIndexEventCmd.Flags().Int64Var(&endHeight, "end-height", 0, "the block height would like to finish for re-index")
+}
+
+func loadEventSinks(cfg *tmcfg.Config) (indexer.BlockIndexer, txindex.TxIndexer, error) {
+	switch strings.ToLower(cfg.TxIndex.Indexer) {
+	case "null":
+		return nil, nil, errors.New("found null event sink, please check the tx-index section in the config.toml")
+	case "psql":
+		conn := cfg.TxIndex.PsqlConn
+		if conn == "" {
+			return nil, nil, errors.New("the psql connection settings cannot be empty")
+		}
+		es, err := psql.NewEventSink(conn, cfg.ChainID())
+		if err != nil {
+			return nil, nil, err
+		}
+		return es.BlockIndexer(), es.TxIndexer(), nil
+	case "kv":
+		store, err := dbm.NewDB("tx_index", dbm.BackendType(cfg.DBBackend), cfg.DBDir())
+		if err != nil {
+			return nil, nil, err
+		}
+
+		txIndexer := kv.NewTxIndex(store)
+		blockIndexer := blockidxkv.New(dbm.NewPrefixDB(store, []byte("block_events")))
+		return blockIndexer, txIndexer, nil
+	default:
+		return nil, nil, fmt.Errorf("unsupported event sink type: %s", cfg.TxIndex.Indexer)
+	}
+}
+
+type eventReIndexArgs struct {
+	startHeight  int64
+	endHeight    int64
+	blockIndexer indexer.BlockIndexer
+	txIndexer    txindex.TxIndexer
+	blockStore   state.BlockStore
+	stateStore   state.Store
+}
+
+func eventReIndex(cmd *cobra.Command, args eventReIndexArgs) error {
+	var bar progressbar.Bar
+	bar.NewOption(args.startHeight-1, args.endHeight)
+
+	fmt.Println("start re-indexing events:")
+	defer bar.Finish()
+	for i := args.startHeight; i <= args.endHeight; i++ {
+		select {
+		case <-cmd.Context().Done():
+			return fmt.Errorf("event re-index terminated at height %d: %w", i, cmd.Context().Err())
+		default:
+			b := args.blockStore.LoadBlock(i)
+			if b == nil {
+				return fmt.Errorf("not able to load block at height %d from the blockstore", i)
+			}
+
+			r, err := args.stateStore.LoadABCIResponses(i)
+			if err != nil {
+				return fmt.Errorf("not able to load ABCI Response at height %d from the statestore", i)
+			}
+
+			e := types.EventDataNewBlockHeader{
+				Header:           b.Header,
+				NumTxs:           int64(len(b.Txs)),
+				ResultBeginBlock: *r.BeginBlock,
+				ResultEndBlock:   *r.EndBlock,
+			}
+
+			var batch *txindex.Batch
+			if e.NumTxs > 0 {
+				batch = txindex.NewBatch(e.NumTxs)
+
+				for i := range b.Data.Txs {
+					tr := abcitypes.TxResult{
+						Height: b.Height,
+						Index:  uint32(i),
+						Tx:     b.Data.Txs[i],
+						Result: *(r.DeliverTxs[i]),
+					}
+
+					if err = batch.Add(&tr); err != nil {
+						return fmt.Errorf("adding tx to batch: %w", err)
+					}
+				}
+
+				if err := args.txIndexer.AddBatch(batch); err != nil {
+					return fmt.Errorf("tx event re-index at height %d failed: %w", i, err)
+				}
+			}
+
+			if err := args.blockIndexer.Index(e); err != nil {
+				return fmt.Errorf("block event re-index at height %d failed: %w", i, err)
+			}
+		}
+
+		bar.Play(i)
+	}
+
+	return nil
+}
+
+func checkValidHeight(bs state.BlockStore) error {
+	base := bs.Base()
+
+	if startHeight == 0 {
+		startHeight = base
+		fmt.Printf("set the start block height to the base height of the blockstore %d \n", base)
+	}
+
+	if startHeight < base {
+		return fmt.Errorf("%s (requested start height: %d, base height: %d)",
+			ErrHeightNotAvailable, startHeight, base)
+	}
+
+	height := bs.Height()
+
+	if startHeight > height {
+		return fmt.Errorf(
+			"%s (requested start height: %d, store height: %d)", ErrHeightNotAvailable, startHeight, height)
+	}
+
+	if endHeight == 0 || endHeight > height {
+		endHeight = height
+		fmt.Printf("set the end block height to the latest height of the blockstore %d \n", height)
+	}
+
+	if endHeight < base {
+		return fmt.Errorf(
+			"%s (requested end height: %d, base height: %d)", ErrHeightNotAvailable, endHeight, base)
+	}
+
+	if endHeight < startHeight {
+		return fmt.Errorf(
+			"%s (requested the end height: %d is less than the start height: %d)",
+			ErrInvalidRequest, startHeight, endHeight)
+	}
+
+	return nil
+}

cmd/tendermint/commands/reindex_event_test.go

@@ -0,0 +1,192 @@
+package commands
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	abcitypes "github.com/tendermint/tendermint/abci/types"
+	tmcfg "github.com/tendermint/tendermint/config"
+	prototmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	blockmocks "github.com/tendermint/tendermint/state/indexer/mocks"
+	"github.com/tendermint/tendermint/state/mocks"
+	txmocks "github.com/tendermint/tendermint/state/txindex/mocks"
+	"github.com/tendermint/tendermint/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+const (
+	height int64 = 10
+	base   int64 = 2
+)
+
+func setupReIndexEventCmd() *cobra.Command {
+	reIndexEventCmd := &cobra.Command{
+		Use: ReIndexEventCmd.Use,
+		Run: func(cmd *cobra.Command, args []string) {},
+	}
+
+	_ = reIndexEventCmd.ExecuteContext(context.Background())
+
+	return reIndexEventCmd
+}
+
+func TestReIndexEventCheckHeight(t *testing.T) {
+	mockBlockStore := &mocks.BlockStore{}
+	mockBlockStore.
+		On("Base").Return(base).
+		On("Height").Return(height)
+
+	testCases := []struct {
+		startHeight int64
+		endHeight   int64
+		validHeight bool
+	}{
+		{0, 0, true},
+		{0, base, true},
+		{0, base - 1, false},
+		{0, height, true},
+		{0, height + 1, true},
+		{0, 0, true},
+		{base - 1, 0, false},
+		{base, 0, true},
+		{base, base, true},
+		{base, base - 1, false},
+		{base, height, true},
+		{base, height + 1, true},
+		{height, 0, true},
+		{height, base, false},
+		{height, height - 1, false},
+		{height, height, true},
+		{height, height + 1, true},
+		{height + 1, 0, false},
+	}
+
+	for _, tc := range testCases {
+		startHeight = tc.startHeight
+		endHeight = tc.endHeight
+
+		err := checkValidHeight(mockBlockStore)
+		if tc.validHeight {
+			require.NoError(t, err)
+		} else {
+			require.Error(t, err)
+		}
+	}
+}
+
+func TestLoadEventSink(t *testing.T) {
+	testCases := []struct {
+		sinks   string
+		connURL string
+		loadErr bool
+	}{
+		{"", "", true},
+		{"NULL", "", true},
+		{"KV", "", false},
+		{"PSQL", "", true}, // true because empty connect url
+		// skip to test PSQL connect with correct url
+		{"UnsupportedSinkType", "wrongUrl", true},
+	}
+
+	for idx, tc := range testCases {
+		cfg := tmcfg.TestConfig()
+		cfg.TxIndex.Indexer = tc.sinks
+		cfg.TxIndex.PsqlConn = tc.connURL
+		_, _, err := loadEventSinks(cfg)
+		if tc.loadErr {
+			require.Error(t, err, idx)
+		} else {
+			require.NoError(t, err, idx)
+		}
+	}
+}
+
+func TestLoadBlockStore(t *testing.T) {
+	cfg := tmcfg.TestConfig()
+	cfg.DBPath = t.TempDir()
+	_, _, err := loadStateAndBlockStore(cfg)
+	require.Error(t, err)
+
+	_, err = dbm.NewDB("blockstore", dbm.GoLevelDBBackend, cfg.DBDir())
+	require.NoError(t, err)
+
+	// Get StateStore
+	_, err = dbm.NewDB("state", dbm.GoLevelDBBackend, cfg.DBDir())
+	require.NoError(t, err)
+
+	bs, ss, err := loadStateAndBlockStore(cfg)
+	require.NoError(t, err)
+	require.NotNil(t, bs)
+	require.NotNil(t, ss)
+
+}
+
+func TestReIndexEvent(t *testing.T) {
+	mockBlockStore := &mocks.BlockStore{}
+	mockStateStore := &mocks.Store{}
+	mockBlockIndexer := &blockmocks.BlockIndexer{}
+	mockTxIndexer := &txmocks.TxIndexer{}
+
+	mockBlockStore.
+		On("Base").Return(base).
+		On("Height").Return(height).
+		On("LoadBlock", base).Return(nil).Once().
+		On("LoadBlock", base).Return(&types.Block{Data: types.Data{Txs: types.Txs{make(types.Tx, 1)}}}).
+		On("LoadBlock", height).Return(&types.Block{Data: types.Data{Txs: types.Txs{make(types.Tx, 1)}}})
+
+	dtx := abcitypes.ResponseDeliverTx{}
+	abciResp := &prototmstate.ABCIResponses{
+		DeliverTxs: []*abcitypes.ResponseDeliverTx{&dtx},
+		EndBlock:   &abcitypes.ResponseEndBlock{},
+		BeginBlock: &abcitypes.ResponseBeginBlock{},
+	}
+
+	mockBlockIndexer.
+		On("Index", mock.AnythingOfType("types.EventDataNewBlockHeader")).Return(errors.New("")).Once().
+		On("Index", mock.AnythingOfType("types.EventDataNewBlockHeader")).Return(nil)
+
+	mockTxIndexer.
+		On("AddBatch", mock.AnythingOfType("*txindex.Batch")).Return(errors.New("")).Once().
+		On("AddBatch", mock.AnythingOfType("*txindex.Batch")).Return(nil)
+
+	mockStateStore.
+		On("LoadABCIResponses", base).Return(nil, errors.New("")).Once().
+		On("LoadABCIResponses", base).Return(abciResp, nil).
+		On("LoadABCIResponses", height).Return(abciResp, nil)
+
+	testCases := []struct {
+		startHeight int64
+		endHeight   int64
+		reIndexErr  bool
+	}{
+		{base, height, true}, // LoadBlock error
+		{base, height, true}, // LoadABCIResponses error
+		{base, height, true}, // index block event error
+		{base, height, true}, // index tx event error
+		{base, base, false},
+		{height, height, false},
+	}
+
+	for _, tc := range testCases {
+		args := eventReIndexArgs{
+			startHeight:  tc.startHeight,
+			endHeight:    tc.endHeight,
+			blockIndexer: mockBlockIndexer,
+			txIndexer:    mockTxIndexer,
+			blockStore:   mockBlockStore,
+			stateStore:   mockStateStore,
+		}
+
+		err := eventReIndex(setupReIndexEventCmd(), args)
+		if tc.reIndexErr {
+			require.Error(t, err)
+		} else {
+			require.NoError(t, err)
+		}
+	}
+}

cmd/tendermint/commands/replay.go

@@ -18,9 +18,11 @@ var ReplayCmd = &cobra.Command{
 // ReplayConsoleCmd allows replaying of messages from the WAL in a
 // console.
 var ReplayConsoleCmd = &cobra.Command{
-	Use:   "replay_console",
-	Short: "Replay messages from WAL in a console",
+	Use:     "replay-console",
+	Aliases: []string{"replay_console"},
+	Short:   "Replay messages from WAL in a console",
 	Run: func(cmd *cobra.Command, args []string) {
 		consensus.RunReplayFile(config.BaseConfig, config.Consensus, true)
 	},
+	PreRun: deprecateSnakeCase,
 }

cmd/tendermint/commands/reset.go

@@ -0,0 +1,186 @@
+package commands
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	"github.com/tendermint/tendermint/libs/log"
+	tmos "github.com/tendermint/tendermint/libs/os"
+	"github.com/tendermint/tendermint/privval"
+)
+
+// ResetAllCmd removes the database of this Tendermint core
+// instance.
+var ResetAllCmd = &cobra.Command{
+	Use:     "unsafe-reset-all",
+	Aliases: []string{"unsafe_reset_all"},
+	Short:   "(unsafe) Remove all the data and WAL, reset this node's validator to genesis state",
+	RunE:    resetAllCmd,
+	PreRun:  deprecateSnakeCase,
+}
+
+var keepAddrBook bool
+
+// ResetStateCmd removes the database of the specified Tendermint core instance.
+var ResetStateCmd = &cobra.Command{
+	Use:    "reset-state",
+	Short:  "Remove all the data and WAL",
+	PreRun: deprecateSnakeCase,
+	RunE: func(cmd *cobra.Command, args []string) (err error) {
+		config, err = ParseConfig(cmd)
+		if err != nil {
+			return err
+		}
+
+		return resetState(config.DBDir(), logger)
+	},
+}
+
+func init() {
+	ResetAllCmd.Flags().BoolVar(&keepAddrBook, "keep-addr-book", false, "keep the address book intact")
+}
+
+// ResetPrivValidatorCmd resets the private validator files.
+var ResetPrivValidatorCmd = &cobra.Command{
+	Use:     "unsafe-reset-priv-validator",
+	Aliases: []string{"unsafe_reset_priv_validator"},
+	Short:   "(unsafe) Reset this node's validator to genesis state",
+	PreRun:  deprecateSnakeCase,
+	RunE:    resetPrivValidator,
+}
+
+// XXX: this is totally unsafe.
+// it's only suitable for testnets.
+func resetAllCmd(cmd *cobra.Command, args []string) (err error) {
+	config, err = ParseConfig(cmd)
+	if err != nil {
+		return err
+	}
+
+	return resetAll(
+		config.DBDir(),
+		config.P2P.AddrBookFile(),
+		config.PrivValidatorKeyFile(),
+		config.PrivValidatorStateFile(),
+		logger,
+	)
+}
+
+// XXX: this is totally unsafe.
+// it's only suitable for testnets.
+func resetPrivValidator(cmd *cobra.Command, args []string) (err error) {
+	config, err = ParseConfig(cmd)
+	if err != nil {
+		return err
+	}
+
+	resetFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile(), logger)
+	return nil
+}
+
+// resetAll removes address book files plus all data, and resets the privValdiator data.
+func resetAll(dbDir, addrBookFile, privValKeyFile, privValStateFile string, logger log.Logger) error {
+	if keepAddrBook {
+		logger.Info("The address book remains intact")
+	} else {
+		removeAddrBook(addrBookFile, logger)
+	}
+
+	if err := os.RemoveAll(dbDir); err == nil {
+		logger.Info("Removed all blockchain history", "dir", dbDir)
+	} else {
+		logger.Error("Error removing all blockchain history", "dir", dbDir, "err", err)
+	}
+
+	if err := tmos.EnsureDir(dbDir, 0700); err != nil {
+		logger.Error("unable to recreate dbDir", "err", err)
+	}
+
+	// recreate the dbDir since the privVal state needs to live there
+	resetFilePV(privValKeyFile, privValStateFile, logger)
+	return nil
+}
+
+// resetState removes address book files plus all databases.
+func resetState(dbDir string, logger log.Logger) error {
+	blockdb := filepath.Join(dbDir, "blockstore.db")
+	state := filepath.Join(dbDir, "state.db")
+	wal := filepath.Join(dbDir, "cs.wal")
+	evidence := filepath.Join(dbDir, "evidence.db")
+	txIndex := filepath.Join(dbDir, "tx_index.db")
+
+	if tmos.FileExists(blockdb) {
+		if err := os.RemoveAll(blockdb); err == nil {
+			logger.Info("Removed all blockstore.db", "dir", blockdb)
+		} else {
+			logger.Error("error removing all blockstore.db", "dir", blockdb, "err", err)
+		}
+	}
+
+	if tmos.FileExists(state) {
+		if err := os.RemoveAll(state); err == nil {
+			logger.Info("Removed all state.db", "dir", state)
+		} else {
+			logger.Error("error removing all state.db", "dir", state, "err", err)
+		}
+	}
+
+	if tmos.FileExists(wal) {
+		if err := os.RemoveAll(wal); err == nil {
+			logger.Info("Removed all cs.wal", "dir", wal)
+		} else {
+			logger.Error("error removing all cs.wal", "dir", wal, "err", err)
+		}
+	}
+
+	if tmos.FileExists(evidence) {
+		if err := os.RemoveAll(evidence); err == nil {
+			logger.Info("Removed all evidence.db", "dir", evidence)
+		} else {
+			logger.Error("error removing all evidence.db", "dir", evidence, "err", err)
+		}
+	}
+
+	if tmos.FileExists(txIndex) {
+		if err := os.RemoveAll(txIndex); err == nil {
+			logger.Info("Removed tx_index.db", "dir", txIndex)
+		} else {
+			logger.Error("error removing tx_index.db", "dir", txIndex, "err", err)
+		}
+	}
+
+	if err := tmos.EnsureDir(dbDir, 0700); err != nil {
+		logger.Error("unable to recreate dbDir", "err", err)
+	}
+	return nil
+}
+
+func resetFilePV(privValKeyFile, privValStateFile string, logger log.Logger) {
+	if _, err := os.Stat(privValKeyFile); err == nil {
+		pv := privval.LoadFilePVEmptyState(privValKeyFile, privValStateFile)
+		pv.Reset()
+		logger.Info(
+			"Reset private validator file to genesis state",
+			"keyFile", privValKeyFile,
+			"stateFile", privValStateFile,
+		)
+	} else {
+		pv := privval.GenFilePV(privValKeyFile, privValStateFile)
+		pv.Save()
+		logger.Info(
+			"Generated private validator file",
+			"keyFile", privValKeyFile,
+			"stateFile", privValStateFile,
+		)
+	}
+}
+
+func removeAddrBook(addrBookFile string, logger log.Logger) {
+	if err := os.Remove(addrBookFile); err == nil {
+		logger.Info("Removed existing address book", "file", addrBookFile)
+	} else if !os.IsNotExist(err) {
+		logger.Info("Error removing address book", "file", addrBookFile, "err", err)
+	}
+}

cmd/tendermint/commands/reset_priv_validator.go

@@ -1,87 +0,0 @@
-package commands
-
-import (
-	"os"
-
-	"github.com/spf13/cobra"
-
-	"github.com/tendermint/tendermint/libs/log"
-	tmos "github.com/tendermint/tendermint/libs/os"
-	"github.com/tendermint/tendermint/privval"
-)
-
-// ResetAllCmd removes the database of this Tendermint core
-// instance.
-var ResetAllCmd = &cobra.Command{
-	Use:   "unsafe_reset_all",
-	Short: "(unsafe) Remove all the data and WAL, reset this node's validator to genesis state",
-	Run:   resetAll,
-}
-
-var keepAddrBook bool
-
-func init() {
-	ResetAllCmd.Flags().BoolVar(&keepAddrBook, "keep-addr-book", false, "Keep the address book intact")
-}
-
-// ResetPrivValidatorCmd resets the private validator files.
-var ResetPrivValidatorCmd = &cobra.Command{
-	Use:   "unsafe_reset_priv_validator",
-	Short: "(unsafe) Reset this node's validator to genesis state",
-	Run:   resetPrivValidator,
-}
-
-// XXX: this is totally unsafe.
-// it's only suitable for testnets.
-func resetAll(cmd *cobra.Command, args []string) {
-	ResetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorKeyFile(),
-		config.PrivValidatorStateFile(), logger)
-}
-
-// XXX: this is totally unsafe.
-// it's only suitable for testnets.
-func resetPrivValidator(cmd *cobra.Command, args []string) {
-	resetFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile(), logger)
-}
-
-// ResetAll removes address book files plus all data, and resets the privValdiator data.
-// Exported so other CLI tools can use it.
-func ResetAll(dbDir, addrBookFile, privValKeyFile, privValStateFile string, logger log.Logger) {
-	if keepAddrBook {
-		logger.Info("The address book remains intact")
-	} else {
-		removeAddrBook(addrBookFile, logger)
-	}
-	if err := os.RemoveAll(dbDir); err == nil {
-		logger.Info("Removed all blockchain history", "dir", dbDir)
-	} else {
-		logger.Error("Error removing all blockchain history", "dir", dbDir, "err", err)
-	}
-	// recreate the dbDir since the privVal state needs to live there
-	if err := tmos.EnsureDir(dbDir, 0700); err != nil {
-		logger.Error("unable to recreate dbDir", "err", err)
-	}
-	resetFilePV(privValKeyFile, privValStateFile, logger)
-}
-
-func resetFilePV(privValKeyFile, privValStateFile string, logger log.Logger) {
-	if _, err := os.Stat(privValKeyFile); err == nil {
-		pv := privval.LoadFilePVEmptyState(privValKeyFile, privValStateFile)
-		pv.Reset()
-		logger.Info("Reset private validator file to genesis state", "keyFile", privValKeyFile,
-			"stateFile", privValStateFile)
-	} else {
-		pv := privval.GenFilePV(privValKeyFile, privValStateFile)
-		pv.Save()
-		logger.Info("Generated private validator file", "keyFile", privValKeyFile,
-			"stateFile", privValStateFile)
-	}
-}
-
-func removeAddrBook(addrBookFile string, logger log.Logger) {
-	if err := os.Remove(addrBookFile); err == nil {
-		logger.Info("Removed existing address book", "file", addrBookFile)
-	} else if !os.IsNotExist(err) {
-		logger.Info("Error removing address book", "file", addrBookFile, "err", err)
-	}
-}

cmd/tendermint/commands/reset_test.go

@@ -0,0 +1,53 @@
+package commands
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/privval"
+)
+
+func Test_ResetAll(t *testing.T) {
+	config := cfg.TestConfig()
+	dir := t.TempDir()
+	config.SetRoot(dir)
+	cfg.EnsureRoot(dir)
+	require.NoError(t, initFilesWithConfig(config))
+	pv := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+	pv.LastSignState.Height = 10
+	pv.Save()
+	require.NoError(t, resetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorKeyFile(),
+		config.PrivValidatorStateFile(), logger))
+	require.DirExists(t, config.DBDir())
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "block.db"))
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "state.db"))
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "evidence.db"))
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "tx_index.db"))
+	require.FileExists(t, config.PrivValidatorStateFile())
+	pv = privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+	require.Equal(t, int64(0), pv.LastSignState.Height)
+}
+
+func Test_ResetState(t *testing.T) {
+	config := cfg.TestConfig()
+	dir := t.TempDir()
+	config.SetRoot(dir)
+	cfg.EnsureRoot(dir)
+	require.NoError(t, initFilesWithConfig(config))
+	pv := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+	pv.LastSignState.Height = 10
+	pv.Save()
+	require.NoError(t, resetState(config.DBDir(), logger))
+	require.DirExists(t, config.DBDir())
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "block.db"))
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "state.db"))
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "evidence.db"))
+	require.NoFileExists(t, filepath.Join(config.DBDir(), "tx_index.db"))
+	require.FileExists(t, config.PrivValidatorStateFile())
+	pv = privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+	// private validator state should still be in tact.
+	require.Equal(t, int64(10), pv.LastSignState.Height)
+}

cmd/tendermint/commands/rollback.go

@@ -0,0 +1,85 @@
+package commands
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	dbm "github.com/tendermint/tm-db"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/os"
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+)
+
+var RollbackStateCmd = &cobra.Command{
+	Use:   "rollback",
+	Short: "rollback tendermint state by one height",
+	Long: `
+A state rollback is performed to recover from an incorrect application state transition,
+when Tendermint has persisted an incorrect app hash and is thus unable to make
+progress. Rollback overwrites a state at height n with the state at height n - 1.
+The application should also roll back to height n - 1. No blocks are removed, so upon
+restarting Tendermint the transactions in block n will be re-executed against the
+application.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		height, hash, err := RollbackState(config)
+		if err != nil {
+			return fmt.Errorf("failed to rollback state: %w", err)
+		}
+
+		fmt.Printf("Rolled back state to height %d and hash %v", height, hash)
+		return nil
+	},
+}
+
+// RollbackState takes the state at the current height n and overwrites it with the state
+// at height n - 1. Note state here refers to tendermint state not application state.
+// Returns the latest state height and app hash alongside an error if there was one.
+func RollbackState(config *cfg.Config) (int64, []byte, error) {
+	// use the parsed config to load the block and state store
+	blockStore, stateStore, err := loadStateAndBlockStore(config)
+	if err != nil {
+		return -1, nil, err
+	}
+	defer func() {
+		_ = blockStore.Close()
+		_ = stateStore.Close()
+	}()
+
+	// rollback the last state
+	return state.Rollback(blockStore, stateStore)
+}
+
+func loadStateAndBlockStore(config *cfg.Config) (*store.BlockStore, state.Store, error) {
+	dbType := dbm.BackendType(config.DBBackend)
+
+	if !os.FileExists(filepath.Join(config.DBDir(), "blockstore.db")) {
+		return nil, nil, fmt.Errorf("no blockstore found in %v", config.DBDir())
+	}
+
+	// Get BlockStore
+	blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir())
+	if err != nil {
+		return nil, nil, err
+	}
+	blockStore := store.NewBlockStore(blockStoreDB)
+
+	if !os.FileExists(filepath.Join(config.DBDir(), "state.db")) {
+		return nil, nil, fmt.Errorf("no statestore found in %v", config.DBDir())
+	}
+
+	// Get StateStore
+	stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
+	if err != nil {
+		return nil, nil, err
+	}
+	stateStore := state.NewStore(stateDB, state.StoreOptions{
+		DiscardABCIResponses: config.Storage.DiscardABCIResponses,
+	})
+
+	return blockStore, stateStore, nil
+}

cmd/tendermint/commands/root.go

@@ -3,6 +3,7 @@ package commands
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -23,48 +24,73 @@ func init() {
 }
 
 func registerFlagsRootCmd(cmd *cobra.Command) {
-	cmd.PersistentFlags().String("log_level", config.LogLevel, "Log level")
+	cmd.PersistentFlags().String("log_level", config.LogLevel, "log level")
 }
 
 // ParseConfig retrieves the default environment configuration,
 // sets up the Tendermint root and ensures that the root exists
-func ParseConfig() (*cfg.Config, error) {
+func ParseConfig(cmd *cobra.Command) (*cfg.Config, error) {
 	conf := cfg.DefaultConfig()
 	err := viper.Unmarshal(conf)
 	if err != nil {
 		return nil, err
 	}
+
+	var home string
+	if os.Getenv("TMHOME") != "" {
+		home = os.Getenv("TMHOME")
+	} else {
+		home, err = cmd.Flags().GetString(cli.HomeFlag)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	conf.RootDir = home
+
 	conf.SetRoot(conf.RootDir)
 	cfg.EnsureRoot(conf.RootDir)
-	if err = conf.ValidateBasic(); err != nil {
+	if err := conf.ValidateBasic(); err != nil {
 		return nil, fmt.Errorf("error in config file: %v", err)
 	}
-	return conf, err
+	return conf, nil
 }
 
 // RootCmd is the root command for Tendermint core.
 var RootCmd = &cobra.Command{
 	Use:   "tendermint",
-	Short: "Tendermint Core (BFT Consensus) in Go",
+	Short: "BFT state machine replication for applications in any programming languages",
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
 		if cmd.Name() == VersionCmd.Name() {
 			return nil
 		}
-		config, err = ParseConfig()
+
+		config, err = ParseConfig(cmd)
 		if err != nil {
 			return err
 		}
+
 		if config.LogFormat == cfg.LogFormatJSON {
 			logger = log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout))
 		}
-		logger, err = tmflags.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel())
+
+		logger, err = tmflags.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel)
 		if err != nil {
 			return err
 		}
+
 		if viper.GetBool(cli.TraceFlag) {
 			logger = log.NewTracingLogger(logger)
 		}
+
 		logger = logger.With("module", "main")
 		return nil
 	},
 }
+
+// deprecateSnakeCase is a util function for 0.34.1. Should be removed in 0.35
+func deprecateSnakeCase(cmd *cobra.Command, args []string) {
+	if strings.Contains(cmd.CalledAs(), "_") {
+		fmt.Println("Deprecated: snake_case commands will be replaced by hyphen-case commands in the next major release")
+	}
+}

cmd/tendermint/commands/run_node.go

@@ -22,34 +22,32 @@ var (
 // These are exposed for convenience of commands embedding a tendermint node
 func AddNodeFlags(cmd *cobra.Command) {
 	// bind flags
-	cmd.Flags().String("moniker", config.Moniker, "Node Name")
+	cmd.Flags().String("moniker", config.Moniker, "node name")
 
 	// priv val flags
 	cmd.Flags().String(
 		"priv_validator_laddr",
 		config.PrivValidatorListenAddr,
-		"Socket address to listen on for connections from external priv_validator process")
+		"socket address to listen on for connections from external priv_validator process")
 
 	// node flags
-	cmd.Flags().Bool("fast_sync", config.FastSyncMode, "Fast blockchain syncing")
+	cmd.Flags().Bool("fast_sync", config.FastSyncMode, "fast blockchain syncing")
 	cmd.Flags().BytesHexVar(
 		&genesisHash,
 		"genesis_hash",
 		[]byte{},
-		"Optional SHA-256 hash of the genesis file")
+		"optional SHA-256 hash of the genesis file")
 	cmd.Flags().Int64("consensus.double_sign_check_height", config.Consensus.DoubleSignCheckHeight,
-		"How many blocks to look back to check existence of the node's "+
+		"how many blocks to look back to check existence of the node's "+
 			"consensus votes before joining consensus")
 
 	// abci flags
 	cmd.Flags().String(
 		"proxy_app",
 		config.ProxyApp,
-		"Proxy app address, or one of: 'kvstore',"+
-			" 'persistent_kvstore',"+
-			" 'counter',"+
-			" 'counter_serial' or 'noop' for local testing.")
-	cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)")
+		"proxy app address, or one of: 'kvstore',"+
+			" 'persistent_kvstore', 'counter', 'e2e' or 'noop' for local testing.")
+	cmd.Flags().String("abci", config.ABCI, "specify abci transport (socket | grpc)")
 
 	// rpc flags
 	cmd.Flags().String("rpc.laddr", config.RPC.ListenAddress, "RPC listen address. Port required")
@@ -57,50 +55,53 @@ func AddNodeFlags(cmd *cobra.Command) {
 		"rpc.grpc_laddr",
 		config.RPC.GRPCListenAddress,
 		"GRPC listen address (BroadcastTx only). Port required")
-	cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "Enabled unsafe rpc methods")
+	cmd.Flags().Bool("rpc.unsafe", config.RPC.Unsafe, "enabled unsafe rpc methods")
 	cmd.Flags().String("rpc.pprof_laddr", config.RPC.PprofListenAddress, "pprof listen address (https://golang.org/pkg/net/http/pprof)")
 
 	// p2p flags
 	cmd.Flags().String(
 		"p2p.laddr",
 		config.P2P.ListenAddress,
-		"Node listen address. (0.0.0.0:0 means any interface, any port)")
-	cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "Comma-delimited ID@host:port seed nodes")
-	cmd.Flags().String("p2p.persistent_peers", config.P2P.PersistentPeers, "Comma-delimited ID@host:port persistent peers")
+		"node listen address. (0.0.0.0:0 means any interface, any port)")
+	cmd.Flags().String("p2p.external-address",
+		config.P2P.ExternalAddress, "ip:port address to advertise to peers for them to dial")
+	cmd.Flags().String("p2p.seeds", config.P2P.Seeds, "comma-delimited ID@host:port seed nodes")
+	cmd.Flags().String("p2p.persistent_peers", config.P2P.PersistentPeers, "comma-delimited ID@host:port persistent peers")
 	cmd.Flags().String("p2p.unconditional_peer_ids",
-		config.P2P.UnconditionalPeerIDs, "Comma-delimited IDs of unconditional peers")
-	cmd.Flags().Bool("p2p.upnp", config.P2P.UPNP, "Enable/disable UPNP port forwarding")
-	cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "Enable/disable Peer-Exchange")
-	cmd.Flags().Bool("p2p.seed_mode", config.P2P.SeedMode, "Enable/disable seed mode")
-	cmd.Flags().String("p2p.private_peer_ids", config.P2P.PrivatePeerIDs, "Comma-delimited private peer IDs")
+		config.P2P.UnconditionalPeerIDs, "comma-delimited IDs of unconditional peers")
+	cmd.Flags().Bool("p2p.upnp", config.P2P.UPNP, "enable/disable UPNP port forwarding")
+	cmd.Flags().Bool("p2p.pex", config.P2P.PexReactor, "enable/disable Peer-Exchange")
+	cmd.Flags().Bool("p2p.seed_mode", config.P2P.SeedMode, "enable/disable seed mode")
+	cmd.Flags().String("p2p.private_peer_ids", config.P2P.PrivatePeerIDs, "comma-delimited private peer IDs")
 
 	// consensus flags
 	cmd.Flags().Bool(
 		"consensus.create_empty_blocks",
 		config.Consensus.CreateEmptyBlocks,
-		"Set this to false to only produce blocks when there are txs or when the AppHash changes")
+		"set this to false to only produce blocks when there are txs or when the AppHash changes")
 	cmd.Flags().String(
 		"consensus.create_empty_blocks_interval",
 		config.Consensus.CreateEmptyBlocksInterval.String(),
-		"The possible interval between empty blocks")
+		"the possible interval between empty blocks")
 
 	// db flags
 	cmd.Flags().String(
 		"db_backend",
 		config.DBBackend,
-		"Database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb")
+		"database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb")
 	cmd.Flags().String(
 		"db_dir",
 		config.DBPath,
-		"Database directory")
+		"database directory")
 }
 
 // NewRunNodeCmd returns the command that allows the CLI to start a node.
 // It can be used with a custom PrivValidator and in-process ABCI application.
 func NewRunNodeCmd(nodeProvider nm.Provider) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "node",
-		Short: "Run the tendermint node",
+		Use:     "start",
+		Aliases: []string{"node", "run"},
+		Short:   "Run the tendermint node",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := checkGenesisHash(config); err != nil {
 				return err

cmd/tendermint/commands/show_node_id.go

@@ -10,9 +10,11 @@ import (
 
 // ShowNodeIDCmd dumps node's ID to the standard output.
 var ShowNodeIDCmd = &cobra.Command{
-	Use:   "show_node_id",
-	Short: "Show this node's ID",
-	RunE:  showNodeID,
+	Use:     "show-node-id",
+	Aliases: []string{"show_node_id"},
+	Short:   "Show this node's ID",
+	RunE:    showNodeID,
+	PreRun:  deprecateSnakeCase,
 }
 
 func showNodeID(cmd *cobra.Command, args []string) error {

cmd/tendermint/commands/show_validator.go

@@ -12,9 +12,11 @@ import (
 
 // ShowValidatorCmd adds capabilities for showing the validator info.
 var ShowValidatorCmd = &cobra.Command{
-	Use:   "show_validator",
-	Short: "Show this node's validator info",
-	RunE:  showValidator,
+	Use:     "show-validator",
+	Aliases: []string{"show_validator"},
+	Short:   "Show this node's validator info",
+	RunE:    showValidator,
+	PreRun:  deprecateSnakeCase,
 }
 
 func showValidator(cmd *cobra.Command, args []string) error {

cmd/tendermint/commands/testnet.go

@@ -42,38 +42,38 @@ const (
 
 func init() {
 	TestnetFilesCmd.Flags().IntVar(&nValidators, "v", 4,
-		"Number of validators to initialize the testnet with")
+		"number of validators to initialize the testnet with")
 	TestnetFilesCmd.Flags().StringVar(&configFile, "config", "",
-		"Config file to use (note some options may be overwritten)")
+		"config file to use (note some options may be overwritten)")
 	TestnetFilesCmd.Flags().IntVar(&nNonValidators, "n", 0,
-		"Number of non-validators to initialize the testnet with")
+		"number of non-validators to initialize the testnet with")
 	TestnetFilesCmd.Flags().StringVar(&outputDir, "o", "./mytestnet",
-		"Directory to store initialization data for the testnet")
+		"directory to store initialization data for the testnet")
 	TestnetFilesCmd.Flags().StringVar(&nodeDirPrefix, "node-dir-prefix", "node",
-		"Prefix the directory name for each node with (node results in node0, node1, ...)")
+		"prefix the directory name for each node with (node results in node0, node1, ...)")
 	TestnetFilesCmd.Flags().Int64Var(&initialHeight, "initial-height", 0,
-		"Initial height of the first block")
+		"initial height of the first block")
 
 	TestnetFilesCmd.Flags().BoolVar(&populatePersistentPeers, "populate-persistent-peers", true,
-		"Update config of each node with the list of persistent peers build using either"+
+		"update config of each node with the list of persistent peers build using either"+
 			" hostname-prefix or"+
 			" starting-ip-address")
 	TestnetFilesCmd.Flags().StringVar(&hostnamePrefix, "hostname-prefix", "node",
-		"Hostname prefix (\"node\" results in persistent peers list ID0@node0:26656, ID1@node1:26656, ...)")
+		"hostname prefix (\"node\" results in persistent peers list ID0@node0:26656, ID1@node1:26656, ...)")
 	TestnetFilesCmd.Flags().StringVar(&hostnameSuffix, "hostname-suffix", "",
-		"Hostname suffix ("+
+		"hostname suffix ("+
 			"\".xyz.com\""+
 			" results in persistent peers list ID0@node0.xyz.com:26656, ID1@node1.xyz.com:26656, ...)")
 	TestnetFilesCmd.Flags().StringVar(&startingIPAddress, "starting-ip-address", "",
-		"Starting IP address ("+
+		"starting IP address ("+
 			"\"192.168.0.1\""+
 			" results in persistent peers list ID0@192.168.0.1:26656, ID1@192.168.0.2:26656, ...)")
 	TestnetFilesCmd.Flags().StringArrayVar(&hostnames, "hostname", []string{},
-		"Manually override all hostnames of validators and non-validators (use --hostname multiple times for multiple hosts)")
+		"manually override all hostnames of validators and non-validators (use --hostname multiple times for multiple hosts)")
 	TestnetFilesCmd.Flags().IntVar(&p2pPort, "p2p-port", 26656,
 		"P2P Port")
 	TestnetFilesCmd.Flags().BoolVar(&randomMonikers, "random-monikers", false,
-		"Randomize the moniker for each generated node")
+		"randomize the moniker for each generated node")
 }
 
 // TestnetFilesCmd allows initialisation of files for a Tendermint testnet.

cmd/tendermint/commands/version.go

@@ -1,6 +1,7 @@
 package commands
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/spf13/cobra"
@@ -13,6 +14,25 @@ var VersionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Show version info",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println(version.Version)
+		if verbose {
+			values, _ := json.MarshalIndent(struct {
+				Tendermint    string `json:"tendermint"`
+				ABCI          string `json:"abci"`
+				BlockProtocol uint64 `json:"block_protocol"`
+				P2PProtocol   uint64 `json:"p2p_protocol"`
+			}{
+				Tendermint:    version.TMCoreSemVer,
+				ABCI:          version.ABCIVersion,
+				BlockProtocol: version.BlockProtocol,
+				P2PProtocol:   version.P2PProtocol,
+			}, "", "  ")
+			fmt.Println(string(values))
+		} else {
+			fmt.Println(version.TMCoreSemVer)
+		}
 	},
 }
+
+func init() {
+	VersionCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Show protocol and library versions")
+}

cmd/tendermint/main.go

@@ -18,15 +18,19 @@ func main() {
 		cmd.InitFilesCmd,
 		cmd.ProbeUpnpCmd,
 		cmd.LightCmd,
+		cmd.ReIndexEventCmd,
 		cmd.ReplayCmd,
 		cmd.ReplayConsoleCmd,
 		cmd.ResetAllCmd,
 		cmd.ResetPrivValidatorCmd,
+		cmd.ResetStateCmd,
 		cmd.ShowValidatorCmd,
 		cmd.TestnetFilesCmd,
 		cmd.ShowNodeIDCmd,
 		cmd.GenNodeKeyCmd,
 		cmd.VersionCmd,
+		cmd.RollbackStateCmd,
+		cmd.CompactGoLevelDBCmd,
 		debug.DebugCmd,
 		cli.NewCompletionCmd(rootCmd, true),
 	)

codecov.yml

@@ -2,8 +2,8 @@ coverage:
   precision: 2
   round: down
   range: "70...100"
-  notify:
-    after_n_builds: 4
+  github_checks:
+    annotations: false
 
   status:
     project:

config/config.go

@@ -20,6 +20,14 @@ const (
 	LogFormatPlain = "plain"
 	// LogFormatJSON is a format for json output
 	LogFormatJSON = "json"
+
+	// DefaultLogLevel defines a default log level as INFO.
+	DefaultLogLevel = "info"
+
+	// Mempool versions. V1 is prioritized mempool, v0 is regular mempool.
+	// Default is v0.
+	MempoolV0 = "v0"
+	MempoolV1 = "v1"
 )
 
 // NOTE: Most of the structs & relevant comments + the
@@ -49,6 +57,9 @@ var (
 
 	defaultNodeKeyPath  = filepath.Join(defaultConfigDir, defaultNodeKeyName)
 	defaultAddrBookPath = filepath.Join(defaultConfigDir, defaultAddrBookName)
+
+	minSubscriptionBufferSize     = 100
+	defaultSubscriptionBufferSize = 200
 )
 
 // Config defines the top level configuration for a Tendermint node
@@ -63,6 +74,7 @@ type Config struct {
 	StateSync       *StateSyncConfig       `mapstructure:"statesync"`
 	FastSync        *FastSyncConfig        `mapstructure:"fastsync"`
 	Consensus       *ConsensusConfig       `mapstructure:"consensus"`
+	Storage         *StorageConfig         `mapstructure:"storage"`
 	TxIndex         *TxIndexConfig         `mapstructure:"tx_index"`
 	Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"`
 }
@@ -77,6 +89,7 @@ func DefaultConfig() *Config {
 		StateSync:       DefaultStateSyncConfig(),
 		FastSync:        DefaultFastSyncConfig(),
 		Consensus:       DefaultConsensusConfig(),
+		Storage:         DefaultStorageConfig(),
 		TxIndex:         DefaultTxIndexConfig(),
 		Instrumentation: DefaultInstrumentationConfig(),
 	}
@@ -92,6 +105,7 @@ func TestConfig() *Config {
 		StateSync:       TestStateSyncConfig(),
 		FastSync:        TestFastSyncConfig(),
 		Consensus:       TestConsensusConfig(),
+		Storage:         TestStorageConfig(),
 		TxIndex:         TestTxIndexConfig(),
 		Instrumentation: TestInstrumentationConfig(),
 	}
@@ -225,7 +239,7 @@ func DefaultBaseConfig() BaseConfig {
 		Moniker:            defaultMoniker,
 		ProxyApp:           "tcp://127.0.0.1:26658",
 		ABCI:               "socket",
-		LogLevel:           DefaultPackageLogLevels(),
+		LogLevel:           DefaultLogLevel,
 		LogFormat:          LogFormatPlain,
 		FastSyncMode:       true,
 		FilterPeers:        false,
@@ -284,17 +298,6 @@ func (cfg BaseConfig) ValidateBasic() error {
 	return nil
 }
 
-// DefaultLogLevel returns a default log level of "error"
-func DefaultLogLevel() string {
-	return "error"
-}
-
-// DefaultPackageLogLevels returns a default log level setting so all packages
-// log at "error", while the `state` and `main` packages log at "info"
-func DefaultPackageLogLevels() string {
-	return fmt.Sprintf("main:info,state:info,statesync:info,*:%s", DefaultLogLevel())
-}
-
 //-----------------------------------------------------------------------------
 // RPCConfig
 
@@ -350,6 +353,29 @@ type RPCConfig struct {
 	// to the estimated maximum number of broadcast_tx_commit calls per block.
 	MaxSubscriptionsPerClient int `mapstructure:"max_subscriptions_per_client"`
 
+	// The number of events that can be buffered per subscription before
+	// returning `ErrOutOfCapacity`.
+	SubscriptionBufferSize int `mapstructure:"experimental_subscription_buffer_size"`
+
+	// The maximum number of responses that can be buffered per WebSocket
+	// client. If clients cannot read from the WebSocket endpoint fast enough,
+	// they will be disconnected, so increasing this parameter may reduce the
+	// chances of them being disconnected (but will cause the node to use more
+	// memory).
+	//
+	// Must be at least the same as `SubscriptionBufferSize`, otherwise
+	// connections may be dropped unnecessarily.
+	WebSocketWriteBufferSize int `mapstructure:"experimental_websocket_write_buffer_size"`
+
+	// If a WebSocket client cannot read fast enough, at present we may
+	// silently drop events instead of generating an error or disconnecting the
+	// client.
+	//
+	// Enabling this parameter will cause the WebSocket connection to be closed
+	// instead if it cannot read fast enough, allowing for greater
+	// predictability in subscription behaviour.
+	CloseOnSlowClient bool `mapstructure:"experimental_close_on_slow_client"`
+
 	// How long to wait for a tx to be committed during /broadcast_tx_commit
 	// WARNING: Using a value larger than 10s will result in increasing the
 	// global HTTP write timeout, which applies to all connections and endpoints.
@@ -363,7 +389,7 @@ type RPCConfig struct {
 	MaxHeaderBytes int `mapstructure:"max_header_bytes"`
 
 	// The path to a file containing certificate that is used to create the HTTPS server.
-	// Migth be either absolute path or path related to tendermint's config directory.
+	// Might be either absolute path or path related to Tendermint's config directory.
 	//
 	// If the certificate is signed by a certificate authority,
 	// the certFile should be the concatenation of the server's certificate, any intermediates,
@@ -374,7 +400,7 @@ type RPCConfig struct {
 	TLSCertFile string `mapstructure:"tls_cert_file"`
 
 	// The path to a file containing matching private key that is used to create the HTTPS server.
-	// Migth be either absolute path or path related to tendermint's config directory.
+	// Might be either absolute path or path related to tendermint's config directory.
 	//
 	// NOTE: both tls_cert_file and tls_key_file must be present for Tendermint to create HTTPS server.
 	// Otherwise, HTTP server is run.
@@ -399,7 +425,9 @@ func DefaultRPCConfig() *RPCConfig {
 
 		MaxSubscriptionClients:    100,
 		MaxSubscriptionsPerClient: 5,
+		SubscriptionBufferSize:    defaultSubscriptionBufferSize,
 		TimeoutBroadcastTxCommit:  10 * time.Second,
+		WebSocketWriteBufferSize:  defaultSubscriptionBufferSize,
 
 		MaxBodyBytes:   int64(1000000), // 1MB
 		MaxHeaderBytes: 1 << 20,        // same as the net/http default
@@ -433,6 +461,18 @@ func (cfg *RPCConfig) ValidateBasic() error {
 	if cfg.MaxSubscriptionsPerClient < 0 {
 		return errors.New("max_subscriptions_per_client can't be negative")
 	}
+	if cfg.SubscriptionBufferSize < minSubscriptionBufferSize {
+		return fmt.Errorf(
+			"experimental_subscription_buffer_size must be >= %d",
+			minSubscriptionBufferSize,
+		)
+	}
+	if cfg.WebSocketWriteBufferSize < cfg.SubscriptionBufferSize {
+		return fmt.Errorf(
+			"experimental_websocket_write_buffer_size must be >= experimental_subscription_buffer_size (%d)",
+			cfg.SubscriptionBufferSize,
+		)
+	}
 	if cfg.TimeoutBroadcastTxCommit < 0 {
 		return errors.New("timeout_broadcast_tx_commit can't be negative")
 	}
@@ -644,6 +684,13 @@ func DefaultFuzzConnConfig() *FuzzConnConfig {
 
 // MempoolConfig defines the configuration options for the Tendermint mempool
 type MempoolConfig struct {
+	// Mempool version to use:
+	//  1) "v0" - (default) FIFO mempool.
+	//  2) "v1" - prioritized mempool.
+	// WARNING: There's a known memory leak with the prioritized mempool
+	// that the team are working on. Read more here:
+	// https://github.com/tendermint/tendermint/issues/8775
+	Version   string `mapstructure:"version"`
 	RootDir   string `mapstructure:"home"`
 	Recheck   bool   `mapstructure:"recheck"`
 	Broadcast bool   `mapstructure:"broadcast"`
@@ -656,27 +703,50 @@ type MempoolConfig struct {
 	MaxTxsBytes int64 `mapstructure:"max_txs_bytes"`
 	// Size of the cache (used to filter transactions we saw earlier) in transactions
 	CacheSize int `mapstructure:"cache_size"`
+	// Do not remove invalid transactions from the cache (default: false)
+	// Set to true if it's not possible for any invalid transaction to become
+	// valid again in the future.
+	KeepInvalidTxsInCache bool `mapstructure:"keep-invalid-txs-in-cache"`
 	// Maximum size of a single transaction
 	// NOTE: the max size of a tx transmitted over the network is {max_tx_bytes}.
 	MaxTxBytes int `mapstructure:"max_tx_bytes"`
 	// Maximum size of a batch of transactions to send to a peer
 	// Including space needed by encoding (one varint per transaction).
+	// XXX: Unused due to https://github.com/tendermint/tendermint/issues/5796
 	MaxBatchBytes int `mapstructure:"max_batch_bytes"`
+
+	// TTLDuration, if non-zero, defines the maximum amount of time a transaction
+	// can exist for in the mempool.
+	//
+	// Note, if TTLNumBlocks is also defined, a transaction will be removed if it
+	// has existed in the mempool at least TTLNumBlocks number of blocks or if it's
+	// insertion time into the mempool is beyond TTLDuration.
+	TTLDuration time.Duration `mapstructure:"ttl-duration"`
+
+	// TTLNumBlocks, if non-zero, defines the maximum number of blocks a transaction
+	// can exist for in the mempool.
+	//
+	// Note, if TTLDuration is also defined, a transaction will be removed if it
+	// has existed in the mempool at least TTLNumBlocks number of blocks or if
+	// it's insertion time into the mempool is beyond TTLDuration.
+	TTLNumBlocks int64 `mapstructure:"ttl-num-blocks"`
 }
 
 // DefaultMempoolConfig returns a default configuration for the Tendermint mempool
 func DefaultMempoolConfig() *MempoolConfig {
 	return &MempoolConfig{
+		Version:   MempoolV0,
 		Recheck:   true,
 		Broadcast: true,
 		WalPath:   "",
 		// Each signature verification takes .5ms, Size reduced until we implement
 		// ABCI Recheck
-		Size:          5000,
-		MaxTxsBytes:   1024 * 1024 * 1024, // 1GB
-		CacheSize:     10000,
-		MaxTxBytes:    1024 * 1024,      // 1MB
-		MaxBatchBytes: 10 * 1024 * 1024, // 10MB
+		Size:         5000,
+		MaxTxsBytes:  1024 * 1024 * 1024, // 1GB
+		CacheSize:    10000,
+		MaxTxBytes:   1024 * 1024, // 1MB
+		TTLDuration:  0 * time.Second,
+		TTLNumBlocks: 0,
 	}
 }
 
@@ -712,12 +782,6 @@ func (cfg *MempoolConfig) ValidateBasic() error {
 	if cfg.MaxTxBytes < 0 {
 		return errors.New("max_tx_bytes can't be negative")
 	}
-	if cfg.MaxBatchBytes < 0 {
-		return errors.New("max_batch_bytes can't be negative")
-	}
-	if cfg.MaxBatchBytes <= cfg.MaxTxBytes {
-		return errors.New("max_batch_bytes can't be less or equal to max_tx_bytes")
-	}
 	return nil
 }
 
@@ -726,13 +790,15 @@ func (cfg *MempoolConfig) ValidateBasic() error {
 
 // StateSyncConfig defines the configuration for the Tendermint state sync service
 type StateSyncConfig struct {
-	Enable        bool          `mapstructure:"enable"`
-	TempDir       string        `mapstructure:"temp_dir"`
-	RPCServers    []string      `mapstructure:"rpc_servers"`
-	TrustPeriod   time.Duration `mapstructure:"trust_period"`
-	TrustHeight   int64         `mapstructure:"trust_height"`
-	TrustHash     string        `mapstructure:"trust_hash"`
-	DiscoveryTime time.Duration `mapstructure:"discovery_time"`
+	Enable              bool          `mapstructure:"enable"`
+	TempDir             string        `mapstructure:"temp_dir"`
+	RPCServers          []string      `mapstructure:"rpc_servers"`
+	TrustPeriod         time.Duration `mapstructure:"trust_period"`
+	TrustHeight         int64         `mapstructure:"trust_height"`
+	TrustHash           string        `mapstructure:"trust_hash"`
+	DiscoveryTime       time.Duration `mapstructure:"discovery_time"`
+	ChunkRequestTimeout time.Duration `mapstructure:"chunk_request_timeout"`
+	ChunkFetchers       int32         `mapstructure:"chunk_fetchers"`
 }
 
 func (cfg *StateSyncConfig) TrustHashBytes() []byte {
@@ -747,8 +813,10 @@ func (cfg *StateSyncConfig) TrustHashBytes() []byte {
 // DefaultStateSyncConfig returns a default configuration for the state sync service
 func DefaultStateSyncConfig() *StateSyncConfig {
 	return &StateSyncConfig{
-		TrustPeriod:   168 * time.Hour,
-		DiscoveryTime: 15 * time.Second,
+		TrustPeriod:         168 * time.Hour,
+		DiscoveryTime:       15 * time.Second,
+		ChunkRequestTimeout: 10 * time.Second,
+		ChunkFetchers:       4,
 	}
 }
 
@@ -763,28 +831,47 @@ func (cfg *StateSyncConfig) ValidateBasic() error {
 		if len(cfg.RPCServers) == 0 {
 			return errors.New("rpc_servers is required")
 		}
+
 		if len(cfg.RPCServers) < 2 {
 			return errors.New("at least two rpc_servers entries is required")
 		}
+
 		for _, server := range cfg.RPCServers {
 			if len(server) == 0 {
 				return errors.New("found empty rpc_servers entry")
 			}
 		}
+
+		if cfg.DiscoveryTime != 0 && cfg.DiscoveryTime < 5*time.Second {
+			return errors.New("discovery time must be 0s or greater than five seconds")
+		}
+
 		if cfg.TrustPeriod <= 0 {
 			return errors.New("trusted_period is required")
 		}
+
 		if cfg.TrustHeight <= 0 {
 			return errors.New("trusted_height is required")
 		}
+
 		if len(cfg.TrustHash) == 0 {
 			return errors.New("trusted_hash is required")
 		}
+
 		_, err := hex.DecodeString(cfg.TrustHash)
 		if err != nil {
 			return fmt.Errorf("invalid trusted_hash: %w", err)
 		}
+
+		if cfg.ChunkRequestTimeout < 5*time.Second {
+			return errors.New("chunk_request_timeout must be at least 5 seconds")
+		}
+
+		if cfg.ChunkFetchers <= 0 {
+			return errors.New("chunk_fetchers is required")
+		}
 	}
+
 	return nil
 }
 
@@ -987,11 +1074,41 @@ func (cfg *ConsensusConfig) ValidateBasic() error {
 }
 
 //-----------------------------------------------------------------------------
+// StorageConfig
+
+// StorageConfig allows more fine-grained control over certain storage-related
+// behavior.
+type StorageConfig struct {
+	// Set to false to ensure ABCI responses are persisted. ABCI responses are
+	// required for `/block_results` RPC queries, and to reindex events in the
+	// command-line tool.
+	DiscardABCIResponses bool `mapstructure:"discard_abci_responses"`
+}
+
+// DefaultStorageConfig returns the default configuration options relating to
+// Tendermint storage optimization.
+func DefaultStorageConfig() *StorageConfig {
+	return &StorageConfig{
+		DiscardABCIResponses: false,
+	}
+}
+
+// TestStorageConfig returns storage configuration that can be used for
+// testing.
+func TestStorageConfig() *StorageConfig {
+	return &StorageConfig{
+		DiscardABCIResponses: false,
+	}
+}
+
+// -----------------------------------------------------------------------------
 // TxIndexConfig
 // Remember that Event has the following structure:
 // type: [
-//  key: value,
-//  ...
+//
+//	key: value,
+//	...
+//
 // ]
 //
 // CompositeKeys are constructed by `type.key`
@@ -1004,7 +1121,12 @@ type TxIndexConfig struct {
 	//   1) "null"
 	//   2) "kv" (default) - the simplest possible indexer,
 	//      backed by key-value storage (defaults to levelDB; see DBBackend).
+	//   3) "psql" - the indexer services backed by PostgreSQL.
 	Indexer string `mapstructure:"indexer"`
+
+	// The PostgreSQL connection configuration, the connection format:
+	// postgresql://<user>:<password>@<host>:<port>/<db>?<opts>
+	PsqlConn string `mapstructure:"psql-conn"`
 }
 
 // DefaultTxIndexConfig returns a default configuration for the transaction indexer.