mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-08-20 22:26:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main/pbts' of github.com:tendermint/tendermint into main/pbts

Browse Source
This commit is contained in:
Daniel Cason
2022-05-25 12:36:18 +02:00
parent be7e3ddada 387c8570fa
commit fcd560282f
156 changed files with 9393 additions and 2588 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/dependabot.yml
+25
View File
@@ -4,6 +4,30 @@ updates:
directory: "/"
schedule:
interval: weekly
day: monday
target-branch: "master"
open-pull-requests-limit: 10
labels:
- T:dependencies
- S:automerge
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: weekly
day: monday
target-branch: "v0.35.x"
open-pull-requests-limit: 10
labels:
- T:dependencies
- S:automerge
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: weekly
day: monday
target-branch: "v0.34.x"
open-pull-requests-limit: 10
labels:
- T:dependencies
@@ -13,6 +37,7 @@ updates:
directory: "/docs"
schedule:
interval: weekly
day: monday
open-pull-requests-limit: 10
###################################
.github/workflows/check-generated.yml
+76
View File
@@ -0,0 +1,76 @@
# Verify that generated code is up-to-date.
#
# Note that we run these checks regardless whether the input files have
# changed, because generated code can change in response to toolchain updates
# even if no files in the repository are modified.
name: Check generated code
on:
pull_request:
branches:
- master
permissions:
contents: read
jobs:
check-mocks:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
with:
go-version: '1.17'
- uses: actions/checkout@v3
- name: "Check generated mocks"
run: |
set -euo pipefail
make mockery 2>/dev/null
if ! git diff --stat --exit-code ; then
echo ">> ERROR:"
echo ">>"
echo ">> Generated mocks require update (either Mockery or source files may have changed)."
echo ">> Ensure your tools are up-to-date, re-run 'make mockery' and update this PR."
echo ">>"
exit 1
fi
check-proto:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v3
with:
go-version: '1.17'
- uses: actions/checkout@v3
with:
fetch-depth: 1 # we need a .git directory to run git diff
- name: "Check protobuf generated code"
run: |
set -euo pipefail
# Install buf and gogo tools, so that differences that arise from
# toolchain differences are also caught.
readonly tools="$(mktemp -d)"
export PATH="${PATH}:${tools}/bin"
export GOBIN="${tools}/bin"
readonly base='https://github.com/bufbuild/buf/releases/latest/download'
readonly OS="$(uname -s)" ARCH="$(uname -m)"
curl -sSL "${base}/buf-${OS}-${ARCH}.tar.gz" \
| tar -xzf - -C "$tools" --strip-components=1
go install github.com/gogo/protobuf/protoc-gen-gogofaster@latest
make proto-gen
if ! git diff --stat --exit-code ; then
echo ">> ERROR:"
echo ">>"
echo ">> Protobuf generated code requires update (either tools or .proto files may have changed)."
echo ">> Ensure your tools are up-to-date, re-run 'make proto-gen' and update this PR."
echo ">>"
exit 1
fi
.github/workflows/docker.yml
+3 -3
View File
@@ -39,17 +39,17 @@ jobs:
platforms: all
- name: Set up Docker Build
uses: docker/setup-buildx-action@v1.7.0
uses: docker/setup-buildx-action@v2.0.0
- name: Login to DockerHub
if: ${{ github.event_name != 'pull_request' }}
uses: docker/login-action@v1.14.1
uses: docker/login-action@v2.0.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Publish to Docker Hub
uses: docker/build-push-action@v2.10.0
uses: docker/build-push-action@v3.0.0
with:
context: .
file: ./DOCKER/Dockerfile
.github/workflows/docs-deployment.yml
+62
View File
@@ -0,0 +1,62 @@
# Build and deploy the docs.tendermint.com website content.
# The static content is published to GitHub Pages.
#
# For documentation build info, see docs/DOCS_README.md.
name: Build static documentation site
on:
workflow_dispatch: # allow manual updates
push:
branches:
- master
paths:
- docs/**
- spec/**
jobs:
# This is split into two jobs so that the build, which runs npm, does not
# have write access to anything. The deploy requires write access to publish
# to the branch used by GitHub Pages, however, so we can't just make the
# whole workflow read-only.
build:
name: VuePress build
runs-on: ubuntu-latest
container:
image: alpine:latest
permissions:
contents: read
steps:
- name: Install generator dependencies
run: |
apk add --no-cache make bash git npm
- uses: actions/checkout@v3
with:
# We need to fetch full history so the backport branches for previous
# versions will be available for the build.
fetch-depth: 0
- name: Build documentation
run: |
git config --global --add safe.directory "$PWD"
make build-docs
- uses: actions/upload-artifact@v3
with:
name: build-output
path: ~/output/
deploy:
name: Deploy to GitHub Pages
runs-on: ubuntu-latest
needs: build
permissions:
contents: write
steps:
- uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: build-output
path: ~/output
- name: Deploy to GitHub Pages
uses: JamesIves/github-pages-deploy-action@v4
with:
branch: 'docs-tendermint-com'
folder: ~/output
single-commit: true
.github/workflows/linkchecker.yml
-12
View File
@@ -1,12 +0,0 @@
name: Check Markdown links
on:
schedule:
- cron: '* */24 * * *'
jobs:
markdown-link-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: creachadair/github-action-markdown-link-check@master
with:
folder-path: "docs"
.github/workflows/lint.yml
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
**/**.go
go.mod
go.sum
- uses: golangci/golangci-lint-action@v3.1.0
- uses: golangci/golangci-lint-action@v3
with:
# Required: the version of golangci-lint is required and
# must be specified without patch version: we always use the
.github/workflows/release.yml
+2 -2
View File
@@ -21,7 +21,7 @@ jobs:
go-version: '1.17'
- name: Build
uses: goreleaser/goreleaser-action@v2
uses: goreleaser/goreleaser-action@v3
if: ${{ github.event_name == 'pull_request' }}
with:
version: latest
@@ -30,7 +30,7 @@ jobs:
- run: echo https://github.com/tendermint/tendermint/blob/${GITHUB_REF#refs/tags/}/CHANGELOG.md#${GITHUB_REF#refs/tags/} > ../release_notes.md
- name: Release
uses: goreleaser/goreleaser-action@v2
uses: goreleaser/goreleaser-action@v3
if: startsWith(github.ref, 'refs/tags/')
with:
version: latest
CHANGELOG_PENDING.md
+1
View File
@@ -21,6 +21,7 @@ Special thanks to external contributors on this release:
- [rpc] \#7982 Add new Events interface and deprecate Subscribe. (@creachadair)
- [cli] \#8081 make the reset command safe to use by intoducing `reset-state` command. Fixed by \#8259. (@marbar3778, @cmwaters)
- [config] \#8222 default indexer configuration to null. (@creachadair)
- [rpc] \#8570 rework timeouts to be per-method instead of global. (@creachadair)
- Apps
Makefile
+17 -1
View File
@@ -226,7 +226,8 @@ DESTINATION = ./index.html.md
build-docs:
@cd docs && \
while read -r branch path_prefix; do \
(git checkout $${branch} && npm ci && VUEPRESS_BASE="/$${path_prefix}/" npm run build) ; \
( git checkout $${branch} && npm ci --quiet && \
VUEPRESS_BASE="/$${path_prefix}/" npm run build --quiet ) ; \
mkdir -p ~/output/$${path_prefix} ; \
cp -r .vuepress/dist/* ~/output/$${path_prefix}/ ; \
cp ~/output/$${path_prefix}/index.html ~/output ; \
@@ -250,6 +251,21 @@ mockery:
go generate -run="./scripts/mockery_generate.sh" ./...
.PHONY: mockery
###############################################################################
### Metrics ###
###############################################################################
metrics: testdata-metrics
go generate -run="scripts/metricsgen" ./...
.PHONY: metrics
# By convention, the go tool ignores subdirectories of directories named
# 'testdata'. This command invokes the generate command on the folder directly
# to avoid this.
testdata-metrics:
ls ./scripts/metricsgen/testdata | xargs -I{} go generate -run="scripts/metricsgen" ./scripts/metricsgen/testdata/{}
.PHONY: testdata-metrics
###############################################################################
### Local testnet using docker ###
###############################################################################
UPGRADING.md
+12 -9
View File
@@ -212,22 +212,25 @@ and one function have moved to the Tendermint `crypto` package:
The format of all tendermint on-disk database keys changes in
0.35. Upgrading nodes must either re-sync all data or run a migration
script provided in this release. The script located in
`github.com/tendermint/tendermint/scripts/keymigrate/migrate.go`
provides the function `Migrate(context.Context, db.DB)` which you can
operationalize as makes sense for your deployment.
script provided in this release.
The script located in
`github.com/tendermint/tendermint/scripts/keymigrate/migrate.go` provides the
function `Migrate(context.Context, db.DB)` which you can operationalize as
makes sense for your deployment.
For ease of use the `tendermint` command includes a CLI version of the
migration script, which you can invoke, as in:
tendermint key-migrate
This reads the configuration file as normal and allows the
`--db-backend` and `--db-dir` flags to change database operations as
needed.
This reads the configuration file as normal and allows the `--db-backend` and
`--db-dir` flags to override the database location as needed.
The migration operation is idempotent and can be run more than once,
if needed.
The migration operation is intended to be idempotent, and should be safe to
rerun on the same database multiple times. As a safety measure, however, we
recommend that operators test out the migration on a copy of the database
first, if it is practical to do so, before applying it to the production data.
### CLI Changes
abci/client/socket_client.go
+11 -12
View File
@@ -17,12 +17,6 @@ import (
"github.com/tendermint/tendermint/libs/service"
)
const (
// reqQueueSize is the max number of queued async requests.
// (memory: 256MB max assuming 1MB transactions)
reqQueueSize = 256
)
// This is goroutine-safe, but users should beware that the application in
// general is not meant to be interfaced with concurrent callers.
type socketClient struct {
@@ -48,7 +42,7 @@ var _ Client = (*socketClient)(nil)
func NewSocketClient(logger log.Logger, addr string, mustConnect bool) Client {
cli := &socketClient{
logger: logger,
reqQueue: make(chan *requestAndResponse, reqQueueSize),
reqQueue: make(chan *requestAndResponse),
mustConnect: mustConnect,
addr: addr,
reqSent: list.New(),
@@ -118,6 +112,11 @@ func (cli *socketClient) sendRequestsRoutine(ctx context.Context, conn io.Writer
case <-ctx.Done():
return
case reqres := <-cli.reqQueue:
// N.B. We must enqueue before sending out the request, otherwise the
// server may reply before we do it, and the receiver will fail for an
// unsolicited reply.
cli.trackRequest(reqres)
if err := types.WriteMessage(reqres.Request, bw); err != nil {
cli.stopForError(fmt.Errorf("write to buffer: %w", err))
return
@@ -158,14 +157,15 @@ func (cli *socketClient) recvResponseRoutine(ctx context.Context, conn io.Reader
}
}
func (cli *socketClient) willSendReq(reqres *requestAndResponse) {
cli.mtx.Lock()
defer cli.mtx.Unlock()
func (cli *socketClient) trackRequest(reqres *requestAndResponse) {
// N.B. We must NOT hold the client state lock while checking this, or we
// may deadlock with shutdown.
if !cli.IsRunning() {
return
}
cli.mtx.Lock()
defer cli.mtx.Unlock()
cli.reqSent.PushBack(reqres)
}
@@ -199,7 +199,6 @@ func (cli *socketClient) doRequest(ctx context.Context, req *types.Request) (*ty
}
reqres := makeReqRes(req)
cli.willSendReq(reqres)
select {
case cli.reqQueue <- reqres:
abci/types/types.go
+45
View File
@@ -5,6 +5,9 @@ import (
"encoding/json"
"github.com/gogo/protobuf/jsonpb"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/crypto/encoding"
"github.com/tendermint/tendermint/internal/jsontypes"
)
const (
@@ -135,6 +138,48 @@ func (r *EventAttribute) UnmarshalJSON(b []byte) error {
return jsonpbUnmarshaller.Unmarshal(reader, r)
}
// validatorUpdateJSON is the JSON encoding of a validator update.
//
// It handles translation of public keys from the protobuf representation to
// the legacy Amino-compatible format expected by RPC clients.
type validatorUpdateJSON struct {
PubKey json.RawMessage `json:"pub_key,omitempty"`
Power int64 `json:"power,string"`
}
func (v *ValidatorUpdate) MarshalJSON() ([]byte, error) {
key, err := encoding.PubKeyFromProto(v.PubKey)
if err != nil {
return nil, err
}
jkey, err := jsontypes.Marshal(key)
if err != nil {
return nil, err
}
return json.Marshal(validatorUpdateJSON{
PubKey: jkey,
Power: v.GetPower(),
})
}
func (v *ValidatorUpdate) UnmarshalJSON(data []byte) error {
var vu validatorUpdateJSON
if err := json.Unmarshal(data, &vu); err != nil {
return err
}
var key crypto.PubKey
if err := jsontypes.Unmarshal(vu.PubKey, &key); err != nil {
return err
}
pkey, err := encoding.PubKeyToProto(key)
if err != nil {
return err
}
v.PubKey = pkey
v.Power = vu.Power
return nil
}
// Some compile time assertions to ensure we don't
// have accidental runtime surprises later on.
abci/types/types.pb.go
+211 -300
View File
@@ -1645,7 +1645,7 @@ type RequestFinalizeBlock struct {
Txs [][]byte `protobuf:"bytes,1,rep,name=txs,proto3" json:"txs,omitempty"`
DecidedLastCommit CommitInfo `protobuf:"bytes,2,opt,name=decided_last_commit,json=decidedLastCommit,proto3" json:"decided_last_commit"`
ByzantineValidators []Misbehavior `protobuf:"bytes,3,rep,name=byzantine_validators,json=byzantineValidators,proto3" json:"byzantine_validators"`
// hash is the merkle root hash of the fields of the proposed block.
// hash is the merkle root hash of the fields of the decided block.
Hash []byte `protobuf:"bytes,4,opt,name=hash,proto3" json:"hash,omitempty"`
Height int64 `protobuf:"varint,5,opt,name=height,proto3" json:"height,omitempty"`
Time time.Time `protobuf:"bytes,6,opt,name=time,proto3,stdtime" json:"time"`
@@ -3255,8 +3255,6 @@ type ResponseFinalizeBlock struct {
TxResults []*ExecTxResult `protobuf:"bytes,2,rep,name=tx_results,json=txResults,proto3" json:"tx_results,omitempty"`
ValidatorUpdates []ValidatorUpdate `protobuf:"bytes,3,rep,name=validator_updates,json=validatorUpdates,proto3" json:"validator_updates"`
ConsensusParamUpdates *types1.ConsensusParams `protobuf:"bytes,4,opt,name=consensus_param_updates,json=consensusParamUpdates,proto3" json:"consensus_param_updates,omitempty"`
AppHash []byte `protobuf:"bytes,5,opt,name=app_hash,json=appHash,proto3" json:"app_hash,omitempty"`
RetainHeight int64 `protobuf:"varint,6,opt,name=retain_height,json=retainHeight,proto3" json:"retain_height,omitempty"`
}
func (m *ResponseFinalizeBlock) Reset() { *m = ResponseFinalizeBlock{} }
@@ -3320,20 +3318,6 @@ func (m *ResponseFinalizeBlock) GetConsensusParamUpdates() *types1.ConsensusPara
return nil
}
func (m *ResponseFinalizeBlock) GetAppHash() []byte {
if m != nil {
return m.AppHash
}
return nil
}
func (m *ResponseFinalizeBlock) GetRetainHeight() int64 {
if m != nil {
return m.RetainHeight
}
return 0
}
type CommitInfo struct {
Round int32 `protobuf:"varint,1,opt,name=round,proto3" json:"round,omitempty"`
Votes []VoteInfo `protobuf:"bytes,2,rep,name=votes,proto3" json:"votes"`
@@ -4235,223 +4219,222 @@ func init() {
func init() { proto.RegisterFile("tendermint/abci/types.proto", fileDescriptor_252557cfdd89a31a) }
var fileDescriptor_252557cfdd89a31a = []byte{
// 3451 bytes of a gzipped FileDescriptorProto
// 3439 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5b, 0xcb, 0x73, 0x23, 0xe5,
0xb5, 0x97, 0x5a, 0xef, 0x23, 0xeb, 0xe1, 0xcf, 0x66, 0xd0, 0x88, 0x19, 0x7b, 0xe8, 0xa9, 0x81,
0x99, 0x01, 0x3c, 0x5c, 0xcf, 0x1d, 0x18, 0xee, 0xc0, 0xa5, 0x6c, 0x59, 0x83, 0xcc, 0x78, 0x6c,
0xd3, 0x96, 0x4d, 0x71, 0x6f, 0x32, 0x4d, 0x4b, 0xfd, 0xd9, 0x6a, 0x46, 0x52, 0x37, 0xdd, 0x2d,
0x23, 0xb3, 0x0c, 0xc5, 0x86, 0x4a, 0x55, 0xd8, 0xa4, 0x92, 0x54, 0x85, 0x5d, 0x52, 0x95, 0xfc,
0x07, 0x59, 0x65, 0x95, 0x05, 0x8b, 0x2c, 0x58, 0x25, 0xa9, 0x2c, 0x48, 0x0a, 0x76, 0xf9, 0x07,
0xb2, 0x4b, 0x52, 0xdf, 0xa3, 0x5f, 0x52, 0xb7, 0x1e, 0x0c, 0x50, 0x95, 0x0a, 0x3b, 0xf5, 0xe9,
0x73, 0x4e, 0x7f, 0x8f, 0xf3, 0x9d, 0xc7, 0xef, 0x7c, 0x82, 0x27, 0x6c, 0xdc, 0x57, 0xb1, 0xd9,
0x23, 0xb3, 0x0c, 0xc5, 0x86, 0xca, 0x82, 0x4d, 0x2a, 0x49, 0x55, 0xd8, 0x25, 0x55, 0xc9, 0x7f,
0x90, 0x55, 0x56, 0x59, 0xb0, 0x48, 0x55, 0x58, 0x25, 0xa9, 0x2c, 0x48, 0x0a, 0x76, 0xf9, 0x07,
0xb2, 0x4b, 0x52, 0xdf, 0xa3, 0x5f, 0x52, 0xb7, 0x1e, 0x0c, 0x50, 0x95, 0x0a, 0x3b, 0x7d, 0xa7,
0xcf, 0x39, 0xfd, 0x3d, 0x4e, 0x9f, 0xc7, 0xef, 0x7c, 0x82, 0x27, 0x6c, 0xdc, 0x57, 0xb1, 0xd9,
0xd3, 0xfa, 0xf6, 0x0d, 0xa5, 0xd5, 0xd6, 0x6e, 0xd8, 0x67, 0x06, 0xb6, 0xd6, 0x0c, 0x53, 0xb7,
0x75, 0x54, 0xf2, 0x5e, 0xae, 0x91, 0x97, 0xd5, 0x8b, 0x3e, 0xee, 0xb6, 0x79, 0x66, 0xd8, 0xfa,
0x0d, 0xc3, 0xd4, 0xf5, 0x63, 0xc6, 0x5f, 0xbd, 0xe0, 0x7b, 0x4d, 0xf5, 0xf8, 0xb5, 0x05, 0xde,
0x72, 0xe1, 0x87, 0xf8, 0xcc, 0x79, 0x7b, 0x71, 0x4c, 0xd6, 0x50, 0x4c, 0xa5, 0xe7, 0xbc, 0x5e,
0x3d, 0xd1, 0xf5, 0x93, 0x2e, 0xbe, 0x41, 0x9f, 0x5a, 0x83, 0xe3, 0x1b, 0xb6, 0xd6, 0xc3, 0x96,
0xad, 0xf4, 0x0c, 0xce, 0xb0, 0x7c, 0xa2, 0x9f, 0xe8, 0xf4, 0xe7, 0x0d, 0xf2, 0x8b, 0x51, 0xc5,
0x7f, 0x02, 0x64, 0x24, 0xfc, 0xee, 0x00, 0x5b, 0x36, 0x5a, 0x87, 0x24, 0x6e, 0x77, 0xf4, 0x4a,
0xfc, 0x52, 0xfc, 0x6a, 0x7e, 0xfd, 0xc2, 0xda, 0xc8, 0xe4, 0xd6, 0x38, 0x5f, 0xbd, 0xdd, 0xd1,
0x1b, 0x31, 0x89, 0xf2, 0xa2, 0x5b, 0x90, 0x3a, 0xee, 0x0e, 0xac, 0x4e, 0x45, 0xa0, 0x42, 0x17,
0xa3, 0x84, 0xee, 0x12, 0xa6, 0x46, 0x4c, 0x62, 0xdc, 0xe4, 0x53, 0x5a, 0xff, 0x58, 0xaf, 0x24,
0x26, 0x7f, 0x6a, 0xbb, 0x7f, 0x4c, 0x3f, 0x45, 0x78, 0xd1, 0x26, 0x80, 0xd6, 0xd7, 0x6c, 0xb9,
0xdd, 0x51, 0xb4, 0x7e, 0x25, 0x49, 0x25, 0x9f, 0x8c, 0x96, 0xd4, 0xec, 0x1a, 0x61, 0x6c, 0xc4,
0xa4, 0x9c, 0xe6, 0x3c, 0x90, 0xe1, 0xbe, 0x3b, 0xc0, 0xe6, 0x59, 0x25, 0x35, 0x79, 0xb8, 0x6f,
0x10, 0x26, 0x32, 0x5c, 0xca, 0x8d, 0xb6, 0x21, 0xdf, 0xc2, 0x27, 0x5a, 0x5f, 0x6e, 0x75, 0xf5,
0xf6, 0xc3, 0x4a, 0x9a, 0x0a, 0x8b, 0x51, 0xc2, 0x9b, 0x84, 0x75, 0x93, 0x70, 0x6e, 0x0a, 0x95,
0x78, 0x23, 0x26, 0x41, 0xcb, 0xa5, 0xa0, 0x97, 0x21, 0xdb, 0xee, 0xe0, 0xf6, 0x43, 0xd9, 0x1e,
0x56, 0x32, 0x54, 0xcf, 0x6a, 0x94, 0x9e, 0x1a, 0xe1, 0x6b, 0x0e, 0x1b, 0x31, 0x29, 0xd3, 0x66,
0x3f, 0xd1, 0x5d, 0x00, 0x15, 0x77, 0xb5, 0x53, 0x6c, 0x12, 0xf9, 0xec, 0xe4, 0x35, 0xd8, 0x62,
0x9c, 0xcd, 0x21, 0x1f, 0x46, 0x4e, 0x75, 0x08, 0xa8, 0x06, 0x39, 0xdc, 0x57, 0xf9, 0x74, 0x72,
0x54, 0xcd, 0xa5, 0xc8, 0xfd, 0xee, 0xab, 0xfe, 0xc9, 0x64, 0x31, 0x7f, 0x46, 0xb7, 0x21, 0xdd,
0xd6, 0x7b, 0x3d, 0xcd, 0xae, 0x00, 0xd5, 0xb0, 0x12, 0x39, 0x11, 0xca, 0xd5, 0x88, 0x49, 0x9c,
0x1f, 0xed, 0x42, 0xb1, 0xab, 0x59, 0xb6, 0x6c, 0xf5, 0x15, 0xc3, 0xea, 0xe8, 0xb6, 0x55, 0xc9,
0x53, 0x0d, 0x57, 0xa2, 0x34, 0xec, 0x68, 0x96, 0x7d, 0xe0, 0x30, 0x37, 0x62, 0x52, 0xa1, 0xeb,
0x27, 0x10, 0x7d, 0xfa, 0xf1, 0x31, 0x36, 0x5d, 0x85, 0x95, 0x85, 0xc9, 0xfa, 0xf6, 0x08, 0xb7,
0x23, 0x4f, 0xf4, 0xe9, 0x7e, 0x02, 0xfa, 0x7f, 0x58, 0xea, 0xea, 0x8a, 0xea, 0xaa, 0x93, 0xdb,
0x9d, 0x41, 0xff, 0x61, 0xa5, 0x40, 0x95, 0x5e, 0x8b, 0x1c, 0xa4, 0xae, 0xa8, 0x8e, 0x8a, 0x1a,
0x11, 0x68, 0xc4, 0xa4, 0xc5, 0xee, 0x28, 0x11, 0x3d, 0x80, 0x65, 0xc5, 0x30, 0xba, 0x67, 0xa3,
0xda, 0x8b, 0x54, 0xfb, 0xf5, 0x28, 0xed, 0x1b, 0x44, 0x66, 0x54, 0x3d, 0x52, 0xc6, 0xa8, 0xa8,
0x09, 0x65, 0xc3, 0xc4, 0x86, 0x62, 0x62, 0xd9, 0x30, 0x75, 0x43, 0xb7, 0x94, 0x6e, 0xa5, 0x44,
0x75, 0x3f, 0x1d, 0xa5, 0x7b, 0x9f, 0xf1, 0xef, 0x73, 0xf6, 0x46, 0x4c, 0x2a, 0x19, 0x41, 0x12,
0xd3, 0xaa, 0xb7, 0xb1, 0x65, 0x79, 0x5a, 0xcb, 0xd3, 0xb4, 0x52, 0xfe, 0xa0, 0xd6, 0x00, 0x09,
0xd5, 0x21, 0x8f, 0x87, 0x44, 0x5c, 0x3e, 0xd5, 0x6d, 0x5c, 0x59, 0x9c, 0x7c, 0xb0, 0xea, 0x94,
0xf5, 0x48, 0xb7, 0x31, 0x39, 0x54, 0xd8, 0x7d, 0x42, 0x0a, 0x3c, 0x76, 0x8a, 0x4d, 0xed, 0xf8,
0x8c, 0xaa, 0x91, 0xe9, 0x1b, 0x4b, 0xd3, 0xfb, 0x15, 0x44, 0x15, 0x3e, 0x13, 0xa5, 0xf0, 0x88,
0x0a, 0x11, 0x15, 0x75, 0x47, 0xa4, 0x11, 0x93, 0x96, 0x4e, 0xc7, 0xc9, 0xc4, 0xc4, 0x8e, 0xb5,
0xbe, 0xd2, 0xd5, 0xde, 0xc7, 0xfc, 0xd8, 0x2c, 0x4d, 0x36, 0xb1, 0xbb, 0x9c, 0x9b, 0x9e, 0x15,
0x75, 0x54, 0xf2, 0x1e, 0xae, 0x91, 0x87, 0xd5, 0x8b, 0x3e, 0xee, 0xb6, 0x79, 0x66, 0xd8, 0xfa,
0x0d, 0xc3, 0xd4, 0xf5, 0x63, 0xc6, 0x5f, 0xbd, 0xe0, 0x7b, 0x4c, 0xf5, 0xf8, 0xb5, 0x05, 0x9e,
0x72, 0xe1, 0x87, 0xf8, 0xcc, 0x79, 0x7a, 0x71, 0x4c, 0xd6, 0x50, 0x4c, 0xa5, 0xe7, 0x3c, 0x5e,
0x3d, 0xd1, 0xf5, 0x93, 0x2e, 0xbe, 0x41, 0x47, 0xad, 0xc1, 0xf1, 0x0d, 0x5b, 0xeb, 0x61, 0xcb,
0x56, 0x7a, 0x06, 0x67, 0x58, 0x3e, 0xd1, 0x4f, 0x74, 0xfa, 0xf3, 0x06, 0xf9, 0xc5, 0xa8, 0xe2,
0x3f, 0x01, 0x32, 0x12, 0x7e, 0x77, 0x80, 0x2d, 0x1b, 0xad, 0x43, 0x12, 0xb7, 0x3b, 0x7a, 0x25,
0x7e, 0x29, 0x7e, 0x35, 0xbf, 0x7e, 0x61, 0x6d, 0x64, 0x71, 0x6b, 0x9c, 0xaf, 0xde, 0xee, 0xe8,
0x8d, 0x98, 0x44, 0x79, 0xd1, 0x2d, 0x48, 0x1d, 0x77, 0x07, 0x56, 0xa7, 0x22, 0x50, 0xa1, 0x8b,
0x51, 0x42, 0x77, 0x09, 0x53, 0x23, 0x26, 0x31, 0x6e, 0xf2, 0x2a, 0xad, 0x7f, 0xac, 0x57, 0x12,
0x93, 0x5f, 0xb5, 0xdd, 0x3f, 0xa6, 0xaf, 0x22, 0xbc, 0x68, 0x13, 0x40, 0xeb, 0x6b, 0xb6, 0xdc,
0xee, 0x28, 0x5a, 0xbf, 0x92, 0xa4, 0x92, 0x4f, 0x46, 0x4b, 0x6a, 0x76, 0x8d, 0x30, 0x36, 0x62,
0x52, 0x4e, 0x73, 0x06, 0x64, 0xba, 0xef, 0x0e, 0xb0, 0x79, 0x56, 0x49, 0x4d, 0x9e, 0xee, 0x1b,
0x84, 0x89, 0x4c, 0x97, 0x72, 0xa3, 0x6d, 0xc8, 0xb7, 0xf0, 0x89, 0xd6, 0x97, 0x5b, 0x5d, 0xbd,
0xfd, 0xb0, 0x92, 0xa6, 0xc2, 0x62, 0x94, 0xf0, 0x26, 0x61, 0xdd, 0x24, 0x9c, 0x9b, 0x42, 0x25,
0xde, 0x88, 0x49, 0xd0, 0x72, 0x29, 0xe8, 0x65, 0xc8, 0xb6, 0x3b, 0xb8, 0xfd, 0x50, 0xb6, 0x87,
0x95, 0x0c, 0xd5, 0xb3, 0x1a, 0xa5, 0xa7, 0x46, 0xf8, 0x9a, 0xc3, 0x46, 0x4c, 0xca, 0xb4, 0xd9,
0x4f, 0x74, 0x17, 0x40, 0xc5, 0x5d, 0xed, 0x14, 0x9b, 0x44, 0x3e, 0x3b, 0x79, 0x0f, 0xb6, 0x18,
0x67, 0x73, 0xc8, 0xa7, 0x91, 0x53, 0x1d, 0x02, 0xaa, 0x41, 0x0e, 0xf7, 0x55, 0xbe, 0x9c, 0x1c,
0x55, 0x73, 0x29, 0xf2, 0xbc, 0xfb, 0xaa, 0x7f, 0x31, 0x59, 0xcc, 0xc7, 0xe8, 0x36, 0xa4, 0xdb,
0x7a, 0xaf, 0xa7, 0xd9, 0x15, 0xa0, 0x1a, 0x56, 0x22, 0x17, 0x42, 0xb9, 0x1a, 0x31, 0x89, 0xf3,
0xa3, 0x5d, 0x28, 0x76, 0x35, 0xcb, 0x96, 0xad, 0xbe, 0x62, 0x58, 0x1d, 0xdd, 0xb6, 0x2a, 0x79,
0xaa, 0xe1, 0x4a, 0x94, 0x86, 0x1d, 0xcd, 0xb2, 0x0f, 0x1c, 0xe6, 0x46, 0x4c, 0x2a, 0x74, 0xfd,
0x04, 0xa2, 0x4f, 0x3f, 0x3e, 0xc6, 0xa6, 0xab, 0xb0, 0xb2, 0x30, 0x59, 0xdf, 0x1e, 0xe1, 0x76,
0xe4, 0x89, 0x3e, 0xdd, 0x4f, 0x40, 0xff, 0x0f, 0x4b, 0x5d, 0x5d, 0x51, 0x5d, 0x75, 0x72, 0xbb,
0x33, 0xe8, 0x3f, 0xac, 0x14, 0xa8, 0xd2, 0x6b, 0x91, 0x93, 0xd4, 0x15, 0xd5, 0x51, 0x51, 0x23,
0x02, 0x8d, 0x98, 0xb4, 0xd8, 0x1d, 0x25, 0xa2, 0x07, 0xb0, 0xac, 0x18, 0x46, 0xf7, 0x6c, 0x54,
0x7b, 0x91, 0x6a, 0xbf, 0x1e, 0xa5, 0x7d, 0x83, 0xc8, 0x8c, 0xaa, 0x47, 0xca, 0x18, 0x15, 0x35,
0xa1, 0x6c, 0x98, 0xd8, 0x50, 0x4c, 0x2c, 0x1b, 0xa6, 0x6e, 0xe8, 0x96, 0xd2, 0xad, 0x94, 0xa8,
0xee, 0xa7, 0xa3, 0x74, 0xef, 0x33, 0xfe, 0x7d, 0xce, 0xde, 0x88, 0x49, 0x25, 0x23, 0x48, 0x62,
0x5a, 0xf5, 0x36, 0xb6, 0x2c, 0x4f, 0x6b, 0x79, 0x9a, 0x56, 0xca, 0x1f, 0xd4, 0x1a, 0x20, 0xa1,
0x3a, 0xe4, 0xf1, 0x90, 0x88, 0xcb, 0xa7, 0xba, 0x8d, 0x2b, 0x8b, 0x93, 0x3f, 0xac, 0x3a, 0x65,
0x3d, 0xd2, 0x6d, 0x4c, 0x3e, 0x2a, 0xec, 0x8e, 0x90, 0x02, 0x8f, 0x9d, 0x62, 0x53, 0x3b, 0x3e,
0xa3, 0x6a, 0x64, 0xfa, 0xc4, 0xd2, 0xf4, 0x7e, 0x05, 0x51, 0x85, 0xcf, 0x44, 0x29, 0x3c, 0xa2,
0x42, 0x44, 0x45, 0xdd, 0x11, 0x69, 0xc4, 0xa4, 0xa5, 0xd3, 0x71, 0x32, 0x31, 0xb1, 0x63, 0xad,
0xaf, 0x74, 0xb5, 0xf7, 0x31, 0xff, 0x6c, 0x96, 0x26, 0x9b, 0xd8, 0x5d, 0xce, 0x4d, 0xbf, 0x15,
0x62, 0x62, 0xc7, 0x7e, 0xc2, 0x66, 0x06, 0x52, 0xa7, 0x4a, 0x77, 0x80, 0xc5, 0xa7, 0x21, 0xef,
0x73, 0xac, 0xa8, 0x02, 0x99, 0x1e, 0xb6, 0x2c, 0xe5, 0x04, 0x53, 0x3f, 0x9c, 0x93, 0x9c, 0x47,
0xb1, 0x08, 0x0b, 0x7e, 0x67, 0x2a, 0x7e, 0x1c, 0x77, 0x25, 0x89, 0x9f, 0x24, 0x92, 0xa7, 0xd8,
0xa4, 0xd3, 0xe6, 0x92, 0xfc, 0x11, 0x5d, 0x86, 0x02, 0x1d, 0xb2, 0xec, 0xbc, 0x27, 0xce, 0x3a,
0x29, 0x2d, 0x50, 0xe2, 0x11, 0x67, 0x5a, 0x85, 0xbc, 0xb1, 0x6e, 0xb8, 0x2c, 0x09, 0xca, 0x02,
0xc6, 0xba, 0xe1, 0x30, 0x3c, 0x09, 0x0b, 0x64, 0x7e, 0x2e, 0x47, 0x92, 0x7e, 0x24, 0x4f, 0x68,
0x9c, 0x45, 0xfc, 0xbd, 0x00, 0xe5, 0x51, 0x07, 0x8c, 0x6e, 0x43, 0x92, 0xc4, 0x22, 0x1e, 0x56,
0xaa, 0x6b, 0x2c, 0x50, 0xad, 0x39, 0x81, 0x6a, 0xad, 0xe9, 0x04, 0xaa, 0xcd, 0xec, 0xa7, 0x9f,
0xaf, 0xc6, 0x3e, 0xfe, 0xcb, 0x6a, 0x5c, 0xa2, 0x12, 0xe8, 0x3c, 0xf1, 0x95, 0x8a, 0xd6, 0x97,
0x35, 0x95, 0x0e, 0x39, 0x47, 0x1c, 0xa1, 0xa2, 0xf5, 0xb7, 0x55, 0xb4, 0x03, 0xe5, 0xb6, 0xde,
0xb7, 0x70, 0xdf, 0x1a, 0x58, 0x32, 0x0b, 0x84, 0x3c, 0x98, 0x04, 0xdc, 0x21, 0x0b, 0xaf, 0x35,
0x87, 0x73, 0x9f, 0x32, 0x4a, 0xa5, 0x76, 0x90, 0x40, 0xdc, 0xea, 0xa9, 0xd2, 0xd5, 0x54, 0xc5,
0xd6, 0x4d, 0xab, 0x92, 0xbc, 0x94, 0x08, 0xf5, 0x87, 0x47, 0x0e, 0xcb, 0xa1, 0xa1, 0x2a, 0x36,
0xde, 0x4c, 0x92, 0xe1, 0x4a, 0x3e, 0x49, 0xf4, 0x14, 0x94, 0x14, 0xc3, 0x90, 0x2d, 0x5b, 0xb1,
0xb1, 0xdc, 0x3a, 0xb3, 0xb1, 0x45, 0x03, 0xcd, 0x82, 0x54, 0x50, 0x0c, 0xe3, 0x80, 0x50, 0x37,
0x09, 0x11, 0x5d, 0x81, 0x22, 0x89, 0x49, 0x9a, 0xd2, 0x95, 0x3b, 0x58, 0x3b, 0xe9, 0xd8, 0x34,
0xa4, 0x24, 0xa4, 0x02, 0xa7, 0x36, 0x28, 0x51, 0x54, 0xdd, 0x1d, 0xa7, 0xf1, 0x08, 0x21, 0x48,
0xaa, 0x8a, 0xad, 0xd0, 0x95, 0x5c, 0x90, 0xe8, 0x6f, 0x42, 0x33, 0x14, 0xbb, 0xc3, 0xd7, 0x87,
0xfe, 0x46, 0xe7, 0x20, 0xcd, 0xd5, 0x26, 0xa8, 0x5a, 0xfe, 0x84, 0x96, 0x21, 0x65, 0x98, 0xfa,
0x29, 0xa6, 0x5b, 0x97, 0x95, 0xd8, 0x83, 0xf8, 0x81, 0x00, 0x8b, 0x63, 0x91, 0x8b, 0xe8, 0xed,
0x28, 0x56, 0xc7, 0xf9, 0x16, 0xf9, 0x8d, 0x5e, 0x20, 0x7a, 0x15, 0x15, 0x9b, 0x3c, 0xda, 0x57,
0xc6, 0x97, 0xba, 0x41, 0xdf, 0xf3, 0xa5, 0xe1, 0xdc, 0xe8, 0x1e, 0x94, 0xbb, 0x8a, 0x65, 0xcb,
0xcc, 0xfb, 0xcb, 0xbe, 0xc8, 0xff, 0xc4, 0xd8, 0x22, 0xb3, 0x58, 0x41, 0x0c, 0x9a, 0x2b, 0x29,
0x12, 0x51, 0x8f, 0x8a, 0x0e, 0x61, 0xb9, 0x75, 0xf6, 0xbe, 0xd2, 0xb7, 0xb5, 0x3e, 0x96, 0xc7,
0x76, 0x6d, 0x3c, 0x95, 0xb8, 0xaf, 0x59, 0x2d, 0xdc, 0x51, 0x4e, 0x35, 0xdd, 0x19, 0xd6, 0x92,
0x2b, 0xef, 0xee, 0xa8, 0x25, 0x4a, 0x50, 0x0c, 0x86, 0x5d, 0x54, 0x04, 0xc1, 0x1e, 0xf2, 0xf9,
0x0b, 0xf6, 0x10, 0x3d, 0x0f, 0x49, 0x32, 0x47, 0x3a, 0xf7, 0x62, 0xc8, 0x87, 0xb8, 0x5c, 0xf3,
0xcc, 0xc0, 0x12, 0xe5, 0x14, 0x45, 0xf7, 0x34, 0xb8, 0xa1, 0x78, 0x54, 0xab, 0x78, 0x0d, 0x4a,
0x23, 0x71, 0xd6, 0xb7, 0x7d, 0x71, 0xff, 0xf6, 0x89, 0x25, 0x28, 0x04, 0x02, 0xaa, 0x78, 0x0e,
0x96, 0xc3, 0xe2, 0xa3, 0xd8, 0x71, 0xe9, 0x81, 0x38, 0x87, 0x6e, 0x41, 0xd6, 0x0d, 0x90, 0xec,
0x34, 0x9e, 0x1f, 0x9b, 0x85, 0xc3, 0x2c, 0xb9, 0xac, 0xe4, 0x18, 0x12, 0xab, 0xa6, 0xe6, 0x20,
0xd0, 0x81, 0x67, 0x14, 0xc3, 0x68, 0x28, 0x56, 0x47, 0x7c, 0x1b, 0x2a, 0x51, 0xc1, 0x6f, 0x64,
0x1a, 0x49, 0xd7, 0x0a, 0xcf, 0x41, 0xfa, 0x58, 0x37, 0x7b, 0x8a, 0x4d, 0x95, 0x15, 0x24, 0xfe,
0x44, 0xac, 0x93, 0x05, 0xc2, 0x04, 0x25, 0xb3, 0x07, 0x51, 0x86, 0xf3, 0x91, 0x01, 0x90, 0x88,
0x68, 0x7d, 0x15, 0xb3, 0xf5, 0x2c, 0x48, 0xec, 0xc1, 0x53, 0xc4, 0x06, 0xcb, 0x1e, 0xc8, 0x67,
0x2d, 0x3a, 0x57, 0xaa, 0x3f, 0x27, 0xf1, 0x27, 0xf1, 0xd7, 0x09, 0x38, 0x17, 0x1e, 0x06, 0xd1,
0x25, 0x58, 0xe8, 0x29, 0x43, 0xd9, 0x1e, 0xf2, 0xb3, 0xcc, 0xb6, 0x03, 0x7a, 0xca, 0xb0, 0x39,
0x64, 0x07, 0xb9, 0x0c, 0x09, 0x7b, 0x68, 0x55, 0x84, 0x4b, 0x89, 0xab, 0x0b, 0x12, 0xf9, 0x89,
0x0e, 0x61, 0xb1, 0xab, 0xb7, 0x95, 0xae, 0xec, 0xb3, 0x78, 0x6e, 0xec, 0x97, 0xc7, 0x16, 0x9b,
0x05, 0x34, 0xac, 0x8e, 0x19, 0x7d, 0x89, 0xea, 0xd8, 0x71, 0x2d, 0xff, 0x1b, 0xb2, 0x7a, 0xdf,
0x1e, 0xa5, 0x02, 0x9e, 0xc2, 0xf1, 0xd9, 0xe9, 0xb9, 0x7d, 0xf6, 0xf3, 0xb0, 0xdc, 0xc7, 0x43,
0xdb, 0x37, 0x46, 0x66, 0x38, 0x19, 0xba, 0x17, 0x88, 0xbc, 0xf3, 0xbe, 0x4f, 0x6c, 0x08, 0x5d,
0xa3, 0x99, 0x85, 0xa1, 0x5b, 0xd8, 0x94, 0x15, 0x55, 0x35, 0xb1, 0x65, 0xd1, 0xcc, 0x76, 0x81,
0xa6, 0x0b, 0x94, 0xbe, 0xc1, 0xc8, 0xe2, 0xcf, 0xfc, 0x7b, 0x15, 0xcc, 0x24, 0xf8, 0x4e, 0xc4,
0xbd, 0x9d, 0x38, 0x80, 0x65, 0x2e, 0xaf, 0x06, 0x36, 0x43, 0x98, 0xd5, 0xf3, 0x20, 0x47, 0x7c,
0x86, 0x7d, 0x48, 0x3c, 0xda, 0x3e, 0x38, 0xde, 0x36, 0xe9, 0xf3, 0xb6, 0xff, 0x66, 0x7b, 0xf3,
0xaa, 0x1b, 0x45, 0xbc, 0x34, 0x2d, 0x34, 0x8a, 0x78, 0xf3, 0x12, 0x02, 0xee, 0xed, 0xe7, 0x71,
0xa8, 0x46, 0xe7, 0x65, 0xa1, 0xaa, 0x9e, 0x81, 0x45, 0x77, 0x2e, 0xee, 0xf8, 0xd8, 0xa9, 0x2f,
0xbb, 0x2f, 0xf8, 0x00, 0x23, 0xa3, 0xe2, 0x15, 0x28, 0x8e, 0x64, 0x8d, 0x6c, 0x17, 0x0a, 0xa7,
0xfe, 0xef, 0x8b, 0x3f, 0x4e, 0xb8, 0x5e, 0x35, 0x90, 0xda, 0x85, 0x58, 0xde, 0x1b, 0xb0, 0xa4,
0xe2, 0xb6, 0xa6, 0x7e, 0x55, 0xc3, 0x5b, 0xe4, 0xd2, 0xdf, 0xd9, 0xdd, 0x0c, 0x76, 0xf7, 0xc7,
0x3c, 0x64, 0x25, 0x6c, 0x19, 0x24, 0xa5, 0x43, 0x9b, 0x90, 0xc3, 0xc3, 0x36, 0x36, 0x6c, 0x27,
0x0b, 0x0e, 0xaf, 0x26, 0x18, 0x77, 0xdd, 0xe1, 0x24, 0xb5, 0xb1, 0x2b, 0x86, 0x6e, 0x72, 0x18,
0x24, 0x1a, 0xd1, 0xe0, 0xe2, 0x7e, 0x1c, 0xe4, 0x05, 0x07, 0x07, 0x49, 0x44, 0x96, 0xc2, 0x4c,
0x6a, 0x04, 0x08, 0xb9, 0xc9, 0x81, 0x90, 0xe4, 0x94, 0x8f, 0x05, 0x90, 0x90, 0x5a, 0x00, 0x09,
0x49, 0x4d, 0x99, 0x66, 0x04, 0x14, 0xf2, 0x82, 0x03, 0x85, 0xa4, 0xa7, 0x8c, 0x78, 0x04, 0x0b,
0x79, 0x3d, 0x88, 0x85, 0x64, 0x22, 0x42, 0x9b, 0x23, 0x3d, 0x11, 0x0c, 0x79, 0xc5, 0x07, 0x86,
0x64, 0x23, 0x51, 0x08, 0xa6, 0x28, 0x04, 0x0d, 0x79, 0x2d, 0x80, 0x86, 0xe4, 0xa6, 0xac, 0xc3,
0x04, 0x38, 0x64, 0xcb, 0x0f, 0x87, 0x40, 0x24, 0xaa, 0xc2, 0xf7, 0x3d, 0x0a, 0x0f, 0x79, 0xc9,
0xc5, 0x43, 0xf2, 0x91, 0xc0, 0x0e, 0x9f, 0xcb, 0x28, 0x20, 0xb2, 0x37, 0x06, 0x88, 0x30, 0x00,
0xe3, 0xa9, 0x48, 0x15, 0x53, 0x10, 0x91, 0xbd, 0x31, 0x44, 0xa4, 0x30, 0x45, 0xe1, 0x14, 0x48,
0xe4, 0x7b, 0xe1, 0x90, 0x48, 0x34, 0x68, 0xc1, 0x87, 0x39, 0x1b, 0x26, 0x22, 0x47, 0x60, 0x22,
0xa5, 0xc8, 0xfa, 0x9d, 0xa9, 0x9f, 0x19, 0x14, 0x39, 0x0c, 0x01, 0x45, 0x18, 0x7c, 0x71, 0x35,
0x52, 0xf9, 0x0c, 0xa8, 0xc8, 0x61, 0x08, 0x2a, 0xb2, 0x38, 0x55, 0xed, 0x54, 0x58, 0xe4, 0x6e,
0x10, 0x16, 0x41, 0x53, 0xce, 0x58, 0x24, 0x2e, 0xd2, 0x8a, 0xc2, 0x45, 0x18, 0x76, 0xf1, 0x6c,
0xa4, 0xc6, 0x39, 0x80, 0x91, 0xbd, 0x31, 0x60, 0x64, 0x79, 0x8a, 0xa5, 0xcd, 0x8a, 0x8c, 0x5c,
0x23, 0x19, 0xc5, 0x88, 0xab, 0x26, 0xc9, 0x3d, 0x36, 0x4d, 0xdd, 0xe4, 0x18, 0x07, 0x7b, 0x10,
0xaf, 0x92, 0x4a, 0xd9, 0x73, 0xcb, 0x13, 0x50, 0x14, 0x5a, 0x44, 0xf9, 0x5c, 0xb1, 0xf8, 0x9b,
0xb8, 0x27, 0x4b, 0x0b, 0x4c, 0x7f, 0x95, 0x9d, 0xe3, 0x55, 0xb6, 0x0f, 0x5b, 0x11, 0x82, 0xd8,
0xca, 0x2a, 0xe4, 0x49, 0x71, 0x34, 0x02, 0x9b, 0x28, 0x86, 0x0b, 0x9b, 0x5c, 0x87, 0x45, 0x9a,
0x04, 0x30, 0x04, 0x86, 0x47, 0xd6, 0x24, 0x8d, 0xac, 0x25, 0xf2, 0x82, 0xad, 0x02, 0x0b, 0xb1,
0xcf, 0xc1, 0x92, 0x8f, 0xd7, 0x2d, 0xba, 0x18, 0x86, 0x50, 0x76, 0xb9, 0x37, 0x78, 0xf5, 0xf5,
0xbb, 0xb8, 0xb7, 0x42, 0x1e, 0xde, 0x12, 0x06, 0x8d, 0xc4, 0xbf, 0x26, 0x68, 0x44, 0xf8, 0xca,
0xd0, 0x88, 0xbf, 0x88, 0x4c, 0x04, 0x8b, 0xc8, 0xbf, 0xc7, 0xbd, 0x3d, 0x71, 0x81, 0x8e, 0xb6,
0xae, 0x62, 0x5e, 0xd6, 0xd1, 0xdf, 0x24, 0xcd, 0xea, 0xea, 0x27, 0xbc, 0x78, 0x23, 0x3f, 0x09,
0x97, 0x1b, 0x3b, 0x73, 0x3c, 0x34, 0xba, 0x15, 0x21, 0xcb, 0x5d, 0x78, 0x45, 0x58, 0x86, 0xc4,
0x43, 0xcc, 0x22, 0xdd, 0x82, 0x44, 0x7e, 0x12, 0x3e, 0x6a, 0x64, 0x3c, 0x07, 0x61, 0x0f, 0xe8,
0x36, 0xe4, 0x68, 0xbb, 0x46, 0xd6, 0x0d, 0x8b, 0x07, 0xa4, 0x40, 0xba, 0xc6, 0xba, 0x32, 0x6b,
0xfb, 0x84, 0x67, 0xcf, 0xb0, 0xa4, 0xac, 0xc1, 0x7f, 0xf9, 0x92, 0xa6, 0x5c, 0x20, 0x69, 0xba,
0x00, 0x39, 0x32, 0x7a, 0xcb, 0x50, 0xda, 0x98, 0x46, 0x96, 0x9c, 0xe4, 0x11, 0xc4, 0x07, 0x80,
0xc6, 0xe3, 0x24, 0x6a, 0x40, 0x1a, 0x9f, 0xe2, 0xbe, 0xcd, 0x72, 0xca, 0xfc, 0xfa, 0xb9, 0xf1,
0xba, 0x91, 0xbc, 0xde, 0xac, 0x90, 0x45, 0xfe, 0xdb, 0xe7, 0xab, 0x65, 0xc6, 0xfd, 0xac, 0xde,
0xd3, 0x6c, 0xdc, 0x33, 0xec, 0x33, 0x89, 0xcb, 0x8b, 0x7f, 0x16, 0xa0, 0x34, 0x12, 0x3f, 0x43,
0xd7, 0xd6, 0x31, 0x79, 0xc1, 0x07, 0x2c, 0xcd, 0xb6, 0xde, 0x17, 0x01, 0x4e, 0x14, 0x4b, 0x7e,
0x4f, 0xe9, 0xdb, 0x58, 0xe5, 0x8b, 0x9e, 0x3b, 0x51, 0xac, 0x37, 0x29, 0x81, 0xec, 0x3a, 0x79,
0x3d, 0xb0, 0xb0, 0xca, 0x21, 0xae, 0xcc, 0x89, 0x62, 0x1d, 0x5a, 0x58, 0xf5, 0xcd, 0x32, 0xf3,
0x68, 0xb3, 0x0c, 0xae, 0x71, 0x76, 0x64, 0x8d, 0x7d, 0x75, 0x7f, 0xce, 0x5f, 0xf7, 0xa3, 0x2a,
0x64, 0x0d, 0x53, 0xd3, 0x4d, 0xcd, 0x3e, 0xa3, 0x1b, 0x93, 0x90, 0xdc, 0x67, 0x74, 0x19, 0x0a,
0x3d, 0xdc, 0x33, 0x74, 0xbd, 0x2b, 0x33, 0x67, 0x93, 0xa7, 0xa2, 0x0b, 0x9c, 0x58, 0xa7, 0x3e,
0xe7, 0x43, 0xc1, 0x3b, 0x7d, 0x1e, 0xbe, 0xf3, 0xf5, 0x2e, 0xef, 0x4a, 0xc8, 0xf2, 0xfa, 0x28,
0x64, 0x12, 0x23, 0xeb, 0xeb, 0x3e, 0x7f, 0x5b, 0x0b, 0x2c, 0xfe, 0x90, 0x82, 0xbe, 0xc1, 0xdc,
0x08, 0x1d, 0xf8, 0x2b, 0xb3, 0x01, 0x75, 0x0a, 0x8e, 0x39, 0xcf, 0xea, 0x3d, 0xbc, 0x0a, 0x8e,
0x91, 0x2d, 0xf4, 0x16, 0x3c, 0x3e, 0xe2, 0xd9, 0x5c, 0xd5, 0xc2, 0xac, 0x0e, 0xee, 0xb1, 0xa0,
0x83, 0x73, 0x54, 0x7b, 0x8b, 0x95, 0x78, 0xc4, 0x33, 0xb7, 0x0d, 0xc5, 0x60, 0x9a, 0x17, 0xba,
0xfd, 0x97, 0xa1, 0x60, 0x62, 0x5b, 0xd1, 0xfa, 0x72, 0xa0, 0x26, 0x5d, 0x60, 0x44, 0x8e, 0xff,
0xee, 0xc3, 0x63, 0xa1, 0xe9, 0x1e, 0x7a, 0x11, 0x72, 0x5e, 0xa6, 0xc8, 0x56, 0x75, 0x02, 0x92,
0xe7, 0xf1, 0x8a, 0xbf, 0x8d, 0x7b, 0x2a, 0x83, 0xd8, 0x60, 0x1d, 0xd2, 0x26, 0xb6, 0x06, 0x5d,
0x86, 0xd6, 0x15, 0xd7, 0x9f, 0x9b, 0x2d, 0x51, 0x24, 0xd4, 0x41, 0xd7, 0x96, 0xb8, 0xb0, 0xf8,
0x00, 0xd2, 0x8c, 0x82, 0xf2, 0x90, 0x39, 0xdc, 0xbd, 0xb7, 0xbb, 0xf7, 0xe6, 0x6e, 0x39, 0x86,
0x00, 0xd2, 0x1b, 0xb5, 0x5a, 0x7d, 0xbf, 0x59, 0x8e, 0xa3, 0x1c, 0xa4, 0x36, 0x36, 0xf7, 0xa4,
0x66, 0x59, 0x20, 0x64, 0xa9, 0xfe, 0x7a, 0xbd, 0xd6, 0x2c, 0x27, 0xd0, 0x22, 0x14, 0xd8, 0x6f,
0xf9, 0xee, 0x9e, 0x74, 0x7f, 0xa3, 0x59, 0x4e, 0xfa, 0x48, 0x07, 0xf5, 0xdd, 0xad, 0xba, 0x54,
0x4e, 0x89, 0xff, 0x05, 0xe7, 0x23, 0x53, 0x4b, 0x0f, 0xf8, 0x8b, 0xfb, 0x80, 0x3f, 0xf1, 0xa7,
0x02, 0x54, 0xa3, 0xf3, 0x45, 0xf4, 0xfa, 0xc8, 0xc4, 0xd7, 0xe7, 0x48, 0x36, 0x47, 0x66, 0x8f,
0xae, 0x40, 0xd1, 0xc4, 0xc7, 0xd8, 0x6e, 0x77, 0x58, 0xfe, 0xca, 0x02, 0x66, 0x41, 0x2a, 0x70,
0x2a, 0x15, 0xb2, 0x18, 0xdb, 0x3b, 0xb8, 0x6d, 0xcb, 0xcc, 0x17, 0x31, 0xa3, 0xcb, 0x11, 0x36,
0x42, 0x3d, 0x60, 0x44, 0xf1, 0xed, 0xb9, 0xd6, 0x32, 0x07, 0x29, 0xa9, 0xde, 0x94, 0xde, 0x2a,
0x27, 0x10, 0x82, 0x22, 0xfd, 0x29, 0x1f, 0xec, 0x6e, 0xec, 0x1f, 0x34, 0xf6, 0xc8, 0x5a, 0x2e,
0x41, 0xc9, 0x59, 0x4b, 0x87, 0x98, 0x12, 0xff, 0x20, 0xc0, 0xe3, 0x11, 0xd9, 0x2e, 0xba, 0x0d,
0x60, 0x0f, 0x65, 0x13, 0xb7, 0x75, 0x53, 0x8d, 0x36, 0xb2, 0xe6, 0x50, 0xa2, 0x1c, 0x52, 0xce,
0xe6, 0xbf, 0xac, 0x09, 0x78, 0x31, 0x7a, 0x99, 0x2b, 0x25, 0xb3, 0x72, 0x8e, 0xda, 0xc5, 0x10,
0x58, 0x14, 0xb7, 0x89, 0x62, 0xba, 0xb6, 0x54, 0x31, 0xe5, 0x47, 0xf7, 0xc3, 0x9c, 0xca, 0x8c,
0xdd, 0x9a, 0xf9, 0xdc, 0x49, 0xea, 0xd1, 0xdc, 0x89, 0xf8, 0x8b, 0x84, 0x7f, 0x61, 0x83, 0xc9,
0xfd, 0x1e, 0xa4, 0x2d, 0x5b, 0xb1, 0x07, 0x16, 0x37, 0xb8, 0x17, 0x67, 0xad, 0x14, 0xd6, 0x9c,
0x1f, 0x07, 0x54, 0x5c, 0xe2, 0x6a, 0xbe, 0x5b, 0x6f, 0x4b, 0xbc, 0x05, 0xc5, 0xe0, 0xe2, 0x44,
0x1f, 0x19, 0xcf, 0xe7, 0x08, 0xe2, 0x1d, 0x2f, 0xff, 0xf2, 0x81, 0x96, 0xe3, 0x80, 0x60, 0x3c,
0x0c, 0x10, 0xfc, 0x65, 0x1c, 0x9e, 0x98, 0x50, 0x2f, 0xa1, 0x37, 0x46, 0xf6, 0xf9, 0xa5, 0x79,
0xaa, 0xad, 0x35, 0x46, 0x0b, 0xee, 0xb4, 0x78, 0x13, 0x16, 0xfc, 0xf4, 0xd9, 0x26, 0xf9, 0xa3,
0x84, 0xe7, 0xf3, 0x83, 0xc8, 0xe5, 0xd7, 0x96, 0x68, 0x8e, 0xd8, 0x99, 0x30, 0xa7, 0x9d, 0x85,
0x26, 0x0b, 0x89, 0x6f, 0x2e, 0x59, 0x48, 0x3e, 0x62, 0xb2, 0xe0, 0x3f, 0x70, 0xa9, 0xe0, 0x81,
0x1b, 0x8b, 0xeb, 0xe9, 0x90, 0xb8, 0xfe, 0x16, 0x80, 0xaf, 0xa1, 0xb9, 0x0c, 0x29, 0x53, 0x1f,
0xf4, 0x55, 0x6a, 0x26, 0x29, 0x89, 0x3d, 0xa0, 0x5b, 0x90, 0x22, 0xe6, 0xe6, 0x2c, 0xe6, 0xb8,
0xe7, 0x25, 0xe6, 0xe2, 0xc3, 0x8c, 0x19, 0xb7, 0xa8, 0x01, 0x1a, 0x6f, 0x2a, 0x45, 0x7c, 0xe2,
0x95, 0xe0, 0x27, 0x9e, 0x8c, 0x6c, 0x4f, 0x85, 0x7f, 0xea, 0x7d, 0x48, 0x51, 0xf3, 0x20, 0xf9,
0x0d, 0x6d, 0x8c, 0xf2, 0x82, 0x99, 0xfc, 0x46, 0xdf, 0x07, 0x50, 0x6c, 0xdb, 0xd4, 0x5a, 0x03,
0xef, 0x03, 0xab, 0xe1, 0xe6, 0xb5, 0xe1, 0xf0, 0x6d, 0x5e, 0xe0, 0x76, 0xb6, 0xec, 0x89, 0xfa,
0x6c, 0xcd, 0xa7, 0x50, 0xdc, 0x85, 0x62, 0x50, 0xd6, 0x29, 0xf1, 0xd8, 0x18, 0x82, 0x25, 0x1e,
0xab, 0xd8, 0x79, 0x89, 0xe7, 0x16, 0x88, 0x09, 0xd6, 0x03, 0xa7, 0x0f, 0xe2, 0x3f, 0xe2, 0xb0,
0xe0, 0xb7, 0xce, 0xff, 0xb4, 0x2a, 0x49, 0xfc, 0x30, 0x0e, 0x59, 0x77, 0xf2, 0x11, 0x0d, 0x68,
0x6f, 0xed, 0x04, 0x7f, 0xbb, 0x95, 0x75, 0xb4, 0x13, 0x6e, 0x9f, 0xfc, 0x8e, 0x9b, 0x50, 0x45,
0x81, 0xda, 0xfe, 0x95, 0x76, 0xae, 0x0a, 0xf0, 0xfc, 0xf1, 0x27, 0x7c, 0x1c, 0x24, 0x93, 0x40,
0xff, 0x03, 0x69, 0xa5, 0xed, 0x42, 0xf9, 0xc5, 0x10, 0x6c, 0xd7, 0x61, 0x5d, 0x6b, 0x0e, 0x37,
0x28, 0xa7, 0xc4, 0x25, 0xf8, 0xa8, 0x04, 0xb7, 0xcf, 0xfe, 0x2a, 0xd1, 0xcb, 0x78, 0x82, 0x6e,
0xb3, 0x08, 0x70, 0xb8, 0x7b, 0x7f, 0x6f, 0x6b, 0xfb, 0xee, 0x76, 0x7d, 0x8b, 0xa7, 0x54, 0x5b,
0x5b, 0xf5, 0xad, 0xb2, 0x40, 0xf8, 0xa4, 0xfa, 0xfd, 0xbd, 0xa3, 0xfa, 0x56, 0x39, 0x21, 0xde,
0x81, 0x9c, 0xeb, 0x7a, 0x50, 0x05, 0x32, 0x4e, 0x5b, 0x22, 0xce, 0x1d, 0x00, 0xef, 0x32, 0x2d,
0x43, 0xca, 0xd0, 0xdf, 0xe3, 0x5d, 0xe6, 0x84, 0xc4, 0x1e, 0x44, 0x15, 0x4a, 0x23, 0x7e, 0x0b,
0xdd, 0x81, 0x8c, 0x31, 0x68, 0xc9, 0x8e, 0xd1, 0x8e, 0x34, 0x71, 0x1c, 0xa4, 0x61, 0xd0, 0xea,
0x6a, 0xed, 0x7b, 0xf8, 0xcc, 0x59, 0x26, 0x63, 0xd0, 0xba, 0xc7, 0x6c, 0x9b, 0x7d, 0x45, 0xf0,
0x7f, 0xe5, 0x14, 0xb2, 0xce, 0x51, 0x45, 0xff, 0x0b, 0x39, 0xd7, 0x25, 0xba, 0x57, 0x6f, 0x22,
0x7d, 0x29, 0x57, 0xef, 0x89, 0xa0, 0xeb, 0xb0, 0x68, 0x69, 0x27, 0x7d, 0xa7, 0x85, 0xc5, 0x90,
0x3d, 0x81, 0x9e, 0x99, 0x12, 0x7b, 0xb1, 0xe3, 0xc0, 0x51, 0x24, 0x12, 0x96, 0x47, 0x7d, 0xc5,
0xb7, 0x39, 0x80, 0x90, 0x88, 0x9d, 0x08, 0x8b, 0xd8, 0x1f, 0x08, 0x90, 0xf7, 0x35, 0xc6, 0xd0,
0x7f, 0xfb, 0x1c, 0x57, 0x31, 0x24, 0xd4, 0xf8, 0x78, 0xbd, 0x5b, 0x1d, 0xc1, 0x89, 0x09, 0xf3,
0x4f, 0x2c, 0xaa, 0x0f, 0xe9, 0xf4, 0xd7, 0x92, 0x73, 0xf7, 0xd7, 0x9e, 0x05, 0x64, 0xeb, 0xb6,
0xd2, 0x95, 0x4f, 0x75, 0x5b, 0xeb, 0x9f, 0xc8, 0xcc, 0x34, 0x98, 0x9b, 0x29, 0xd3, 0x37, 0x47,
0xf4, 0xc5, 0x3e, 0xb5, 0x92, 0x1f, 0xc4, 0x21, 0xeb, 0x96, 0x7d, 0xf3, 0x5e, 0xd2, 0x38, 0x07,
0x69, 0x5e, 0xd9, 0xb0, 0x5b, 0x1a, 0xfc, 0x29, 0xb4, 0x91, 0x58, 0x85, 0x6c, 0x0f, 0xdb, 0x0a,
0xf5, 0x99, 0x2c, 0x4c, 0xba, 0xcf, 0xd7, 0x5f, 0x82, 0xbc, 0xef, 0xbe, 0x0c, 0x71, 0xa3, 0xbb,
0xf5, 0x37, 0xcb, 0xb1, 0x6a, 0xe6, 0xa3, 0x4f, 0x2e, 0x25, 0x76, 0xf1, 0x7b, 0xe4, 0x84, 0x49,
0xf5, 0x5a, 0xa3, 0x5e, 0xbb, 0x57, 0x8e, 0x57, 0xf3, 0x1f, 0x7d, 0x72, 0x29, 0x23, 0x61, 0xda,
0xf7, 0xb9, 0x7e, 0x0f, 0x4a, 0x23, 0x1b, 0x13, 0x3c, 0xd0, 0x08, 0x8a, 0x5b, 0x87, 0xfb, 0x3b,
0xdb, 0xb5, 0x8d, 0x66, 0x5d, 0x3e, 0xda, 0x6b, 0xd6, 0xcb, 0x71, 0xf4, 0x38, 0x2c, 0xed, 0x6c,
0xbf, 0xd6, 0x68, 0xca, 0xb5, 0x9d, 0xed, 0xfa, 0x6e, 0x53, 0xde, 0x68, 0x36, 0x37, 0x6a, 0xf7,
0xca, 0xc2, 0xfa, 0xaf, 0xf2, 0x50, 0xda, 0xd8, 0xac, 0x6d, 0x93, 0xda, 0x4e, 0x6b, 0x2b, 0xd4,
0x3d, 0xd4, 0x20, 0x49, 0x41, 0xe4, 0x89, 0x37, 0xa0, 0xab, 0x93, 0x1b, 0x83, 0xe8, 0x2e, 0xa4,
0x28, 0xbe, 0x8c, 0x26, 0x5f, 0x89, 0xae, 0x4e, 0xe9, 0x14, 0x92, 0xc1, 0xd0, 0xe3, 0x34, 0xf1,
0x8e, 0x74, 0x75, 0x72, 0xe3, 0x10, 0xed, 0x40, 0xc6, 0x81, 0xff, 0xa6, 0xdd, 0x36, 0xae, 0x4e,
0xed, 0xc0, 0x91, 0xa9, 0x31, 0x98, 0x76, 0xf2, 0xf5, 0xe9, 0xea, 0x94, 0x96, 0x22, 0xda, 0x86,
0x34, 0x47, 0x48, 0xa6, 0xdc, 0x1c, 0xae, 0x4e, 0xeb, 0xa4, 0x21, 0x09, 0x72, 0x1e, 0x00, 0x3e,
0xfd, 0x52, 0x78, 0x75, 0x86, 0x6e, 0x29, 0x7a, 0x00, 0x85, 0x20, 0xea, 0x32, 0xdb, 0xed, 0xe4,
0xea, 0x8c, 0x3d, 0x3b, 0xa2, 0x3f, 0x08, 0xc1, 0xcc, 0x76, 0x5b, 0xb9, 0x3a, 0x63, 0x0b, 0x0f,
0xbd, 0x03, 0x8b, 0xe3, 0x10, 0xc9, 0xec, 0x97, 0x97, 0xab, 0x73, 0x34, 0xf5, 0x50, 0x0f, 0x50,
0x08, 0xb4, 0x32, 0xc7, 0x5d, 0xe6, 0xea, 0x3c, 0x3d, 0x3e, 0xa4, 0x42, 0x69, 0x14, 0xae, 0x98,
0xf5, 0x6e, 0x73, 0x75, 0xe6, 0x7e, 0x1f, 0xfb, 0x4a, 0xb0, 0x76, 0x9f, 0xf5, 0xae, 0x73, 0x75,
0xe6, 0xf6, 0x1f, 0x3a, 0x04, 0xf0, 0xd5, 0x9e, 0x33, 0xdc, 0x7d, 0xae, 0xce, 0xd2, 0x08, 0x44,
0x06, 0x2c, 0x85, 0x15, 0xa5, 0xf3, 0x5c, 0x85, 0xae, 0xce, 0xd5, 0x1f, 0x24, 0xf6, 0x1c, 0x2c,
0x2f, 0x67, 0xbb, 0x1a, 0x5d, 0x9d, 0xb1, 0x51, 0xb8, 0x59, 0xff, 0xf4, 0x8b, 0x95, 0xf8, 0x67,
0x5f, 0xac, 0xc4, 0xff, 0xfa, 0xc5, 0x4a, 0xfc, 0xe3, 0x2f, 0x57, 0x62, 0x9f, 0x7d, 0xb9, 0x12,
0xfb, 0xd3, 0x97, 0x2b, 0xb1, 0xff, 0x7b, 0xe6, 0x44, 0xb3, 0x3b, 0x83, 0xd6, 0x5a, 0x5b, 0xef,
0xdd, 0xf0, 0xff, 0x4b, 0x26, 0xec, 0x9f, 0x3b, 0xad, 0x34, 0x0d, 0xa8, 0x37, 0xff, 0x15, 0x00,
0x00, 0xff, 0xff, 0xa5, 0x05, 0x49, 0xaf, 0xd9, 0x33, 0x00, 0x00,
0x73, 0xac, 0xa8, 0x02, 0x99, 0x1e, 0xb6, 0x2c, 0xe5, 0x04, 0x53, 0x3f, 0x9c, 0x93, 0x9c, 0xa1,
0x58, 0x84, 0x05, 0xbf, 0x33, 0x15, 0x3f, 0x8e, 0xbb, 0x92, 0xc4, 0x4f, 0x12, 0xc9, 0x53, 0x6c,
0xd2, 0x65, 0x73, 0x49, 0x3e, 0x44, 0x97, 0xa1, 0x40, 0xa7, 0x2c, 0x3b, 0xcf, 0x89, 0xb3, 0x4e,
0x4a, 0x0b, 0x94, 0x78, 0xc4, 0x99, 0x56, 0x21, 0x6f, 0xac, 0x1b, 0x2e, 0x4b, 0x82, 0xb2, 0x80,
0xb1, 0x6e, 0x38, 0x0c, 0x4f, 0xc2, 0x02, 0x59, 0x9f, 0xcb, 0x91, 0xa4, 0x2f, 0xc9, 0x13, 0x1a,
0x67, 0x11, 0x7f, 0x27, 0x40, 0x79, 0xd4, 0x01, 0xa3, 0xdb, 0x90, 0x24, 0xb1, 0x88, 0x87, 0x95,
0xea, 0x1a, 0x0b, 0x54, 0x6b, 0x4e, 0xa0, 0x5a, 0x6b, 0x3a, 0x81, 0x6a, 0x33, 0xfb, 0xe9, 0xe7,
0xab, 0xb1, 0x8f, 0xff, 0xb2, 0x1a, 0x97, 0xa8, 0x04, 0x3a, 0x4f, 0x7c, 0xa5, 0xa2, 0xf5, 0x65,
0x4d, 0xa5, 0x53, 0xce, 0x11, 0x47, 0xa8, 0x68, 0xfd, 0x6d, 0x15, 0xed, 0x40, 0xb9, 0xad, 0xf7,
0x2d, 0xdc, 0xb7, 0x06, 0x96, 0xcc, 0x02, 0x21, 0x0f, 0x26, 0x01, 0x77, 0xc8, 0xc2, 0x6b, 0xcd,
0xe1, 0xdc, 0xa7, 0x8c, 0x52, 0xa9, 0x1d, 0x24, 0x10, 0xb7, 0x7a, 0xaa, 0x74, 0x35, 0x55, 0xb1,
0x75, 0xd3, 0xaa, 0x24, 0x2f, 0x25, 0x42, 0xfd, 0xe1, 0x91, 0xc3, 0x72, 0x68, 0xa8, 0x8a, 0x8d,
0x37, 0x93, 0x64, 0xba, 0x92, 0x4f, 0x12, 0x3d, 0x05, 0x25, 0xc5, 0x30, 0x64, 0xcb, 0x56, 0x6c,
0x2c, 0xb7, 0xce, 0x6c, 0x6c, 0xd1, 0x40, 0xb3, 0x20, 0x15, 0x14, 0xc3, 0x38, 0x20, 0xd4, 0x4d,
0x42, 0x44, 0x57, 0xa0, 0x48, 0x62, 0x92, 0xa6, 0x74, 0xe5, 0x0e, 0xd6, 0x4e, 0x3a, 0x36, 0x0d,
0x29, 0x09, 0xa9, 0xc0, 0xa9, 0x0d, 0x4a, 0x14, 0x55, 0xf7, 0xc4, 0x69, 0x3c, 0x42, 0x08, 0x92,
0xaa, 0x62, 0x2b, 0x74, 0x27, 0x17, 0x24, 0xfa, 0x9b, 0xd0, 0x0c, 0xc5, 0xee, 0xf0, 0xfd, 0xa1,
0xbf, 0xd1, 0x39, 0x48, 0x73, 0xb5, 0x09, 0xaa, 0x96, 0x8f, 0xd0, 0x32, 0xa4, 0x0c, 0x53, 0x3f,
0xc5, 0xf4, 0xe8, 0xb2, 0x12, 0x1b, 0x88, 0x1f, 0x08, 0xb0, 0x38, 0x16, 0xb9, 0x88, 0xde, 0x8e,
0x62, 0x75, 0x9c, 0x77, 0x91, 0xdf, 0xe8, 0x05, 0xa2, 0x57, 0x51, 0xb1, 0xc9, 0xa3, 0x7d, 0x65,
0x7c, 0xab, 0x1b, 0xf4, 0x39, 0xdf, 0x1a, 0xce, 0x8d, 0xee, 0x41, 0xb9, 0xab, 0x58, 0xb6, 0xcc,
0xbc, 0xbf, 0xec, 0x8b, 0xfc, 0x4f, 0x8c, 0x6d, 0x32, 0x8b, 0x15, 0xc4, 0xa0, 0xb9, 0x92, 0x22,
0x11, 0xf5, 0xa8, 0xe8, 0x10, 0x96, 0x5b, 0x67, 0xef, 0x2b, 0x7d, 0x5b, 0xeb, 0x63, 0x79, 0xec,
0xd4, 0xc6, 0x53, 0x89, 0xfb, 0x9a, 0xd5, 0xc2, 0x1d, 0xe5, 0x54, 0xd3, 0x9d, 0x69, 0x2d, 0xb9,
0xf2, 0xee, 0x89, 0x5a, 0xa2, 0x04, 0xc5, 0x60, 0xd8, 0x45, 0x45, 0x10, 0xec, 0x21, 0x5f, 0xbf,
0x60, 0x0f, 0xd1, 0xf3, 0x90, 0x24, 0x6b, 0xa4, 0x6b, 0x2f, 0x86, 0xbc, 0x88, 0xcb, 0x35, 0xcf,
0x0c, 0x2c, 0x51, 0x4e, 0x51, 0x74, 0xbf, 0x06, 0x37, 0x14, 0x8f, 0x6a, 0x15, 0xaf, 0x41, 0x69,
0x24, 0xce, 0xfa, 0x8e, 0x2f, 0xee, 0x3f, 0x3e, 0xb1, 0x04, 0x85, 0x40, 0x40, 0x15, 0xcf, 0xc1,
0x72, 0x58, 0x7c, 0x14, 0x3b, 0x2e, 0x3d, 0x10, 0xe7, 0xd0, 0x2d, 0xc8, 0xba, 0x01, 0x92, 0x7d,
0x8d, 0xe7, 0xc7, 0x56, 0xe1, 0x30, 0x4b, 0x2e, 0x2b, 0xf9, 0x0c, 0x89, 0x55, 0x53, 0x73, 0x10,
0xe8, 0xc4, 0x33, 0x8a, 0x61, 0x34, 0x14, 0xab, 0x23, 0xbe, 0x0d, 0x95, 0xa8, 0xe0, 0x37, 0xb2,
0x8c, 0xa4, 0x6b, 0x85, 0xe7, 0x20, 0x7d, 0xac, 0x9b, 0x3d, 0xc5, 0xa6, 0xca, 0x0a, 0x12, 0x1f,
0x11, 0xeb, 0x64, 0x81, 0x30, 0x41, 0xc9, 0x6c, 0x20, 0xca, 0x70, 0x3e, 0x32, 0x00, 0x12, 0x11,
0xad, 0xaf, 0x62, 0xb6, 0x9f, 0x05, 0x89, 0x0d, 0x3c, 0x45, 0x6c, 0xb2, 0x6c, 0x40, 0x5e, 0x6b,
0xd1, 0xb5, 0x52, 0xfd, 0x39, 0x89, 0x8f, 0xc4, 0x5f, 0x25, 0xe0, 0x5c, 0x78, 0x18, 0x44, 0x97,
0x60, 0xa1, 0xa7, 0x0c, 0x65, 0x7b, 0xc8, 0xbf, 0x65, 0x76, 0x1c, 0xd0, 0x53, 0x86, 0xcd, 0x21,
0xfb, 0x90, 0xcb, 0x90, 0xb0, 0x87, 0x56, 0x45, 0xb8, 0x94, 0xb8, 0xba, 0x20, 0x91, 0x9f, 0xe8,
0x10, 0x16, 0xbb, 0x7a, 0x5b, 0xe9, 0xca, 0x3e, 0x8b, 0xe7, 0xc6, 0x7e, 0x79, 0x6c, 0xb3, 0x59,
0x40, 0xc3, 0xea, 0x98, 0xd1, 0x97, 0xa8, 0x8e, 0x1d, 0xd7, 0xf2, 0xbf, 0x21, 0xab, 0xf7, 0x9d,
0x51, 0x2a, 0xe0, 0x29, 0x1c, 0x9f, 0x9d, 0x9e, 0xdb, 0x67, 0x3f, 0x0f, 0xcb, 0x7d, 0x3c, 0xb4,
0x7d, 0x73, 0x64, 0x86, 0x93, 0xa1, 0x67, 0x81, 0xc8, 0x33, 0xef, 0xfd, 0xc4, 0x86, 0xd0, 0x35,
0x9a, 0x59, 0x18, 0xba, 0x85, 0x4d, 0x59, 0x51, 0x55, 0x13, 0x5b, 0x16, 0xcd, 0x6c, 0x17, 0x68,
0xba, 0x40, 0xe9, 0x1b, 0x8c, 0x2c, 0xfe, 0xd4, 0x7f, 0x56, 0xc1, 0x4c, 0x82, 0x9f, 0x44, 0xdc,
0x3b, 0x89, 0x03, 0x58, 0xe6, 0xf2, 0x6a, 0xe0, 0x30, 0x84, 0x59, 0x3d, 0x0f, 0x72, 0xc4, 0x67,
0x38, 0x87, 0xc4, 0xa3, 0x9d, 0x83, 0xe3, 0x6d, 0x93, 0x3e, 0x6f, 0xfb, 0x6f, 0x76, 0x36, 0xaf,
0xba, 0x51, 0xc4, 0x4b, 0xd3, 0x42, 0xa3, 0x88, 0xb7, 0x2e, 0x21, 0xe0, 0xde, 0x7e, 0x16, 0x87,
0x6a, 0x74, 0x5e, 0x16, 0xaa, 0xea, 0x19, 0x58, 0x74, 0xd7, 0xe2, 0xce, 0x8f, 0x7d, 0xf5, 0x65,
0xf7, 0x01, 0x9f, 0x60, 0x64, 0x54, 0xbc, 0x02, 0xc5, 0x91, 0xac, 0x91, 0x9d, 0x42, 0xe1, 0xd4,
0xff, 0x7e, 0xf1, 0x47, 0x09, 0xd7, 0xab, 0x06, 0x52, 0xbb, 0x10, 0xcb, 0x7b, 0x03, 0x96, 0x54,
0xdc, 0xd6, 0xd4, 0xaf, 0x6a, 0x78, 0x8b, 0x5c, 0xfa, 0x3b, 0xbb, 0x9b, 0xc1, 0xee, 0xfe, 0x98,
0x87, 0xac, 0x84, 0x2d, 0x83, 0xa4, 0x74, 0x68, 0x13, 0x72, 0x78, 0xd8, 0xc6, 0x86, 0xed, 0x64,
0xc1, 0xe1, 0xd5, 0x04, 0xe3, 0xae, 0x3b, 0x9c, 0xa4, 0x36, 0x76, 0xc5, 0xd0, 0x4d, 0x0e, 0x83,
0x44, 0x23, 0x1a, 0x5c, 0xdc, 0x8f, 0x83, 0xbc, 0xe0, 0xe0, 0x20, 0x89, 0xc8, 0x52, 0x98, 0x49,
0x8d, 0x00, 0x21, 0x37, 0x39, 0x10, 0x92, 0x9c, 0xf2, 0xb2, 0x00, 0x12, 0x52, 0x0b, 0x20, 0x21,
0xa9, 0x29, 0xcb, 0x8c, 0x80, 0x42, 0x5e, 0x70, 0xa0, 0x90, 0xf4, 0x94, 0x19, 0x8f, 0x60, 0x21,
0xaf, 0x07, 0xb1, 0x90, 0x4c, 0x44, 0x68, 0x73, 0xa4, 0x27, 0x82, 0x21, 0xaf, 0xf8, 0xc0, 0x90,
0x6c, 0x24, 0x0a, 0xc1, 0x14, 0x85, 0xa0, 0x21, 0xaf, 0x05, 0xd0, 0x90, 0xdc, 0x94, 0x7d, 0x98,
0x00, 0x87, 0x6c, 0xf9, 0xe1, 0x10, 0x88, 0x44, 0x55, 0xf8, 0xb9, 0x47, 0xe1, 0x21, 0x2f, 0xb9,
0x78, 0x48, 0x3e, 0x12, 0xd8, 0xe1, 0x6b, 0x19, 0x05, 0x44, 0xf6, 0xc6, 0x00, 0x11, 0x06, 0x60,
0x3c, 0x15, 0xa9, 0x62, 0x0a, 0x22, 0xb2, 0x37, 0x86, 0x88, 0x14, 0xa6, 0x28, 0x9c, 0x02, 0x89,
0x7c, 0x2f, 0x1c, 0x12, 0x89, 0x06, 0x2d, 0xf8, 0x34, 0x67, 0xc3, 0x44, 0xe4, 0x08, 0x4c, 0xa4,
0x14, 0x59, 0xbf, 0x33, 0xf5, 0x33, 0x83, 0x22, 0x87, 0x21, 0xa0, 0x08, 0x83, 0x2f, 0xae, 0x46,
0x2a, 0x9f, 0x01, 0x15, 0x39, 0x0c, 0x41, 0x45, 0x16, 0xa7, 0xaa, 0x9d, 0x0a, 0x8b, 0xdc, 0x0d,
0xc2, 0x22, 0x68, 0xca, 0x37, 0x16, 0x89, 0x8b, 0xb4, 0xa2, 0x70, 0x11, 0x86, 0x5d, 0x3c, 0x1b,
0xa9, 0x71, 0x0e, 0x60, 0x64, 0x6f, 0x0c, 0x18, 0x59, 0x9e, 0x62, 0x69, 0xb3, 0x22, 0x23, 0xd7,
0x48, 0x46, 0x31, 0xe2, 0xaa, 0x49, 0x72, 0x8f, 0x4d, 0x53, 0x37, 0x39, 0xc6, 0xc1, 0x06, 0xe2,
0x55, 0x52, 0x29, 0x7b, 0x6e, 0x79, 0x02, 0x8a, 0x42, 0x8b, 0x28, 0x9f, 0x2b, 0x16, 0x7f, 0x1d,
0xf7, 0x64, 0x69, 0x81, 0xe9, 0xaf, 0xb2, 0x73, 0xbc, 0xca, 0xf6, 0x61, 0x2b, 0x42, 0x10, 0x5b,
0x59, 0x85, 0x3c, 0x29, 0x8e, 0x46, 0x60, 0x13, 0xc5, 0x70, 0x61, 0x93, 0xeb, 0xb0, 0x48, 0x93,
0x00, 0x86, 0xc0, 0xf0, 0xc8, 0x9a, 0xa4, 0x91, 0xb5, 0x44, 0x1e, 0xb0, 0x5d, 0x60, 0x21, 0xf6,
0x39, 0x58, 0xf2, 0xf1, 0xba, 0x45, 0x17, 0xc3, 0x10, 0xca, 0x2e, 0xf7, 0x06, 0xaf, 0xbe, 0x7e,
0x1b, 0xf7, 0x76, 0xc8, 0xc3, 0x5b, 0xc2, 0xa0, 0x91, 0xf8, 0xd7, 0x04, 0x8d, 0x08, 0x5f, 0x19,
0x1a, 0xf1, 0x17, 0x91, 0x89, 0x60, 0x11, 0xf9, 0xf7, 0xb8, 0x77, 0x26, 0x2e, 0xd0, 0xd1, 0xd6,
0x55, 0xcc, 0xcb, 0x3a, 0xfa, 0x9b, 0xa4, 0x59, 0x5d, 0xfd, 0x84, 0x17, 0x6f, 0xe4, 0x27, 0xe1,
0x72, 0x63, 0x67, 0x8e, 0x87, 0x46, 0xb7, 0x22, 0x64, 0xb9, 0x0b, 0xaf, 0x08, 0xcb, 0x90, 0x78,
0x88, 0x59, 0xa4, 0x5b, 0x90, 0xc8, 0x4f, 0xc2, 0x47, 0x8d, 0x8c, 0xe7, 0x20, 0x6c, 0x80, 0x6e,
0x43, 0x8e, 0xb6, 0x6b, 0x64, 0xdd, 0xb0, 0x78, 0x40, 0x0a, 0xa4, 0x6b, 0xac, 0x2b, 0xb3, 0xb6,
0x4f, 0x78, 0xf6, 0x0c, 0x4b, 0xca, 0x1a, 0xfc, 0x97, 0x2f, 0x69, 0xca, 0x05, 0x92, 0xa6, 0x0b,
0x90, 0x23, 0xb3, 0xb7, 0x0c, 0xa5, 0x8d, 0x69, 0x64, 0xc9, 0x49, 0x1e, 0x41, 0x7c, 0x00, 0x68,
0x3c, 0x4e, 0xa2, 0x06, 0xa4, 0xf1, 0x29, 0xee, 0xdb, 0x2c, 0xa7, 0xcc, 0xaf, 0x9f, 0x1b, 0xaf,
0x1b, 0xc9, 0xe3, 0xcd, 0x0a, 0xd9, 0xe4, 0xbf, 0x7d, 0xbe, 0x5a, 0x66, 0xdc, 0xcf, 0xea, 0x3d,
0xcd, 0xc6, 0x3d, 0xc3, 0x3e, 0x93, 0xb8, 0xbc, 0xf8, 0x67, 0x01, 0x4a, 0x23, 0xf1, 0x33, 0x74,
0x6f, 0x1d, 0x93, 0x17, 0x7c, 0xc0, 0xd2, 0x6c, 0xfb, 0x7d, 0x11, 0xe0, 0x44, 0xb1, 0xe4, 0xf7,
0x94, 0xbe, 0x8d, 0x55, 0xbe, 0xe9, 0xb9, 0x13, 0xc5, 0x7a, 0x93, 0x12, 0xc8, 0xa9, 0x93, 0xc7,
0x03, 0x0b, 0xab, 0x1c, 0xe2, 0xca, 0x9c, 0x28, 0xd6, 0xa1, 0x85, 0x55, 0xdf, 0x2a, 0x33, 0x8f,
0xb6, 0xca, 0xe0, 0x1e, 0x67, 0x47, 0xf6, 0xd8, 0x57, 0xf7, 0xe7, 0xfc, 0x75, 0x3f, 0xaa, 0x42,
0xd6, 0x30, 0x35, 0xdd, 0xd4, 0xec, 0x33, 0x7a, 0x30, 0x09, 0xc9, 0x1d, 0xa3, 0xcb, 0x50, 0xe8,
0xe1, 0x9e, 0xa1, 0xeb, 0x5d, 0x99, 0x39, 0x9b, 0x3c, 0x15, 0x5d, 0xe0, 0xc4, 0x3a, 0xf5, 0x39,
0x1f, 0x0a, 0xde, 0xd7, 0xe7, 0xe1, 0x3b, 0x5f, 0xef, 0xf6, 0xae, 0x84, 0x6c, 0xaf, 0x8f, 0x42,
0x16, 0x31, 0xb2, 0xbf, 0xee, 0xf8, 0xdb, 0xda, 0x60, 0xf1, 0x87, 0x14, 0xf4, 0x0d, 0xe6, 0x46,
0xe8, 0xc0, 0x5f, 0x99, 0x0d, 0xa8, 0x53, 0x70, 0xcc, 0x79, 0x56, 0xef, 0xe1, 0x55, 0x70, 0x8c,
0x6c, 0xa1, 0xb7, 0xe0, 0xf1, 0x11, 0xcf, 0xe6, 0xaa, 0x16, 0x66, 0x75, 0x70, 0x8f, 0x05, 0x1d,
0x9c, 0xa3, 0xda, 0xdb, 0xac, 0xc4, 0x23, 0x7e, 0x73, 0xdb, 0x50, 0x0c, 0xa6, 0x79, 0xa1, 0xc7,
0x7f, 0x19, 0x0a, 0x26, 0xb6, 0x15, 0xad, 0x2f, 0x07, 0x6a, 0xd2, 0x05, 0x46, 0xe4, 0xf8, 0xef,
0x3e, 0x3c, 0x16, 0x9a, 0xee, 0xa1, 0x17, 0x21, 0xe7, 0x65, 0x8a, 0x6c, 0x57, 0x27, 0x20, 0x79,
0x1e, 0xaf, 0xf8, 0x9b, 0xb8, 0xa7, 0x32, 0x88, 0x0d, 0xd6, 0x21, 0x6d, 0x62, 0x6b, 0xd0, 0x65,
0x68, 0x5d, 0x71, 0xfd, 0xb9, 0xd9, 0x12, 0x45, 0x42, 0x1d, 0x74, 0x6d, 0x89, 0x0b, 0x8b, 0x0f,
0x20, 0xcd, 0x28, 0x28, 0x0f, 0x99, 0xc3, 0xdd, 0x7b, 0xbb, 0x7b, 0x6f, 0xee, 0x96, 0x63, 0x08,
0x20, 0xbd, 0x51, 0xab, 0xd5, 0xf7, 0x9b, 0xe5, 0x38, 0xca, 0x41, 0x6a, 0x63, 0x73, 0x4f, 0x6a,
0x96, 0x05, 0x42, 0x96, 0xea, 0xaf, 0xd7, 0x6b, 0xcd, 0x72, 0x02, 0x2d, 0x42, 0x81, 0xfd, 0x96,
0xef, 0xee, 0x49, 0xf7, 0x37, 0x9a, 0xe5, 0xa4, 0x8f, 0x74, 0x50, 0xdf, 0xdd, 0xaa, 0x4b, 0xe5,
0x94, 0xf8, 0x5f, 0x70, 0x3e, 0x32, 0xb5, 0xf4, 0x80, 0xbf, 0xb8, 0x0f, 0xf8, 0x13, 0x7f, 0x22,
0x40, 0x35, 0x3a, 0x5f, 0x44, 0xaf, 0x8f, 0x2c, 0x7c, 0x7d, 0x8e, 0x64, 0x73, 0x64, 0xf5, 0xe8,
0x0a, 0x14, 0x4d, 0x7c, 0x8c, 0xed, 0x76, 0x87, 0xe5, 0xaf, 0x2c, 0x60, 0x16, 0xa4, 0x02, 0xa7,
0x52, 0x21, 0x8b, 0xb1, 0xbd, 0x83, 0xdb, 0xb6, 0xcc, 0x7c, 0x11, 0x33, 0xba, 0x1c, 0x61, 0x23,
0xd4, 0x03, 0x46, 0x14, 0xdf, 0x9e, 0x6b, 0x2f, 0x73, 0x90, 0x92, 0xea, 0x4d, 0xe9, 0xad, 0x72,
0x02, 0x21, 0x28, 0xd2, 0x9f, 0xf2, 0xc1, 0xee, 0xc6, 0xfe, 0x41, 0x63, 0x8f, 0xec, 0xe5, 0x12,
0x94, 0x9c, 0xbd, 0x74, 0x88, 0x29, 0xf1, 0x0f, 0x02, 0x3c, 0x1e, 0x91, 0xed, 0xa2, 0xdb, 0x00,
0xf6, 0x50, 0x36, 0x71, 0x5b, 0x37, 0xd5, 0x68, 0x23, 0x6b, 0x0e, 0x25, 0xca, 0x21, 0xe5, 0x6c,
0xfe, 0xcb, 0x9a, 0x80, 0x17, 0xa3, 0x97, 0xb9, 0x52, 0xb2, 0x2a, 0xe7, 0x53, 0xbb, 0x18, 0x02,
0x8b, 0xe2, 0x36, 0x51, 0x4c, 0xf7, 0x96, 0x2a, 0xa6, 0xfc, 0xe8, 0x7e, 0x98, 0x53, 0x99, 0xb1,
0x5b, 0x33, 0x9f, 0x3b, 0x49, 0x3d, 0x9a, 0x3b, 0x11, 0x7f, 0x9e, 0xf0, 0x6f, 0x6c, 0x30, 0xb9,
0xdf, 0x83, 0xb4, 0x65, 0x2b, 0xf6, 0xc0, 0xe2, 0x06, 0xf7, 0xe2, 0xac, 0x95, 0xc2, 0x9a, 0xf3,
0xe3, 0x80, 0x8a, 0x4b, 0x5c, 0xcd, 0x77, 0xfb, 0x6d, 0x89, 0xb7, 0xa0, 0x18, 0xdc, 0x9c, 0xe8,
0x4f, 0xc6, 0xf3, 0x39, 0x82, 0x78, 0xc7, 0xcb, 0xbf, 0x7c, 0xa0, 0xe5, 0x38, 0x20, 0x18, 0x0f,
0x03, 0x04, 0x7f, 0x11, 0x87, 0x27, 0x26, 0xd4, 0x4b, 0xe8, 0x8d, 0x91, 0x73, 0x7e, 0x69, 0x9e,
0x6a, 0x6b, 0x8d, 0xd1, 0x82, 0x27, 0x2d, 0xde, 0x84, 0x05, 0x3f, 0x7d, 0xb6, 0x45, 0xfe, 0x5e,
0xf0, 0x7c, 0x7e, 0x10, 0xb9, 0xfc, 0xda, 0x12, 0xcd, 0x11, 0x3b, 0x13, 0xe6, 0xb4, 0xb3, 0xd0,
0x64, 0x21, 0xf1, 0xcd, 0x25, 0x0b, 0xc9, 0x47, 0xb4, 0xb6, 0xb7, 0x00, 0x7c, 0x0d, 0xc9, 0x65,
0x48, 0x99, 0xfa, 0xa0, 0xaf, 0xd2, 0x63, 0x4e, 0x49, 0x6c, 0x80, 0x6e, 0x41, 0x8a, 0x98, 0x8b,
0xb3, 0x19, 0xe3, 0x9e, 0x93, 0x1c, 0xb7, 0x0f, 0xf3, 0x65, 0xdc, 0xa2, 0x06, 0x68, 0xbc, 0x29,
0x14, 0xf1, 0x8a, 0x57, 0x82, 0xaf, 0x78, 0x32, 0xb2, 0xbd, 0x14, 0xfe, 0xaa, 0xf7, 0x21, 0x45,
0x8f, 0x97, 0xe4, 0x27, 0xb4, 0xb1, 0xc9, 0x0b, 0x5e, 0xf2, 0x1b, 0x7d, 0x1f, 0x40, 0xb1, 0x6d,
0x53, 0x6b, 0x0d, 0xbc, 0x17, 0xac, 0x86, 0x9b, 0xc7, 0x86, 0xc3, 0xb7, 0x79, 0x81, 0xdb, 0xc9,
0xb2, 0x27, 0xea, 0xb3, 0x15, 0x9f, 0x42, 0x71, 0x17, 0x8a, 0x41, 0x59, 0xa7, 0x44, 0x63, 0x73,
0x08, 0x96, 0x68, 0xac, 0xe2, 0xe6, 0x25, 0x9a, 0x5b, 0xe0, 0x25, 0x58, 0x0f, 0x9b, 0x0e, 0xc4,
0x7f, 0xc4, 0x61, 0xc1, 0x6f, 0x5d, 0xff, 0x69, 0x55, 0x8e, 0xf8, 0x61, 0x1c, 0xb2, 0xee, 0xe2,
0x23, 0x1a, 0xc8, 0xde, 0xde, 0x09, 0xfe, 0x76, 0x29, 0xeb, 0x48, 0x27, 0xdc, 0x3e, 0xf7, 0x1d,
0x37, 0x21, 0x8a, 0x02, 0xa5, 0xfd, 0x3b, 0xed, 0xb4, 0xfa, 0x79, 0xfe, 0xf7, 0x63, 0x3e, 0x0f,
0x92, 0x09, 0xa0, 0xff, 0x81, 0xb4, 0xd2, 0x76, 0xa1, 0xf8, 0x62, 0x08, 0x36, 0xeb, 0xb0, 0xae,
0x35, 0x87, 0x1b, 0x94, 0x53, 0xe2, 0x12, 0x7c, 0x56, 0x82, 0xdb, 0x27, 0x7f, 0x95, 0xe8, 0x65,
0x3c, 0x41, 0xb7, 0x57, 0x04, 0x38, 0xdc, 0xbd, 0xbf, 0xb7, 0xb5, 0x7d, 0x77, 0xbb, 0xbe, 0xc5,
0x53, 0xa2, 0xad, 0xad, 0xfa, 0x56, 0x59, 0x20, 0x7c, 0x52, 0xfd, 0xfe, 0xde, 0x51, 0x7d, 0xab,
0x9c, 0x10, 0xef, 0x40, 0xce, 0x75, 0x1d, 0xa8, 0x02, 0x19, 0xa7, 0xad, 0x10, 0xe7, 0x11, 0x93,
0x77, 0x89, 0x96, 0x21, 0x65, 0xe8, 0xef, 0xf1, 0x2e, 0x71, 0x42, 0x62, 0x03, 0x51, 0x85, 0xd2,
0x88, 0xdf, 0x41, 0x77, 0x20, 0x63, 0x0c, 0x5a, 0xb2, 0x63, 0xb4, 0x23, 0x4d, 0x18, 0x07, 0x29,
0x18, 0xb4, 0xba, 0x5a, 0xfb, 0x1e, 0x3e, 0x73, 0xb6, 0xc9, 0x18, 0xb4, 0xee, 0x31, 0xdb, 0x66,
0x6f, 0x11, 0xfc, 0x6f, 0x39, 0x85, 0xac, 0xf3, 0xa9, 0xa2, 0xff, 0x85, 0x9c, 0xeb, 0xd2, 0xdc,
0xab, 0x33, 0x91, 0xbe, 0x90, 0xab, 0xf7, 0x44, 0xd0, 0x75, 0x58, 0xb4, 0xb4, 0x93, 0xbe, 0xd3,
0x82, 0x62, 0xc8, 0x9c, 0x40, 0xbf, 0x99, 0x12, 0x7b, 0xb0, 0xe3, 0xc0, 0x49, 0x24, 0x92, 0x95,
0x47, 0x7d, 0xc5, 0xb7, 0x39, 0x81, 0x90, 0x88, 0x9b, 0x08, 0x8b, 0xb8, 0x1f, 0x08, 0x90, 0xf7,
0x35, 0xb6, 0xd0, 0x7f, 0xfb, 0x1c, 0x57, 0x31, 0x24, 0x54, 0xf8, 0x78, 0xbd, 0x5b, 0x19, 0xc1,
0x85, 0x09, 0xf3, 0x2f, 0x2c, 0xaa, 0x8f, 0xe8, 0xf4, 0xc7, 0x92, 0x73, 0xf7, 0xc7, 0x9e, 0x05,
0x64, 0xeb, 0xb6, 0xd2, 0x95, 0x4f, 0x75, 0x5b, 0xeb, 0x9f, 0xc8, 0xcc, 0x34, 0x98, 0x9b, 0x29,
0xd3, 0x27, 0x47, 0xf4, 0xc1, 0x3e, 0xb5, 0x92, 0x1f, 0xc4, 0x21, 0xeb, 0x96, 0x6d, 0xf3, 0x5e,
0xb2, 0x38, 0x07, 0x69, 0x5e, 0x99, 0xb0, 0x5b, 0x16, 0x7c, 0x14, 0xda, 0x08, 0xac, 0x42, 0xb6,
0x87, 0x6d, 0x85, 0xfa, 0x4c, 0x06, 0x41, 0xba, 0xe3, 0xeb, 0x2f, 0x41, 0xde, 0x77, 0xdf, 0x85,
0xb8, 0xd1, 0xdd, 0xfa, 0x9b, 0xe5, 0x58, 0x35, 0xf3, 0xd1, 0x27, 0x97, 0x12, 0xbb, 0xf8, 0x3d,
0xf2, 0x85, 0x49, 0xf5, 0x5a, 0xa3, 0x5e, 0xbb, 0x57, 0x8e, 0x57, 0xf3, 0x1f, 0x7d, 0x72, 0x29,
0x23, 0x61, 0xda, 0xb7, 0xb9, 0x7e, 0x0f, 0x4a, 0x23, 0x07, 0x13, 0xfc, 0xa0, 0x11, 0x14, 0xb7,
0x0e, 0xf7, 0x77, 0xb6, 0x6b, 0x1b, 0xcd, 0xba, 0x7c, 0xb4, 0xd7, 0xac, 0x97, 0xe3, 0xe8, 0x71,
0x58, 0xda, 0xd9, 0x7e, 0xad, 0xd1, 0x94, 0x6b, 0x3b, 0xdb, 0xf5, 0xdd, 0xa6, 0xbc, 0xd1, 0x6c,
0x6e, 0xd4, 0xee, 0x95, 0x85, 0xf5, 0x5f, 0xe6, 0xa1, 0xb4, 0xb1, 0x59, 0xdb, 0x26, 0xb5, 0x99,
0xd6, 0x56, 0xa8, 0x7b, 0xa8, 0x41, 0x92, 0x82, 0xc0, 0x13, 0x6f, 0x30, 0x57, 0x27, 0x37, 0xf6,
0xd0, 0x5d, 0x48, 0x51, 0x7c, 0x18, 0x4d, 0xbe, 0xd2, 0x5c, 0x9d, 0xd2, 0xe9, 0x23, 0x93, 0xa1,
0x9f, 0xd3, 0xc4, 0x3b, 0xce, 0xd5, 0xc9, 0x8d, 0x3f, 0xb4, 0x03, 0x19, 0x07, 0xbe, 0x9b, 0x76,
0x5b, 0xb8, 0x3a, 0xb5, 0x83, 0x46, 0x96, 0xc6, 0x60, 0xd6, 0xc9, 0xd7, 0x9f, 0xab, 0x53, 0x5a,
0x82, 0x68, 0x1b, 0xd2, 0x1c, 0xe1, 0x98, 0x72, 0xf3, 0xb7, 0x3a, 0xad, 0x13, 0x86, 0x24, 0xc8,
0x79, 0x00, 0xf6, 0xf4, 0x4b, 0xdd, 0xd5, 0x19, 0xba, 0x9d, 0xe8, 0x01, 0x14, 0x82, 0xa8, 0xc9,
0x6c, 0xb7, 0x8b, 0xab, 0x33, 0xf6, 0xdc, 0x88, 0xfe, 0x20, 0x84, 0x32, 0xdb, 0x6d, 0xe3, 0xea,
0x8c, 0x2d, 0x38, 0xf4, 0x0e, 0x2c, 0x8e, 0x43, 0x1c, 0xb3, 0x5f, 0x3e, 0xae, 0xce, 0xd1, 0x94,
0x43, 0x3d, 0x40, 0x21, 0xd0, 0xc8, 0x1c, 0x77, 0x91, 0xab, 0xf3, 0xf4, 0xe8, 0x90, 0x0a, 0xa5,
0x51, 0xb8, 0x61, 0xd6, 0xbb, 0xc9, 0xd5, 0x99, 0xfb, 0x75, 0xec, 0x2d, 0xc1, 0xda, 0x7b, 0xd6,
0xbb, 0xca, 0xd5, 0x99, 0xdb, 0x77, 0xe8, 0x10, 0xc0, 0x57, 0x3b, 0xce, 0x70, 0x77, 0xb9, 0x3a,
0x4b, 0x23, 0x0f, 0x19, 0xb0, 0x14, 0x56, 0x54, 0xce, 0x73, 0x95, 0xb9, 0x3a, 0x57, 0x7f, 0x8f,
0xd8, 0x73, 0xb0, 0x3c, 0x9c, 0xed, 0x6a, 0x73, 0x75, 0xc6, 0x46, 0xdf, 0x66, 0xfd, 0xd3, 0x2f,
0x56, 0xe2, 0x9f, 0x7d, 0xb1, 0x12, 0xff, 0xeb, 0x17, 0x2b, 0xf1, 0x8f, 0xbf, 0x5c, 0x89, 0x7d,
0xf6, 0xe5, 0x4a, 0xec, 0x4f, 0x5f, 0xae, 0xc4, 0xfe, 0xef, 0x99, 0x13, 0xcd, 0xee, 0x0c, 0x5a,
0x6b, 0x6d, 0xbd, 0x77, 0xc3, 0xff, 0x2f, 0x97, 0xb0, 0x7f, 0xde, 0xb4, 0xd2, 0x34, 0xa0, 0xde,
0xfc, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1e, 0xe0, 0x41, 0x05, 0x99, 0x33, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.
@@ -7859,18 +7842,6 @@ func (m *ResponseFinalizeBlock) MarshalToSizedBuffer(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.RetainHeight != 0 {
i = encodeVarintTypes(dAtA, i, uint64(m.RetainHeight))
i--
dAtA[i] = 0x30
}
if len(m.AppHash) > 0 {
i -= len(m.AppHash)
copy(dAtA[i:], m.AppHash)
i = encodeVarintTypes(dAtA, i, uint64(len(m.AppHash)))
i--
dAtA[i] = 0x2a
}
if m.ConsensusParamUpdates != nil {
{
size, err := m.ConsensusParamUpdates.MarshalToSizedBuffer(dAtA[:i])
@@ -9900,13 +9871,6 @@ func (m *ResponseFinalizeBlock) Size() (n int) {
l = m.ConsensusParamUpdates.Size()
n += 1 + l + sovTypes(uint64(l))
}
l = len(m.AppHash)
if l > 0 {
n += 1 + l + sovTypes(uint64(l))
}
if m.RetainHeight != 0 {
n += 1 + sovTypes(uint64(m.RetainHeight))
}
return n
}
@@ -17380,59 +17344,6 @@ func (m *ResponseFinalizeBlock) Unmarshal(dAtA []byte) error {
return err
}
iNdEx = postIndex
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field AppHash", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTypes
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
byteLen |= int(b&0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthTypes
}
postIndex := iNdEx + byteLen
if postIndex < 0 {
return ErrInvalidLengthTypes
}
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.AppHash = append(m.AppHash[:0], dAtA[iNdEx:postIndex]...)
if m.AppHash == nil {
m.AppHash = []byte{}
}
iNdEx = postIndex
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field RetainHeight", wireType)
}
m.RetainHeight = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowTypes
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.RetainHeight |= int64(b&0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipTypes(dAtA[iNdEx:])
cmd/tendermint/commands/light.go
+2 -1
View File
@@ -171,7 +171,8 @@ for applications built w/ Cosmos SDK).
// If necessary adjust global WriteTimeout to ensure it's greater than
// TimeoutBroadcastTxCommit.
// See https://github.com/tendermint/tendermint/issues/3435
if cfg.WriteTimeout <= conf.RPC.TimeoutBroadcastTxCommit {
// Note we don't need to adjust anything if the timeout is already unlimited.
if cfg.WriteTimeout > 0 && cfg.WriteTimeout <= conf.RPC.TimeoutBroadcastTxCommit {
cfg.WriteTimeout = conf.RPC.TimeoutBroadcastTxCommit + 1*time.Second
}
config/config.go
+1 -1
View File
@@ -523,7 +523,7 @@ func DefaultRPCConfig() *RPCConfig {
MaxSubscriptionClients: 100,
MaxSubscriptionsPerClient: 5,
ExperimentalDisableWebsocket: false, // compatible with TM v0.35 and earlier
EventLogWindowSize: 0, // disables /events RPC by default
EventLogWindowSize: 30 * time.Second,
EventLogMaxItems: 0,
TimeoutBroadcastTxCommit: 10 * time.Second,
docs/nodes/README.md
+1 -1
View File
@@ -45,4 +45,4 @@ We will cover the various types of node types within Tendermint.
Validators are nodes that participate in the security of a network. Validators have an associated power in Tendermint, this power can represent stake in a [proof of stake](https://en.wikipedia.org/wiki/Proof_of_stake) system, reputation in [proof of authority](https://en.wikipedia.org/wiki/Proof_of_authority) or any sort of measurable unit. Running a secure and consistently online validator is crucial to a networks health. A validator must be secure and fault tolerant, it is recommended to run your validator with 2 or more sentry nodes.
As a validator there is the potential to have your weight reduced, this is defined by the application. Tendermint is notified by the application if a validator should have there weight increased or reduced. Application have different types of malicious behavior which lead to slashing of the validators power. Please check the documentation of the application you will be running in order to find more information.
As a validator there is the potential to have your weight reduced, this is defined by the application. Tendermint is notified by the application if a validator should have their weight increased or reduced. Application have different types of malicious behavior which lead to slashing of the validators power. Please check the documentation of the application you will be running in order to find more information.
docs/nodes/metrics.md
+47 -34
View File
@@ -18,40 +18,53 @@ Listen address can be changed in the config file (see
The following metrics are available:
| **Name** | **Type** | **Tags** | **Description** |
| -------------------------------------- | --------- | ------------- | ---------------------------------------------------------------------- |
| abci_connection_method_timing | Histogram | method, type | Timings for each of the ABCI methods |
| consensus_height | Gauge | | Height of the chain |
| consensus_validators | Gauge | | Number of validators |
| consensus_validators_power | Gauge | | Total voting power of all validators |
| consensus_validator_power | Gauge | | Voting power of the node if in the validator set |
| consensus_validator_last_signed_height | Gauge | | Last height the node signed a block, if the node is a validator |
| consensus_validator_missed_blocks | Gauge | | Total amount of blocks missed for the node, if the node is a validator |
| consensus_missing_validators | Gauge | | Number of validators who did not sign |
| consensus_missing_validators_power | Gauge | | Total voting power of the missing validators |
| consensus_byzantine_validators | Gauge | | Number of validators who tried to double sign |
| consensus_byzantine_validators_power | Gauge | | Total voting power of the byzantine validators |
| consensus_block_interval_seconds | Histogram | | Time between this and last block (Block.Header.Time) in seconds |
| consensus_rounds | Gauge | | Number of rounds |
| consensus_num_txs | Gauge | | Number of transactions |
| consensus_total_txs | Gauge | | Total number of transactions committed |
| consensus_block_parts | counter | peer_id | number of blockparts transmitted by peer |
| consensus_latest_block_height | gauge | | /status sync_info number |
| consensus_fast_syncing | gauge | | either 0 (not fast syncing) or 1 (syncing) |
| consensus_state_syncing | gauge | | either 0 (not state syncing) or 1 (syncing) |
| consensus_block_size_bytes | Gauge | | Block size in bytes |
| evidence_pool_num_evidence | Gauge | | Number of evidence in the evidence pool
| p2p_peers | Gauge | | Number of peers node's connected to |
| p2p_peer_receive_bytes_total | counter | peer_id, chID | number of bytes per channel received from a given peer |
| p2p_peer_send_bytes_total | counter | peer_id, chID | number of bytes per channel sent to a given peer |
| p2p_peer_pending_send_bytes | gauge | peer_id | number of pending bytes to be sent to a given peer |
| p2p_num_txs | gauge | peer_id | number of transactions submitted by each peer_id |
| p2p_pending_send_bytes | gauge | peer_id | amount of data pending to be sent to peer |
| mempool_size | Gauge | | Number of uncommitted transactions |
| mempool_tx_size_bytes | histogram | | transaction sizes in bytes |
| mempool_failed_txs | counter | | number of failed transactions |
| mempool_recheck_times | counter | | number of transactions rechecked in the mempool |
| state_block_processing_time | histogram | | time between BeginBlock and EndBlock in ms |
| **Name** | **Type** | **Tags** | **Description** |
|-----------------------------------------|-----------|-----------------|--------------------------------------------------------------------------------------------------------------------------------------------|
| abci_connection_method_timing | Histogram | method, type | Timings for each of the ABCI methods |
| consensus_height | Gauge | | Height of the chain |
| consensus_validators | Gauge | | Number of validators |
| consensus_validators_power | Gauge | | Total voting power of all validators |
| consensus_validator_power | Gauge | | Voting power of the node if in the validator set |
| consensus_validator_last_signed_height | Gauge | | Last height the node signed a block, if the node is a validator |
| consensus_validator_missed_blocks | Gauge | | Total amount of blocks missed for the node, if the node is a validator |
| consensus_missing_validators | Gauge | | Number of validators who did not sign |
| consensus_missing_validators_power | Gauge | | Total voting power of the missing validators |
| consensus_byzantine_validators | Gauge | | Number of validators who tried to double sign |
| consensus_byzantine_validators_power | Gauge | | Total voting power of the byzantine validators |
| consensus_block_interval_seconds | Histogram | | Time between this and last block (Block.Header.Time) in seconds |
| consensus_rounds | Gauge | | Number of rounds |
| consensus_num_txs | Gauge | | Number of transactions |
| consensus_total_txs | Gauge | | Total number of transactions committed |
| consensus_block_parts | Counter | peer_id | number of blockparts transmitted by peer |
| consensus_latest_block_height | gauge | | /status sync_info number |
| consensus_fast_syncing | gauge | | either 0 (not fast syncing) or 1 (syncing) |
| consensus_state_syncing | gauge | | either 0 (not state syncing) or 1 (syncing) |
| consensus_block_size_bytes | Gauge | | Block size in bytes |
| consensus_step_duration | Histogram | step | Histogram of durations for each step in the consensus protocol |
| consensus_block_gossip_receive_latency | Histogram | | Histogram of time taken to receive a block in seconds, measure between when a new block is first discovered to when the block is completed |
| consensus_block_gossip_parts_received | Counter | matches_current | Number of block parts received by the node |
| consensus_quorum_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the earliest prevote that achieved a quorum |
| consensus_full_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted |
| consensus_proposal_timestamp_difference | Histogram | | Difference between the timestamp in the proposal message and the local time of the validator at the time it received the message |
| consensus_vote_extension_receive_count | Counter | status | Number of vote extensions received |
| consensus_proposal_receive_count | Counter | status | Total number of proposals received by the node since process start |
| consensus_proposal_create_count | Counter | | Total number of proposals created by the node since process start |
| consensus_round_voting_power_percent | Gauge | vote_type | A value between 0 and 1.0 representing the percentage of the total voting power per vote type received within a round |
| consensus_late_votes | Counter | vote_type | Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in. |
| evidence_pool_num_evidence | Gauge | | Number of evidence in the evidence pool |
| p2p_peers | Gauge | | Number of peers node's connected to |
| p2p_peer_receive_bytes_total | Counter | peer_id, chID | number of bytes per channel received from a given peer |
| p2p_peer_send_bytes_total | Counter | peer_id, chID | number of bytes per channel sent to a given peer |
| p2p_peer_pending_send_bytes | Gauge | peer_id | number of pending bytes to be sent to a given peer |
| p2p_num_txs | Gauge | peer_id | number of transactions submitted by each peer_id |
| p2p_pending_send_bytes | Gauge | peer_id | amount of data pending to be sent to peer |
| mempool_size | Gauge | | Number of uncommitted transactions |
| mempool_tx_size_bytes | Histogram | | transaction sizes in bytes |
| mempool_failed_txs | Counter | | number of failed transactions |
| mempool_recheck_times | Counter | | number of transactions rechecked in the mempool |
| state_block_processing_time | Histogram | | time between BeginBlock and EndBlock in ms |
| state_consensus_param_updates | Counter | | number of consensus parameter updates returned by the application since process start |
| state_validator_set_updates | Counter | | number of validator set updates returned by the application since process start |
## Useful queries
docs/package.json
+1 -1
View File
@@ -15,7 +15,7 @@
"serve": "trap 'exit 0' SIGINT; vuepress dev --no-cache",
"postserve": "./post.sh",
"prebuild": "./pre.sh",
"build": "trap 'exit 0' SIGINT; vuepress build --no-cache",
"build": "trap 'exit 0' SIGINT; vuepress build --no-cache --silent",
"postbuild": "./post.sh"
},
"author": "",
docs/rfc/README.md
+4
View File
@@ -53,6 +53,10 @@ sections.
- [RFC-013: ABCI++](./rfc-013-abci++.md)
- [RFC-014: Semantic Versioning](./rfc-014-semantic-versioning.md)
- [RFC-015: ABCI++ Tx Mutation](./rfc-015-abci++-tx-mutation.md)
- [RFC-016: Node Architecture](./rfc-016-node-architecture.md)
- [RFC-017: ABCI++ Vote Extension Propagation](./rfc-017-abci++-vote-extension-propag.md)
- [RFC-018: BLS Signature Aggregation Exploration](./rfc-018-bls-agg-exploration.md)
- [RFC-019: Configuration File Versioning](./rfc-019-config-version.md)
- [RFC-020: Onboarding Projects](./rfc-020-onboarding-projects.rst)
<!-- - [RFC-NNN: Title](./rfc-NNN-title.md) -->
docs/rfc/images/node-dependency-tree.svg
+3
View File
File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 24 KiB

docs/rfc/rfc-009-consensus-parameter-upgrades.md
+3 -3
View File
@@ -31,12 +31,12 @@ not reference the new parameters. Any nodes joining the network with the newer
version of Tendermint will have the new consensus parameters. Tendermint will need
to handle this case so that new versions of Tendermint with new consensus parameters
can still validate old blocks correctly without having to do anything overly complex
or hacky.
or hacky.
### Allowing Developer-Defined Values and the `EndBlock` Problem
When new consensus parameters are added, application developers may wish to set
values for them so that the developer-defined values may be used as soon as the
values for them so that the developer-defined values may be used as soon as the
software upgrades. We do not currently have a clean mechanism for handling this.
Consensus parameter updates are communicated from the application to Tendermint
@@ -51,7 +51,7 @@ can take effect is height `H+1`. As of now, height `H` must run with the default
### Hash Compatibility
This section discusses possible solutions to the problem of maintaining backwards-compatibility
This section discusses possible solutions to the problem of maintaining backwards-compatibility
of hashed parameters while adding new parameters.
#### Never Hash Defaults
docs/rfc/rfc-016-node-architecture.md
+83
View File
@@ -0,0 +1,83 @@
# RFC 016: Node Architecture
## Changelog
- April 8, 2022: Initial draft (@cmwaters)
- April 15, 2022: Incorporation of feedback
## Abstract
The `node` package is the entry point into the Tendermint codebase, used both by the command line and programatically to create the nodes that make up a network. The package has suffered the most from the evolution of the codebase, becoming bloated as developers clipped on their bits of code here and there to get whatever feature they wanted working.
The decisions made at the node level have the biggest impact to simplifying the protocols within them, unlocking better internal designs and making Tendermint more intuitive to use and easier to understand from the outside. Work, in minor increments, has already begun on this section of the codebase. This document exists to spark forth the necessary discourse in a few related areas that will help the team to converge on the long term makeup of the node.
## Discussion
The following is a list of points of discussion around the architecture of the node:
### Dependency Tree
The node object is currently stuffed with every component that possibly exists within Tendermint. In the constructor, all objects are built and interlaid with one another in some awkward dance. My guiding principle is that the node should only be made up of the components that it wants to have direct control of throughout its life. The node is a service which currently has the purpose of starting other services up in a particular order and stopping them all when commanded to do so. However, there are many services which are not direct dependents i.e. the mempool and evidence services should only be working when the consensus service is running. I propose to form more of a hierarchical structure of dependents which forces us to be clear about the relations that one component has to the other. More concretely, I propose the following dependency tree:
![node dependency tree](./images/node-dependency-tree.svg)
Many of the further discussion topics circle back to this representation of the node.
It's also important to distinguish two dimensions which may require different characteristics of the architecture. There is the starting and stopping of services and their general lifecycle management. What is the correct order of operations to starting a node for example. Then there is the question of the needs of the service during actual operation. Then there is the question of what resources each service needs access to during its operation. Some need to publish events, others need access to data stores, and so forth.
An alternative model and one that perhaps better suits the latter of these dimensions is the notion of an internal message passing system. Either the events bus or p2p layer could serve as a viable transport. This would essentially allow all services to communicate with any other service and could perhaps provide a solution to the coordination problem (presented below) without a centralized coordinator. The other main advantage is that such a system would be more robust to disruptions and changes to the code which may make a hierarchical structure quickly outdated and suboptimal. The addition of message routing is an added complexity to implement, will increase the degree of asynchronicity in the system and may make it harder to debug problems that are across multiple services.
### Coordination of State Advancing Mechanisms
Advancement of state in Tendermint is simply defined in heights: If the node is at height n, how does it get to height n + 1 and so on. Based on this definition we have three components that help a node to advance in height: consensus, statesync and blocksync. The way these components behave currently is very tightly coupled to one another with references passed back and forth. My guiding principle is that each of these should be able to operate completely independently of each other, e.g. a node should be able to run solely blocksync indefinitely. There have been several ideas suggested towards improving this flow. I've been leaning strongly towards a centralized system, whereby an orchestrator (in this case the node) decides what services to start and stop.
In a decentralized message passing system, individual services make their decision based upon a "global" shared state i.e. if my height is less that 10 below the average peer height, I as consensus, should stop (knowing that blocksync has the same condition for starting). As the example illustrates, each mechanism will still need to be aware of the presence of other mechanisms.
Both centralized and decentralized systems rely on the communication of the nodes current height and a judgement on the height of the head of the chain. The latter, working out the head of the chain, is quite a difficult challenge as their is nothing preventing the node from acting maliciously and providing a different height. Currently both blocksync, consensus (and to a certain degree statesync), have parallel systems where peers communicate their height. This could be streamlined with the consensus (or even the p2p layer), broadcasting peer heights and either the node or the other state advancing mechanisms acting accordingly.
Currently, when a node starts, it turns on every service that it is attached to. This means that while a node is syncing up by requesting blocks, it is also receiving transactions and votes, as well as snapshot and block requests. This is a needless use of bandwidth. An implementation of an orchestrator, regardless of whether the system is heirachical or not, should look to be able to open and close channels dynamically and effectively broadcast which services it is running. Integrating this with service discovery may also lead to a better serivce to peers.
The orchestrator allows for some deal of variablity in how a node is constructed. Does it just run blocksync, shadowing the head of the chain and be highly available for querying. Does it rely on state sync at all? An important question that arises from this dynamicism is we ideally want to encourage nodes to provide as much of their resources as possible so that their is a healthy amount of providers to consumers. Do we make all services compulsory or allow for them to be disabled? Arguably it's possible that a user forks the codebase and rips out the blocksync code because they want to reduce bandwidth so this is more a question of how easy do we want to make this for users.
### Block Executor
The block executor is an important component that is currently used by both consensus and blocksync to execute transactions and update application state. Principally, I think it should be the only component that can write (and possibly even read) the block and state stores, and we should clean up other direct dependencies on the storage engine if we can. This would mean:
- The reactors Consensus, BlockSync and StateSync should all import the executor for advancing state ie. `ApplyBlock` and `BootstrapState`.
- Pruning should also be a concern of the block executor as well as `FinalizeBlock` and `Commit`. This can simplify consensus to focus just on the consensus part.
### The Interprocess communication systems: RPC, P2P, ABCI, and Events
The schematic supplied above shows the relations between the different services, the node, the block executor, and the storage layer. Represented as colored dots are the components responsible for different roles of interprocess communication (IPC). These components permeate throughout the code base, seeping into most services. What can provide powerful functionality on one hand can also become a twisted vine, creating messy corner cases and convoluting the protocols themselves. A lot of the thinking around
how we want our IPC systens to function has been summarised in this [RFC](./rfc-002-ipc-ecosystem.md). In this section, I'd like to focus the reader on the relation between the IPC and the node structure. An issue that has frequently risen is that the RPC has control of the components where it strikes me as being more logical for the component to dictate the information that is emitted/available and the knobs it wishes to expose. The RPC is also inextricably tied to the node instance and has situations where it is passed pointers directly to the storage engine and other components.
I am currently convinced of the approach that the p2p layer takes and would like to see other IPC components follow suit. This would mean that the RPC and events system would be constructed in the node yet would pass the adequate methods to register endpoints and topics to the sub components. For example,
```go
// Methods from the RPC and event bus that would be passed into the constructor of components like "consensus"
// NOTE: This is a hypothetical construction to convey the idea. An actual implementation may differ.
func RegisterRoute(path string, handler func(http.ResponseWriter, *http.Request))
func RegisterTopic(name string) EventPublisher
type EventPublisher func (context.Context, types.EventData, []abci.Event)
```
This would give the components control to the information they want to expose and keep all relevant logic within that package. It accomodates more to a dynamic system where services can switch on and off. Each component would also receive access to the logger and metrics system for introspection and debuggability.
#### IPC Rubric
I'd like to aim to reach a state where we as a team have either an implicit or explicit rubric which can determine, in the event of some new need to communicate information, what tool it should use for doing this. In the case of inter node communication, this is obviously the p2p stack (with perhaps the exception of the light client). Metrics and logging also have clear usage patterns. RPC and the events system are less clear. The RPC is used for debugging data and fine tuned operator control as it is for general public querying and transaction submission. The RPC is also known to have been plumbed back into the application for historical queries. The events system, similarly, is used for consuming transaction events as it is for the testing of consensus state transitions.
Principally, I think we should look to change our language away from what the actual transport is and more towards what it's being used for and to whom. We call it a peer to peer layer and not the underlying tcp connection. In the same way, we should look to split RPC into an operator interface (RPC Internal), a public interface (RPC External) and a bidirectional ABCI.
### Seperation of consumers and suppliers
When a service such as blocksync is turned on, it automatically begins requesting blocks to verify and apply them as it also tries to serve them to other peers catching up. We should look to distinguish these two aspects: supplying of information and consuming of information in many of these components. More concretely, I'd suggest:
- The blocksync and statesync service, i.e. supplying information for those trying to catch up should only start running once a node has caught up i.e. after running the blocksync and/or state sync *processes*
- The blocksync and state sync processes have defined termination clauses that inform the orchestrator when they are done and where they finished.
- One way of achieving this would be that every process both passes and returns the `State` object
- In some cases, a node may specify that it wants to run blocksync indefinitely.
- The mempool should also indicate whether it wants to receive transactions or to send them only (one-directional mempool)
- Similarly, the light client itself only requests information whereas the light client service (currently part of state sync) can do both.
- This distinction needs to be communicated in the p2p layer handshake itself but should also be changeable over the lifespan of the connection.
docs/rfc/rfc-017-abci++-vote-extension-propag.md
+571
View File
@@ -0,0 +1,571 @@
# RFC 017: ABCI++ Vote Extension Propagation
## Changelog
- 11-Apr-2022: Initial draft (@sergio-mena).
- 15-Apr-2022: Addressed initial comments. First complete version (@sergio-mena).
- 09-May-2022: Addressed all outstanding comments.
## Abstract
According to the
[ABCI++ specification](https://github.com/tendermint/tendermint/blob/4743a7ad0/spec/abci%2B%2B/README.md)
(as of 11-Apr-2022), a validator MUST provide a signed vote extension for each non-`nil` precommit vote
of height *h* that it uses to propose a block in height *h+1*. When a validator is up to
date, this is easy to do, but when a validator needs to catch up this is far from trivial as this data
cannot be retrieved from the blockchain.
This RFC presents and compares the different options to address this problem, which have been proposed
in several discussions by the Tendermint Core team.
## Document Structure
The RFC is structured as follows. In the [Background](#background) section,
subsections [Problem Description](#problem-description) and [Cases to Address](#cases-to-address)
explain the problem at hand from a high level perspective, i.e., abstracting away from the current
Tendermint implementation. In contrast, subsection
[Current Catch-up Mechanisms](#current-catch-up-mechanisms) delves into the details of the current
Tendermint code.
In the [Discussion](#discussion) section, subsection [Solutions Proposed](#solutions-proposed) is also
worded abstracting away from implementation details, whilst subsections
[Feasibility of the Proposed Solutions](#feasibility-of-the-proposed-solutions) and
[Current Limitations and Possible Implementations](#current-limitations-and-possible-implementations)
analize the viability of one of the proposed solutions in the context of Tendermint's architecture
based on reactors. Finally, [Formalization Work](#formalization-work) briefly discusses the work
still needed demonstrate the correctness of the chosen solution.
The high level subsections are aimed at readers who are familiar with consensus algorithms, in
particular with the one described in the Tendermint (white paper), but who are not necessarily
acquainted with the details of the Tendermint codebase. The other subsections, which go into
implementation details, are best understood by engineers with deep knowledge of the implementation of
Tendermint's blocksync and consensus reactors.
## Background
### Basic Definitions
This document assumes that all validators have equal voting power for the sake of simplicity. This is done
without loss of generality.
There are two types of votes in Tendermint: *prevotes* and *precommits*. Votes can be `nil` or refer to
a proposed block. This RFC focuses on precommits,
also known as *precommit votes*. In this document we sometimes call them simply *votes*.
Validators send precommit votes to their peer nodes in *precommit messages*. According to the
[ABCI++ specification](https://github.com/tendermint/tendermint/blob/4743a7ad0/spec/abci%2B%2B/README.md),
a precommit message MUST also contain a *vote extension*.
This mandatory vote extension can be empty, but MUST be signed with the same key as the precommit
vote (i.e., the sending validator's).
Nevertheless, the vote extension is signed independently from the vote, so a vote can be separated from
its extension.
The reason for vote extensions to be mandatory in precommit messages is that, otherwise, a (malicious)
node can omit a vote extension while still providing/forwarding/sending the corresponding precommit vote.
The validator set at height *h* is denoted *valset<sub>h</sub>*. A *commit* for height *h* consists of more
than *2n<sub>h</sub>/3* precommit votes voting for a block *b*, where *n<sub>h</sub>* denotes the size of
*valset<sub>h</sub>*. A commit does not contain `nil` precommit votes, and all votes in it refer to the
same block. An *extended commit* is a *commit* where every precommit vote has its respective vote extension
attached.
### Problem Description
In the version of [ABCI](https://github.com/tendermint/spec/blob/4fb99af/spec/abci/README.md) present up to
Tendermint v0.35, for any height *h*, a validator *v* MUST have the decided block *b* and a commit for
height *h* in order to decide at height *h*. Then, *v* just needs a commit for height *h* to propose at
height *h+1*, in the rounds of *h+1* where *v* is a proposer.
In [ABCI++](https://github.com/tendermint/tendermint/blob/4743a7ad0/spec/abci%2B%2B/README.md),
the information that a validator *v* MUST have to be able to decide in *h* does not change with
respect to pre-existing ABCI: the decided block *b* and a commit for *h*.
In contrast, for proposing in *h+1*, a commit for *h* is not enough: *v* MUST now have an extended
commit.
When a validator takes an active part in consensus at height *h*, it has all the data it needs in memory,
in its consensus state, to decide on *h* and propose in *h+1*. Things are not so easy in the cases when
*v* cannot take part in consensus because it is late (e.g., it falls behind, it crashes
and recovers, or it just starts after the others). If *v* does not take part, it cannot actively
gather precommit messages (which include vote extensions) in order to decide.
Before ABCI++, this was not a problem: full nodes are supposed to persist past blocks in the block store,
so other nodes would realise that *v* is late and send it the missing decided block at height *h* and
the corresponding commit (kept in block *h+1*) so that *v* can catch up.
However, we cannot apply this catch-up technique for ABCI++, as the vote extensions, which are part
of the needed *extended commit* are not part of the blockchain.
### Cases to Address
Before we tackle the description of the possible cases we need to address, let us describe the following
incremental improvement to the ABCI++ logic. Upon decision, a full node persists (e.g., in the block
store) the extended commit that allowed the node to decide. For the moment, let us assume the node only
needs to keep its *most recent* extended commit, and MAY remove any older extended commits from persistent
storage.
This improvement is so obvious that all solutions described in the [Discussion](#discussion) section use
it as a building block. Moreover, it completely addresses by itself some of the cases described in this
subsection.
We now describe the cases (i.e. possible *runs* of the system) that have been raised in different
discussions and need to be addressed. They are (roughly) ordered from easiest to hardest to deal with.
- **(a)** *Happy path: all validators advance together, no crash*.
This case is included for completeness. All validators have taken part in height *h*.
Even if some of them did not manage to send a precommit message for the decided block, they all
receive enough precommit messages to be able to decide. As vote extensions are mandatory in
precommit messages, every validator *v* trivially has all the information, namely the decided block
and the extended commit, needed to propose in height *h+1* for the rounds in which *v* is the
proposer.
No problem to solve here.
- **(b)** *All validators advance together, then all crash at the same height*.
This case has been raised in some discussions, the main concern being whether the vote extensions
for the previous height would be lost across the network. With the improvement described above,
namely persisting the latest extended commit at decision time, this case is solved.
When a crashed validator recovers, it recovers the last extended commit from persistent storage
and handshakes with the Application.
If need be, it also reconstructs messages for the unfinished height
(including all precommits received) from the WAL.
Then, the validator can resume where it was at the time of the crash. Thus, as extensions are
persisted, either in the WAL (in the form of received precommit messages), or in the latest
extended commit, the only way that vote extensions needed to start the next height could be lost
forever would be if all validators crashed and never recovered (e.g. disk corruption).
Since a *correct* node MUST eventually recover, this violates Tendermint's assumption of more than
*2n<sub>h</sub>/3* correct validators for every height *h*.
No problem to solve here.
- **(c)** *Lagging majority*.
Let us assume the validator set does not change between *h* and *h+1*.
It is not possible by the nature of the Tendermint algorithm, which requires more
than *2n<sub>h</sub>/3* precommit votes for some round of height *h* in order to make progress.
So, only up to *n<sub>h</sub>/3* validators can lag behind.
On the other hand, for the case where there are changes to the validator set between *h* and
*h+1* please see case (d) below, where the extreme case is discussed.
- **(d)** *Validator set changes completely between* h *and* h+1.
If sets *valset<sub>h</sub>* and *valset<sub>h+1</sub>* are disjoint,
more than *2n<sub>h</sub>/3* of validators in height *h* should
have actively participated in conensus in *h*. So, as of height *h*, only a minority of validators
in *h* can be lagging behind, although they could all lag behind from *h+1* on, as they are no
longer validators, only full nodes. This situation falls under the assumptions of case (h) below.
As for validators in *valset<sub>h+1</sub>*, as they were not validators as of height *h*, they
could all be lagging behind by that time. However, by the time *h* finishes and *h+1* begins, the
chain will halt until more than *2n<sub>h+1</sub>/3* of them have caught up and started consensus
at height *h+1*. If set *valset<sub>h+1</sub>* does not change in *h+2* and subsequent
heights, only up to *n<sub>h+1</sub>/3* validators will be able to lag behind. Thus, we have
converted this case into case (h) below.
- **(e)** *Enough validators crash to block the rest*.
In this case, blockchain progress halts, i.e. surviving full nodes keep increasing rounds
indefinitely, until some of the crashed validators are able to recover.
Those validators that recover first will handshake with the Application and recover at the height
they crashed, which is still the same the nodes that did not crash are stuck in, so they don't need
to catch up.
Further, they had persisted the extended commit for the previous height. Nothing to solve.
For those validators recovering later, we are in case (h) below.
- **(f)** *Some validators crash, but not enough to block progress*.
When the correct processes that crashed recover, they handshake with the Application and resume at
the height they were at when they crashed. As the blockchain did not stop making progress, the
recovered processes are likely to have fallen behind with respect to the progressing majority.
At this point, the recovered processes are in case (h) below.
- **(g)** *A new full node starts*.
The reasoning here also applies to the case when more than one full node are starting.
When the full node starts from scratch, it has no state (its current height is 0). Ignoring
statesync for the time being, the node just needs to catch up by applying past blocks one by one
(after verifying them).
Thus, the node is in case (h) below.
- **(h)** *Advancing majority, lagging minority*
In this case, some nodes are late. More precisely, at the present time, a set of full nodes,
denoted *L<sub>h<sub>p</sub></sub>*, are falling behind
(e.g., temporary disconnection or network partition, memory thrashing, crashes, new nodes)
an arbitrary
number of heights:
between *h<sub>s</sub>* and *h<sub>p</sub>*, where *h<sub>s</sub> < h<sub>p</sub>*, and
*h<sub>p</sub>* is the highest height
any correct full node has reached so far.
The correct full nodes that reached *h<sub>p</sub>* were able to decide for *h<sub>p</sub>-1*.
Therefore, less than *n<sub>h<sub>p</sub>-1</sub>/3* validators of *h<sub>p</sub>-1* can be part
of *L<sub>h<sub>p</sub></sub>*, since enough up-to-date validators needed to actively participate
in consensus for *h<sub>p</sub>-1*.
Since, at the present time,
no node in *L<sub>h<sub>p</sub></sub>* took part in any consensus between
*h<sub>s</sub>* and *h<sub>p</sub>-1*,
the reasoning above can be extended to validator set changes between *h<sub>s</sub>* and
*h<sub>p</sub>-1*. This results in the following restriction on the full nodes that can be part of *L<sub>h<sub>p</sub></sub>*.
- &forall; *h*, where *h<sub>s</sub> ≤ h < h<sub>p</sub>*,
| *valset<sub>h</sub>* &cap; *L<sub>h<sub>p</sub></sub>* | *< n<sub>h</sub>/3*
If this property does not hold for a particular height *h*, where
*h<sub>s</sub> ≤ h < h<sub>p</sub>*, Tendermint could not have progressed beyond *h* and
therefore no full node could have reached *h<sub>p</sub>* (a contradiction).
These lagging nodes in *L<sub>h<sub>p</sub></sub>* need to catch up. They have to obtain the
information needed to make
progress from other nodes. For each height *h* between *h<sub>s</sub>* and *h<sub>p</sub>-2*,
this includes the decided block for *h*, and the
precommit votes also for *deciding h* (which can be extracted from the block at height *h+1*).
At a given height *h<sub>c</sub>* (where possibly *h<sub>c</sub> << h<sub>p</sub>*),
a full node in *L<sub>h<sub>p</sub></sub>* will consider itself *caught up*, based on the
(maybe out of date) information it is getting from its peers. Then, the node needs to be ready to
propose at height *h<sub>c</sub>+1*, which requires having received the vote extensions for
*h<sub>c</sub>*.
As the vote extensions are *not* stored in the blocks, and it is difficult to have strong
guarantees on *when* a late node considers itself caught up, providing the late node with the right
vote extensions for the right height poses a problem.
At this point, we have described and compared all cases raised in discussions leading up to this
RFC. The list above aims at being exhaustive. The analysis of each case included above makes all of
them converge into case (h).
### Current Catch-up Mechanisms
We now briefly describe the current catch-up mechanisms in the reactors concerned in Tendermint.
#### Statesync
Full nodes optionally run statesync just after starting, when they start from scratch.
If statesync succeeds, an Application snapshot is installed, and Tendermint jumps from height 0 directly
to the height the Application snapshop represents, without applying the block of any previous height.
Some light blocks are received and stored in the block store for running light-client verification of
all the skipped blocks. Light blocks are incomplete blocks, typically containing the header and the
canonical commit but, e.g., no transactions. They are stored in the block store as "signed headers".
The statesync reactor is not really relevant for solving the problem discussed in this RFC. We will
nevertheless mention it when needed; in particular, to understand some corner cases.
#### Blocksync
The blocksync reactor kicks in after start up or recovery (and, optionally, after statesync is done)
and sends the following messages to its peers:
- `StatusRequest` to query the height its peers are currently at, and
- `BlockRequest`, asking for blocks of heights the local node is missing.
Using `BlockResponse` messages received from peers, the blocksync reactor validates each received
block using the block of the following height, saves the block in the block store, and sends the
block to the Application for execution.
If blocksync has validated and applied the block for the height *previous* to the highest seen in
a `StatusResponse` message, or if no progress has been made after a timeout, the node considers
itself as caught up and switches to the consensus reactor.
#### Consensus Reactor
The consensus reactor runs the full Tendermint algorithm. For a validator this means it has to
propose blocks, and send/receive prevote/precommit messages, as mandated by Tendermint, before it can
decide and move on to the next height.
If a full node that is running the consensus reactor falls behind at height *h*, when a peer node
realises this it will retrieve the canonical commit of *h+1* from the block store, and *convert*
it into a set of precommit votes and will send those to the late node.
## Discussion
### Solutions Proposed
These are the solutions proposed in discussions leading up to this RFC.
- **Solution 0.** *Vote extensions are made **best effort** in the specification*.
This is the simplest solution, considered as a way to provide vote extensions in a simple enough
way so that it can be part of v0.36.
It consists in changing the specification so as to not *require* that precommit votes used upon
`PrepareProposal` contain their corresponding vote extensions. In other words, we render vote
extensions optional.
There are strong implications stemming from such a relaxation of the original specification.
- As a vote extension is signed *separately* from the vote it is extending, an intermediate node
can now remove (i.e., censor) vote extensions from precommit messages at will.
- Further, there is no point anymore in the spec requiring the Application to accept a vote extension
passed via `VerifyVoteExtension` to consider a precommit message valid in its entirety. Remember
this behavior of `VerifyVoteExtension` is adding a constraint to Tendermint's conditions for
liveness.
In this situation, it is better and simpler to just drop the vote extension rejected by the
Application via `VerifyVoteExtension`, but still consider the precommit vote itself valid as long
as its signature verifies.
- **Solution 1.** *Include vote extensions in the blockchain*.
Another obvious solution, which has somehow been considered in the past, is to include the vote
extensions and their signatures in the blockchain.
The blockchain would thus include the extended commit, rather than a regular commit, as the structure
to be canonicalized in the next block.
With this solution, the current mechanisms implemented both in the blocksync and consensus reactors
would still be correct, as all the information a node needs to catch up, and to start proposing when
it considers itself as caught-up, can now be recovered from past blocks saved in the block store.
This solution has two main drawbacks.
- As the block format must change, upgrading a chain requires a hard fork. Furthermore,
all existing light client implementations will stop working until they are upgraded to deal with
the new format (e.g., how certain hashes calculated and/or how certain signatures are checked).
For instance, let us consider IBC, which relies on light clients. An IBC connection between
two chains will be broken if only one chain upgrades.
- The extra information (i.e., the vote extensions) that is now kept in the blockchain is not really
needed *at every height* for a late node to catch up.
- This information is only needed to be able to *propose* at the height the validator considers
itself as caught-up. If a validator is indeed late for height *h*, it is useless (although
correct) for it to call `PrepareProposal`, or `ExtendVote`, since the block is already decided.
- Moreover, some use cases require pretty sizeable vote extensions, which would result in an
important waste of space in the blockchain.
- **Solution 2.** *Skip* propose *step in Tendermint algorithm*.
This solution consists in modifying the Tendermint algorithm to skip the *send proposal* step in
heights where the node does not have the required vote extensions to populate the call to
`PrepareProposal`. The main idea behind this is that it should only happen when the validator is late
and, therefore, up-to-date validators have already proposed (and decided) for that height.
A small variation of this solution is, rather than skipping the *send proposal* step, the validator
sends a special *empty* or *bottom* (⊥) proposal to signal other nodes that it is not ready to propose
at (any round of) the current height.
The appeal of this solution is its simplicity. A possible implementation does not need to extend
the data structures, or change the current catch-up mechanisms implemented in the blocksync or
in the consensus reactor. When we lack the needed information (vote extensions), we simply rely
on another correct validator to propose a valid block in other rounds of the current height.
However, this solution can be attacked by a byzantine node in the network in the following way.
Let us consider the following scenario:
- all validators in *valset<sub>h</sub>* send out precommit messages, with vote extensions,
for height *h*, round 0, roughly at the same time,
- all those precommit messages contain non-`nil` precommit votes, which vote for block *b*
- all those precommit messages sent in height *h*, round 0, and all messages sent in
height *h*, round *r > 0* get delayed indefinitely, so,
- all validators in *valset<sub>h</sub>* keep waiting for enough precommit
messages for height *h*, round 0, needed for deciding in height *h*
- an intermediate (malicious) full node *m* manages to receive block *b*, and gather more than
*2n<sub>h</sub>/3* precommit messages for height *h*, round 0,
- one way or another, the solution should have either (a) a mechanism for a full node to *tell*
another full node it is late, or (b) a mechanism for a full node to conclude it is late based
on other full nodes' messages; any of these mechanisms should, at the very least,
require the late node receiving the decided block and a commit (not necessarily an extended
commit) for *h*,
- node *m* uses the gathered precommit messages to build a commit for height *h*, round 0,
- in order to convince full nodes that they are late, node *m* either (a) *tells* them they
are late, or (b) shows them it (i.e. *m*) is ahead, by sending them block *b*, along with the
commit for height *h*, round 0,
- all full nodes conclude they are late from *m*'s behavior, and use block *b* and the commit for
height *h*, round 0, to decide on height *h*, and proceed to height *h+1*.
At this point, *all* full nodes, including all validators in *valset<sub>h+1</sub>*, have advanced
to height *h+1* believing they are late, and so, expecting the *hypothetical* leading majority of
validators in *valset<sub>h+1</sub>* to propose for *h+1*. As a result, the blockhain
grinds to a halt.
A (rather complex) ad-hoc mechanism would need to be carried out by node operators to roll
back all validators to the precommit step of height *h*, round *r*, so that they can regenerate
vote extensions (remember vote extensions are non-deterministic) and continue execution.
- **Solution 3.** *Require extended commits to be available at switching time*.
This one is more involved than all previous solutions, and builds on an idea present in Solution 2:
vote extensions are actually not needed for Tendermint to make progress as long as the
validator is *certain* it is late.
We define two modes. The first is denoted *catch-up mode*, and Tendermint only calls
`FinalizeBlock` for each height when in this mode. The second is denoted *consensus mode*, in
which the validator considers itself up to date and fully participates in consensus and calls
`PrepareProposal`/`ProcessProposal`, `ExtendVote`, and `VerifyVoteExtension`, before calling
`FinalizeBlock`.
The catch-up mode does not need vote extension information to make progress, as all it needs is the
decided block at each height to call `FinalizeBlock` and keep the state-machine replication making
progress. The consensus mode, on the other hand, does need vote extension information when
starting every height.
Validators are in consensus mode by default. When a validator in consensus mode falls behind
for whatever reason, e.g. cases (b), (d), (e), (f), (g), or (h) above, we introduce the following
key safety property:
- for every height *h<sub>p</sub>*, a full node *f* in *h<sub>p</sub>* refuses to switch to catch-up
mode **until** there exists a height *h'* such that:
- *p* has received and (light-client) verified the blocks of
all heights *h*, where *h<sub>p</sub> ≤ h ≤ h'*
- it has received an extended commit for *h'* and has verified:
- the precommit vote signatures in the extended commit
- the vote extension signatures in the extended commit: each is signed with the same
key as the precommit vote it extends
If the condition above holds for *h<sub>p</sub>*, namely receiving a valid sequence of blocks in
the *f*'s future, and an extended commit corresponding to the last block in the sequence, then
node *f*:
- switches to catch-up mode,
- applies all blocks between *h<sub>p</sub>* and *h'* (calling `FinalizeBlock` only), and
- switches back to consensus mode using the extended commit for *h'* to propose in the rounds of
*h' + 1* where it is the proposer.
This mechanism, together with the invariant it uses, ensures that the node cannot be attacked by
being fed a block without extensions to make it believe it is late, in a similar way as explained
for Solution 2.
### Feasibility of the Proposed Solutions
Solution 0, besides the drawbacks described in the previous section, provides guarantees that are
weaker than the rest. The Application does not have the assurance that more than *2n<sub>h</sub>/3* vote
extensions will *always* be available when calling `PrepareProposal` at height *h+1*.
This level of guarantees is probably not strong enough for vote extensions to be useful for some
important use cases that motivated them in the first place, e.g., encrypted mempool transactions.
Solution 1, while being simple in that the changes needed in the current Tendermint codebase would
be rather small, is changing the block format, and would therefore require all blockchains using
Tendermint v0.35 or earlier to hard-fork when upgrading to v0.36.
Since Solution 2 can be attacked, one might prefer Solution 3, even if it is more involved
to implement. Further, we must elaborate on how we can turn Solution 3, described in abstract
terms in the previous section, into a concrete implementation compatible with the current
Tendermint codebase.
### Current Limitations and Possible Implementations
The main limitations affecting the current version of Tendermint are the following.
- The current version of the blocksync reactor does not use the full
[light client verification](https://github.com/tendermint/tendermint/blob/4743a7ad0/spec/light-client/README.md)
algorithm to validate blocks coming from other peers.
- The code being structured into the blocksync and consensus reactors, only switching from the
blocksync reactor to the consensus reactor is supported; switching in the opposite direction is
not supported. Alternatively, the consensus reactor could have a mechanism allowing a late node
to catch up by skipping calls to `PrepareProposal`/`ProcessProposal`, and
`ExtendVote`/`VerifyVoteExtension` and only calling `FinalizeBlock` for each height.
Such a mechanism does not exist at the time of writing this RFC.
The blocksync reactor featuring light client verification is being actively worked on (tentatively
for v0.37). So it is best if this RFC does not try to delve into that problem, but just makes sure
its outcomes are compatible with that effort.
In subsection [Cases to Address](#cases-to-address), we concluded that we can focus on
solving case (h) in theoretical terms.
However, as the current Tendermint version does not yet support switching back to blocksync once a
node has switched to consensus, we need to split case (h) into two cases. When a full node needs to
catch up...
- **(h.1)** ... it has not switched yet from the blocksync reactor to the consensus reactor, or
- **(h.2)** ... it has already switched to the consensus reactor.
This is important in order to discuss the different possible implementations.
#### Base Implementation: Persist and Propagate Extended Commit History
In order to circumvent the fact that we cannot switch from the consensus reactor back to blocksync,
rather than just keeping the few most recent extended commits, nodes will need to keep
and gossip a backlog of extended commits so that the consensus reactor can still propose and decide
in out-of-date heights (even if those proposals will be useless).
The base implementation － for which an experimental patch exists － consists in the conservative
approach of persisting in the block store *all* extended commits for which we have also stored
the full block. Currently, when statesync is run at startup, it saves light blocks.
This base implementation does not seek
to receive or persist extended commits for those light blocks as they would not be of any use.
Then, we modify the blocksync reactor so that peers *always* send requested full blocks together
with the corresponding extended commit in the `BlockResponse` messages. This guarantees that the
block store being reconstructed by blocksync has the same information as that of peers that are
up to date (at least starting from the latest snapshot applied by statesync before starting blocksync).
Thus, blocksync has all the data it requires to switch to the consensus reactor, as long as one of
the following exit conditions are met:
- The node is still at height 0 (where no commit or extended commit is needed)
- The node has processed at least 1 block in blocksync
The second condition is needed in case the node has installed an Application snapshot during statesync.
If that is the case, at the time blocksync starts, the block store only has the data statesync has saved:
light blocks, and no extended commits.
Hence we need to blocksync at least one block from another node, which will be sent with its corresponding extended commit, before we can switch to consensus.
As a side note, a chain might be started at a height *h<sub>i</sub> > 0*, all other heights
*h < h<sub>i</sub>* being non-existent. In this case, the chain is still considered to be at height 0 before
block *h<sub>i</sub>* is applied, so the first condition above allows the node to switch to consensus even
if blocksync has not processed any block (which is always the case if all nodes are starting from scratch).
When a validator falls behind while having already switched to the consensus reactor, a peer node can
simply retrieve the extended commit for the required height from the block store and reconstruct a set of
precommit votes together with their extensions and send them in the form of precommit messages to the
validator falling behind, regardless of whether the peer node holds the extended commit because it
actually participated in that consensus and thus received the precommit messages, or it received the extended commit via a `BlockResponse` message while running blocksync.
This solution requires a few changes to the consensus reactor:
- upon saving the block for a given height in the block store at decision time, save the
corresponding extended commit as well
- in the catch-up mechanism, when a node realizes that another peer is more than 2 heights
behind, it uses the extended commit (rather than the canoncial commit as done previously) to
reconstruct the precommit votes with their corresponding extensions
The changes to the blocksync reactor are more substantial:
- the `BlockResponse` message is extended to include the extended commit of the same height as
the block included in the response (just as they are stored in the block store)
- structure `bpRequester` is likewise extended to hold the received extended commits coming in
`BlockResponse` messages
- method `PeekTwoBlocks` is modified to also return the extended commit corresponding to the first block
- when successfully verifying a received block, the reactor saves its corresponding extended commit in
the block store
The two main drawbacks of this base implementation are:
- the increased size taken by the block store, in particular with big extensions
- the increased bandwith taken by the new format of `BlockResponse`
#### Possible Optimization: Pruning the Extended Commit History
If we cannot switch from the consensus reactor back to the blocksync reactor we cannot prune the extended commit backlog in the block store without sacrificing the implementation's correctness. The asynchronous
nature of our distributed system model allows a process to fall behing an arbitrary number of
heights, and thus all extended commits need to be kept *just in case* a node that late had
previously switched to the consensus reactor.
However, there is a possibility to optimize the base implementation. Every time we enter a new height,
we could prune from the block store all extended commits that are more than *d* heights in the past.
Then, we need to handle two new situations, roughly equivalent to cases (h.1) and (h.2) described above.
- (h.1) A node starts from scratch or recovers after a crash. In thisy case, we need to modify the
blocksync reactor's base implementation.
- when receiving a `BlockResponse` message, it MUST accept that the extended commit set to `nil`,
- when sending a `BlockResponse` message, if the block store contains the extended commit for that
height, it MUST set it in the message, otherwise it sets it to `nil`,
- the exit conditions used for the base implementation are no longer valid; the only reliable exit
condition now consists in making sure that the last block processed by blocksync was received with
the corresponding commit, and not `nil`; this extended commit will allow the node to switch from
the blocksync reactor to the consensus reactor and immediately act as a proposer if required.
- (h.2) A node already running the consensus reactor falls behind beyond *d* heights. In principle,
the node will be stuck forever as no other node can provide the vote extensions it needs to make
progress (they all have pruned the corresponding extended commit).
However we can manually have the node crash and recover as a workaround. This effectively converts
this case into (h.1).
### Formalization Work
A formalization work to show or prove the correctness of the different use cases and solutions
presented here (and any other that may be found) needs to be carried out.
A question that needs a precise answer is how many extended commits (one?, two?) a node needs
to keep in persistent memory when implementing Solution 3 described above without Tendermint's
current limitations.
Another important invariant we need to prove formally is that the set of vote extensions
required to make progress will always be held somewhere in the network.
## References
- [ABCI++ specification](https://github.com/tendermint/tendermint/blob/4743a7ad0/spec/abci%2B%2B/README.md)
- [ABCI as of v0.35](https://github.com/tendermint/spec/blob/4fb99af/spec/abci/README.md)
- [Vote extensions issue](https://github.com/tendermint/tendermint/issues/8174)
- [Light client verification](https://github.com/tendermint/tendermint/blob/4743a7ad0/spec/light-client/README.md)
docs/rfc/rfc-018-bls-agg-exploration.md
+555
View File
@@ -0,0 +1,555 @@
# RFC 018: BLS Signature Aggregation Exploration
## Changelog
- 01-April-2022: Initial draft (@williambanfield).
- 15-April-2022: Draft complete (@williambanfield).
## Abstract
## Background
### Glossary
The terms that are attached to these types of cryptographic signing systems
become confusing quickly. Different sources appear to use slightly different
meanings of each term and this can certainly add to the confusion. Below is
a brief glossary that may be helpful in understanding the discussion that follows.
* **Short Signature**: A signature that does not vary in length with the
number of signers.
* **Multi-Signature**: A signature generated over a single message
where, given the message and signature, a verifier is able to determine that
all parties signed the message. May be short or may vary with the number of signers.
* **Aggregated Signature**: A _short_ signature generated over messages with
possibly different content where, given the messages and signature, a verifier
should be able to determine that all the parties signed the designated messages.
* **Threshold Signature**: A _short_ signature generated from multiple signers
where, given a message and the signature, a verifier is able to determine that
a large enough share of the parties signed the message. The identities of the
parties that contributed to the signature are not revealed.
* **BLS Signature**: An elliptic-curve pairing-based signature system that
has some nice properties for short multi-signatures. May stand for
*Boneh-Lynn-Schacham* or *Barreto-Lynn-Scott* depending on the context. A
BLS signature is type of signature scheme that is distinct from other forms
of elliptic-curve signatures such as ECDSA and EdDSA.
* **Interactive**: Cryptographic scheme where parties need to perform one or
more request-response cycles to produce the cryptographic material. For
example, an interactive signature scheme may require the signer and the
verifier to cooperate to create and/or verify the signature, rather than a
signature being created ahead of time.
* **Non-interactive**: Cryptographic scheme where parties do not need to
perform any request-response cycles to produce the cryptographic material.
### Brief notes on pairing-based elliptic-curve cryptography
Pairing-based elliptic-curve cryptography is quite complex and relies on several
types of high-level math. Cryptography, in general, relies on being able to find
problems with an asymmetry between the difficulty of calculating the solution
and verifying that a given solution is correct.
Pairing-based cryptography works by operating on mathematical functions that
satisfy the property of **bilinear mapping**. This property is satisfied for
functions `e` with values `P`, `Q`, `R` and `S` where `e(P, Q + R) = e(P, Q) * e(P, R)`
and `e(P + S, Q) = e(P, Q) * e(S, Q)`. The most familiar example of this is
exponentiation. Written in common notation, `g^P*(Q+R) = g^(P*Q) * g^(P*R)` for
some value `g`.
Pairing-based elliptic-curve cryptography creates a bilinear mapping using
elliptic curves over a finite field. With some original curve, you can define two groups,
`G1` and `G2` which are points of the original curve _modulo_ different values.
Finally, you define a third group `Gt`, where points from `G1` and `G2` satisfy
the property of bilinearity with `Gt`. In this scheme, the function `e` takes
as inputs points in `G1` and `G2` and outputs values in `Gt`. Succintly, given
some point `P` in `G1` and some point `Q` in `G1`, `e(P, Q) = C` where `C` is in `Gt`.
You can efficiently compute the mapping of points in `G1` and `G2` into `Gt`,
but you cannot efficiently determine what points were summed and paired to
produce the value in `Gt`.
Functions are then defined to map digital signatures, messages, and keys into
and out of points of `G1` or `G2` and signature verification is the process
of calculating if a set of values representing a message, public key, and digital
signature produce the same value in `Gt` through `e`.
Signatures can be created as either points in `G1` with public keys being
created as points in `G2` or vice versa. For the case of BLS12-381, the popular
curve used, points in `G1` are represented with 48 bytes and points in `G2` are
represented with 96 bytes. It is up to the implementer of the cryptosystem to
decide which should be larger, the public keys or the signatures.
BLS signatures rely on pairing-based elliptic-curve cryptography to produce
various types of signatures. For a more in-depth but still high level discussion
pairing-based elliptic-curve cryptography, see Vitalik Buterin's post on
[Exploring Elliptic Curve Pairings][vitalik-pairing-post]. For much more in
depth discussion, see the specific paper on BLS12-381, [Short signatures from
the Weil Pairing][bls-weil-pairing] and
[Compact Multi-Signatures for Smaller Blockchains][multi-signatures-smaller-blockchains].
### Adoption
BLS signatures have already gained traction within several popular projects.
* Algorand is working on an implementation.
* [Zcash][zcash-adoption] has adopted BLS12-381 into the protocol.
* [Ethereum 2.0][eth-2-adoption] has adopted BLS12-381 into the protocol.
* [Chia Network][chia-adoption] has adopted BLS for signing blocks.
* [Ostracon][line-ostracon-pr], a fork of Tendermint has adopted BLS for signing blocks.
### What systems may be affected by adding aggregated signatures?
#### Gossip
Gossip could be updated to aggregate vote signatures during a consensus round.
This appears to be of frankly little utility. Creating an aggregated signature
incurs overhead, so frequently re-aggregating may incur a significant
overhead. How costly this is is still subject to further investigation and
performance testing.
Even if vote signatures were aggregated before gossip, each validator would still
need to receive and verify vote extension data from each (individual) peer validator in
order for consensus to proceed. That displaces any advantage gained by aggregating signatures across the vote message in the presence of vote extensions.
#### Block Creation
When creating a block, the proposer may create a small set of short
multi-signatures and attach these to the block instead of including one
signature per validator.
#### Block Verification
Currently, we verify each validator signature using the public key associated
with that validator. With signature aggregation, verification of blocks would
not verify many signatures individually, but would instead check the (single)
multi-signature using the public keys stored by the validator. This would also
require a mechanism for indicating which validators are included in the
aggregated signature.
#### IBC Relaying
IBC would no longer need to transmit a large set of signatures when
updating state. These state updates do not happen for every IBC packet, only
when changing an IBC light client's view of the counterparty chain's state.
General [IBC packets][ibc-packet] only contain enough information to correctly
route the data to the counterparty chain.
IBC does persist commit signatures to the chain in these `MsgUpdateClient`
message when updating state. This message would no longer need the full set
of unique signatures and would instead only need one signature for all of the
data in the header.
Adding BLS signatures would create a new signature type that must be
understood by the IBC module and by the relayers. For some operations, such
as state updates, the set of data written into the chain and received by the
IBC module could be slightly smaller.
## Discussion
### What are the proposed benefits to aggregated signatures?
#### Reduce Block Size
At the moment, a commit contains a 64-byte (512-bit) signature for each validator
that voted for the block. For the Cosmos Hub, which has 175 validators in the
active set, this amounts to about 11 KiB per block. That gives an upper bound of
around 113 GiB over the lifetime of the chain's 10.12M blocks. (Note, the Hub has
increased the number of validators in the active set over time so the total
signature size over the history of the chain is likely somewhat less than that).
Signature aggregation would only produce two signatures for the entire block.
One for the yeas and one for the nays. Each BLS aggregated signature is 48
bytes, per the [IETF standard of BLS signatures][bls-ietf-ecdsa-compare].
Over the lifetime of the same Cosmos Hub chain, that would amount to about 1
GB, a savings of 112 GB. While that is a large factor of reduction it's worth
bearing in mind that, at [GCP's cost][gcp-storage-pricing] of $.026 USD per GB,
that is a total savings of around $2.50 per month.
#### Reduce Signature Creation and Verification Time
From the [IETF draft standard on BLS Signatures][bls-ietf], BLS signatures can be
created in 370 microseconds and verified in 2700 microseconds. Our current
[Ed25519 implementation][voi-ed25519-perf] was benchmarked locally to take
13.9 microseconds to produce a signature and 2.03 milliseconds to batch verify
128 signatures, which is slightly fewer than the 175 in the Hub. blst, a popular
implementation of BLS signature aggregation was benchmarked to perform verification
on 100 signatures in 1.5 milliseconds [when run locally][blst-verify-bench]
on an 8 thread machine and pre-aggregated public keys. It is worth noting that
the `ed25519` library verification time grew steadily with the number of signatures,
whereas the bls library verification time remains constant. This is because the
number of operations used to verify a signature does not grow at all with the
number of signatures included in the aggregate signature (as long as the signers
signed over the same message data as is the case in Tendermint).
It is worth noting that this would also represent a _degredation_ in signature
verification time for chains with small validator sets. When batch verifying
only 32 signatures, our ed25519 library takes .57 milliseconds, whereas BLS
would still require the same 1.5 milliseconds.
For massive validator sets, blst dominates, taking the same 1.5 milliseconds to
check an aggregated signature from 1024 validators versus our ed25519 library's
13.066 milliseconds to batch verify a set of that size.
#### Reduce Light-Client Verification Time
The light client aims to be a faster and lighter-weight way to verify that a
block was voted on by a Tendermint network. The light client fetches
Tendermint block headers and commit signatures, performing public key
verification to ensure that the associated validator set signed the block.
Reducing the size of the commit signature would allow the light client to fetch
block data more quickly.
Additionally, the faster signature verification times of BLS signatures mean
that light client verification would proceed more quickly.
However, verification of an aggregated signature is all-or-nothing. The verifier
cannot check that some singular signer had a signature included in the block.
Instead, the verifier must use all public keys to check if some signature
was included. This does mean that any light client implementation must always
be able to fetch all public keys for any height instead of potentially being
able to check if some singular validator's key signed the block.
#### Reduce Gossip Bandwidth
##### Vote Gossip
It is possible to aggregate subsets of signatures during voting, so that the
network need not gossip all *n* validator signatures to all *n* validators.
Theoretically, subsets of the signatures could be aggregated during consensus
and vote messages could carry those aggregated signatures. Implementing this
would certainly increase the complexity of the gossip layer but could possibly
reduce the total number of signatures required to be verified by each validator.
##### Block Gossip
A reduction in the block size as a result of signature aggregation would
naturally lead to a reduction in the bandwidth required to gossip a block.
Each validator would only send and receive the smaller aggregated signatures
instead of the full list of multi-signatures as we have them now.
### What are the drawbacks to aggregated signatures?
#### Heterogeneous key types cannot be aggregated
Aggregation requires a specific signature algorithm, and our legacy signing schemes
cannot be aggregated. In practice, this means that aggregated signatures could
be created for a subset of validators using BLS signatures, and validators
with other key types (such as Ed25519) would still have to be be separately
propagated in blocks and votes.
#### Many HSMs do not support aggregated signatures
**Hardware Signing Modules** (HSM) are a popular way to manage private keys.
They provide additional security for key management and should be used when
possible for storing highly sensitive private key material.
Below is a list of popular HSMs along with their support for BLS signatures.
* YubiKey
* [No support][yubi-key-bls-support]
* Amazon Cloud HSM
* [No support][cloud-hsm-support]
* Ledger
* [Lists support for the BLS12-381 curve][ledger-bls-announce]
I cannot find support listed for Google Cloud, although perhaps it exists.
## Feasibility of implementation
This section outlines the various hurdles that would exist to implementing BLS
signature aggregation into Tendermint. It aims to demonstrate that we _could_
implement BLS signatures but that it would incur risk and require breaking changes for a
reasonably unclear benefit.
### Can aggregated signatures be added as soft-upgrades?
In my estimation, yes. With the implementation of proposer-based timestamps,
all validators now produce signatures on only one of two messages:
1. A [CanonicalVote][canonical-vote-proto] where the BlockID is the hash of the block or
2. A `CanonicalVote` where the `BlockID` is nil.
The block structure can be updated to perform hashing and validation in a new
way as a soft upgrade. This would look like adding a new section to the [Block.Commit][commit-proto] structure
alongside the current `Commit.Signatures` field. This new field, tentatively named
`AggregatedSignature` would contain the following structure:
```proto
message AggregatedSignature {
// yeas is a BitArray representing which validators in the active validator
// set issued a 'yea' vote for the block.
tendermint.libs.bits.BitArray yeas = 1;
// absent is a BitArray representing which validators in the active
// validator set did not issue votes for the block.
tendermint.libs.bits.BitArray absent = 2;
// yea_signature is an aggregated signature produced from all of the vote
// signatures for the block.
repeated bytes yea_signature = 3;
// nay_signature is an aggregated signature produced from all of the vote
// signatures from votes for 'nil' for this block.
// nay_signature should be made from all of the validators that were both not
// in the 'yeas' BitArray and not in the 'absent' BitArray.
repeated bytes nay_signature = 4;
}
```
Adding this new field as a soft upgrade would mean hashing this data structure
into the blockID along with the old `Commit.Signatures` when both are present
as well as ensuring that the voting power represented in the new
`AggregatedSignature` and `Signatures` field was enough to commit the block
during block validation. One can certainly imagine other possible schemes for
implementing this but the above should serve as a simple enough proof of concept.
### Implementing vote-time and commit-time signature aggregation separately
Implementing aggregated BLS signatures as part of the block structure can easily be
achieved without implementing any 'vote-time' signature aggregation.
The block proposer would gather all of the votes, complete with signatures,
as it does now, and produce a set of aggregate signatures from all of the
individual vote signatures.
Implementing 'vote-time' signature aggregation cannot be achieved without
also implementing commit-time signature aggregation. This is because such
signatures cannot be dis-aggregated into their constituent pieces. Therefore,
in order to implement 'vote-time' signature aggregation, we would need to
either first implement 'commit-time' signature aggregation, or implement both
'vote-time' signature aggregation while also updating the block creation and
verification protocols to allow for aggregated signatures.
### Updating IBC clients
In order for IBC clients to function, they must be able to perform light-client
verification of blocks on counterparty chains. Because BLS signatures are not
currently part of light-clients, chains that transmit messages over IBC
cannot update to using BLS signatures without their counterparties first
being upgraded to parse and verify BLS. If chains upgrade without their
counterparties first updating, they will lose the ability to interoperate with
non-updated chains.
### New attack surfaces
BLS signatures and signature aggregation comes with a new set of attack surfaces.
Additionally, it's not clear that all possible major attacks are currently known
on the BLS aggregation schemes since new ones have been discovered since the ietf
draft standard was written. The known attacks are manageable and are listed below.
Our implementation would need to prevent against these but this does not appear
to present a significant hurdle to implementation.
#### Rogue key attack prevention
Generating an aggregated signature requires guarding against what is called
a [rogue key attack][bls-ietf-terms]. A rogue key attack is one in which a
malicious actor can craft an _aggregate_ key that can produce signatures that
appear to include a signature from a private key that the malicious actor
does not actually know. In Tendermint terms, this would look like a Validator
producing a vote signed by both itself and some other validator where the other
validator did not actually produce the vote itself.
The main mechanisms for preventing this require that each entity prove that it
can can sign data with just their private key. The options involve either
ensuring that each entity sign a _different_ message when producing every
signature _or_ producing a [proof of possession][bls-ietf-pop] (PoP) when announcing
their key to the network.
A PoP is a message that demonstrates ownership of a private
key. A simple scheme for PoP is one where the entity announcing
its new public key to the network includes a digital signature over the bytes
of the public key generated using the associated private key. Everyone receiving
the public key and associated proof-of-possession can easily verify the
signature and be sure the entity owns the private key.
This PoP scheme suits the Tendermint use case quite well since
validator keys change infrequently so the associated PoPs would not be onerous
to produce, verify, and store. Using this scheme allows signature verification
to proceed more quickly, since all signatures are over identical data and
can therefore be checked using an aggregated public key instead of one at a
time, public key by public key.
#### Summing Zero Attacks
[Summing zero attacks][summing-zero-paper] are attacks that rely on using the '0' point of an
elliptic curve. For BLS signatures, if the point 0 is chosen as the private
key, then the 0 point will also always be the public key and all signatures
produced by the key will also be the 0 point. This is easy enough to
detect when verifying each signature individually.
However, because BLS signature aggregation creates an aggregated signature and
an aggregated public key, a set of colluding signers can create a pair or set
of signatures that are non-zero but which aggregate ("sum") to 0. The signatures that sum zero along with the
summed public key of the colluding signers will verify any message. This would
allow the colluding signers to sign any block or message with the same signature.
This would be reasonably easy to detect and create evidence for because, in
all other cases, the same signature should not verify more than message. It's
not exactly clear how such an attack would advantage the colluding validators
because the normal mechanisms of evidence gathering would still detect the
double signing, regardless of the signatures on both blocks being identical.
### Backwards Compatibility
Backwards compatibility is an important consideration for signature verification.
Specifically, it is important to consider whether chains using current versions
of IBC would be able to interact with chains adopting BLS.
Because the `Block` shared by IBC and Tendermint is produced and parsed using
protobuf, new structures can be added to the Block without breaking the
ability of legacy users to parse the new structure. Breaking changes between
current users of IBC and new Tendermint blocks only occur if data that is
relied upon by the current users is no longer included in the current fields.
For the case of BLS aggregated signatures, a new `AggregatedSignature` field
can therefore be added to the `Commit` field without breaking current users.
Current users will be broken when counterparty chains upgrade to the new version
and _begin using_ BLS signatures. Once counterparty chains begin using BLS
signatures, the BlockID hashes will include hashes of the `AggregatedSignature`
data structure that the legacy users will not be able to compute. Additionally,
the legacy software will not be able to parse and verify the signatures to
ensure that a supermajority of validators from the counterparty chain signed
the block.
### Library Support
Libraries for BLS signature creation are limited in number, although active
development appears to be ongoing. Cryptographic algorithms are difficult to
implement correctly and correctness issues are extremely serious and dangerous.
No further exploration of BLS should be undertaken without strong assurance of
a well-tested library with continuing support for creating and verifying BLS
signatures.
At the moment, there is one candidate, `blst`, that appears to be the most
mature and well vetted. While this library is undergoing continuing auditing
and is supported by funds from the Ethereum foundation, adopting a new cryptographic
library presents some serious risks. Namely, if the support for the library were
to be discontinued, Tendermint may become saddled with the requirement of supporting
a very complex piece of software or force a massive ecosystem-wide migration away
from BLS signatures.
This is one of the more serious reasons to avoid adopting BLS signatures at this
time. There is no gold standard library. Some projects look promising, but no
project has been formally verified with a long term promise of being supported
well into the future.
#### Go Standard Library
The Go Standard library has no implementation of BLS signatures.
#### BLST
[blst][blst], or 'blast' is an implementation of BLS signatures written in C
that provides bindings into Go as part of the repository. This library is
actively undergoing formal verification by Galois and previously received an
initial audit by NCC group, a firm I'd never heard of.
`blst` is [targeted for use in prysm][prysm-blst], the golang implementation of Ethereum 2.0.
#### Gnark-Crypto
[Gnark-Crypto][gnark] is a Go-native implementation of elliptic-curve pairing-based
cryptography. It is not audited and is documented as 'as-is', although
development appears to be active so formal verification may be forthcoming.
#### CIRCL
[CIRCL][circl] is a go-native implementation of several cryptographic primitives,
bls12-381 among them. The library is written and maintained by Cloudflare and
appears to receive frequent contributions. However, it lists itself as experimental
and urges users to take caution before using it in production.
### Added complexity to light client verification
Implementing BLS signature aggregation in Tendermint would pose issues for the
light client. The light client currently validates a subset of the signatures
on a block when performing the verification algorithm. This is no longer possible
with an aggregated signature. Aggregated signature verification is all-or-nothing.
The light client could no longer check that a subset of validators from some
set of validators is represented in the signature. Instead, it would need to create
a new aggregated key with all the stated signers for each height it verified where
the validator set changed.
This means that the speed advantages gained by using BLS cannot be fully realized
by the light client since the client needs to perform the expensive operation
of re-aggregating the public key. Aggregation is _not_ constant time in the
number of keys and instead grows linearly. When [benchmarked locally][blst-verify-bench-agg],
blst public key aggregation of 128 keys took 2.43 milliseconds. This, along with
the 1.5 milliseconds to verify a signature would raise light client signature
verification time to 3.9 milliseconds, a time above the previously mentioned
batch verification time using our ed25519 library of 2.0 milliseconds.
Schemes to cache aggregated subsets of keys could certainly cut this time down at the
cost of adding complexity to the light client.
### Added complexity to evidence handling
Implementing BLS signature aggregation in Tendermint would add complexity to
the evidence handling within Tendermint. Currently, the light client can submit
evidence of a fork attempt to the chain. This evidence consists of the set of
validators that double-signed, including their public keys, with the conflicting
block.
We can quickly check that the listed validators double signed by verifying
that each of their signatures are in the submitted conflicting block. A BLS
signature scheme would change this by requiring the light client to submit
the public keys of all of the validators that signed the conflicting block so
that the aggregated signature may be checked against the full signature set.
Again, aggregated signature verification is all-or-nothing, so without all of
the public keys, we cannot verify the signature at all. These keys would be
retrievable. Any party that wanted to create a fork would want to convince a
network that its fork is legitimate, so it would need to gossip the public keys.
This does not hamper the feasibility of implementing BLS signature aggregation
into Tendermint, but does represent yet another piece of added complexity to
the associated protocols.
## Open Questions
* *Q*: Can you aggregate Ed25519 signatures in Tendermint?
* There is a suggested scheme in github issue [7892][suggested-ed25519-agg],
but additional rigor would be required to fully verify its correctness.
## Current Consideration
Adopting a signature aggregation scheme presents some serious risks and costs
to the Tendermint project. It requires multiple backwards-incompatible changes
to the code, namely a change in the structure of the block and a new backwards-incompatible
signature and key type. It risks adding a new signature type for which new attack
types are still being discovered _and_ for which no industry standard, battle-tested
library yet exists.
The gains boasted by this new signing scheme are modest: Verification time is
marginally faster and block sizes shrink by a few kilobytes. These are relatively
minor gains in exchange for the complexity of the change and the listed risks of the technology.
We should take a wait-and-see approach to BLS signature aggregation, monitoring
the up-and-coming projects and consider implementing it as the libraries and
standards develop.
### References
[line-ostracon-repo]: https://github.com/line/ostracon
[line-ostracon-pr]: https://github.com/line/ostracon/pull/117
[mit-BLS-lecture]: https://youtu.be/BFwc2XA8rSk?t=2521
[gcp-storage-pricing]: https://cloud.google.com/storage/pricing#north-america_2
[yubi-key-bls-support]: https://github.com/Yubico/yubihsm-shell/issues/66
[cloud-hsm-support]: https://docs.aws.amazon.com/cloudhsm/latest/userguide/pkcs11-key-types.html
[bls-ietf]: https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04
[bls-ietf-terms]: https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04#section-1.3
[bls-ietf-pop]: https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04#section-3.3
[multi-signatures-smaller-blockchains]: https://eprint.iacr.org/2018/483.pdf
[ibc-tendermint]: https://github.com/cosmos/ibc/tree/master/spec/client/ics-007-tendermint-client
[zcash-adoption]: https://github.com/zcash/zcash/issues/2502
[chia-adoption]: https://github.com/Chia-Network/chia-blockchain#chia-blockchain
[bls-ietf-ecdsa-compare]: https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04#section-1.1
[voi-ed25519-perf]: https://github.com/williambanfield/curve25519-voi/blob/benchmark/primitives/ed25519/PERFORMANCE.txt#L79
[blst-verify-bench]: https://github.com/williambanfield/blst/blame/bench/bindings/go/PERFORMANCE.md#L9
[blst-verify-bench-agg]: https://github.com/williambanfield/blst/blame/bench/bindings/go/PERFORMANCE.md#L23
[vitalik-pairing-post]: https://medium.com/@VitalikButerin/exploring-elliptic-curve-pairings-c73c1864e627
[ledger-bls-announce]: https://www.ledger.com/first-ever-firmware-update-coming-to-the-ledger-nano-x
[commit-proto]: https://github.com/tendermint/tendermint/blob/be7cb50bb3432ee652f88a443e8ee7b8ef7122bc/proto/tendermint/types/types.proto#L121
[canonical-vote-proto]: https://github.com/tendermint/tendermint/blob/be7cb50bb3432ee652f88a443e8ee7b8ef7122bc/spec/core/encoding.md#L283
[blst]: https://github.com/supranational/blst
[prysm-blst]: https://github.com/prysmaticlabs/prysm/blob/develop/go.mod#L75
[gnark]: https://github.com/ConsenSys/gnark-crypto/
[eth-2-adoption]: https://notes.ethereum.org/@GW1ZUbNKR5iRjjKYx6_dJQ/Skxf3tNcg_
[bls-weil-pairing]: https://www.iacr.org/archive/asiacrypt2001/22480516.pdf
[summing-zero-paper]: https://eprint.iacr.org/2021/323.pdf
[circl]: https://github.com/cloudflare/circl
[light-client-evidence]: https://github.com/tendermint/tendermint/blob/a6fd1fe20116d4b1f7e819cded81cece8e5c1ac7/types/evidence.go#L245
[suggested-ed25519-agg]: https://github.com/tendermint/tendermint/issues/7892
docs/rfc/rfc-020-onboarding-projects.rst
+240
View File
@@ -0,0 +1,240 @@
=======================================
RFC 020: Tendermint Onboarding Projects
=======================================
.. contents::
:backlinks: none
Changelog
---------
- 2022-03-30: Initial draft. (@tychoish)
- 2022-04-25: Imported document to tendermint repository. (@tychoish)
Overview
--------
This document describes a collection of projects that might be good for new
engineers joining the Tendermint Core team. These projects mostly describe
features that we'd be very excited to see land in the code base, but that are
intentionally outside of the critical path of a release on the roadmap, and
have the following properties that we think make good on-boarding projects:
- require relatively little context for the project or its history beyond a
more isolated area of the code.
- provide exposure to different areas of the codebase, so new team members
will have reason to explore the code base, build relationships with people
on the team, and gain experience with more than one area of the system.
- be of moderate size, striking a healthy balance between trivial or
mechanical changes (which provide little insight) and large intractable
changes that require deeper insight than is available during onboarding to
address well. A good size project should have natural touchpoints or
check-ins.
Projects
--------
Before diving into one of these projects, have a conversation about the
project or aspects of Tendermint that you're excited to work on with your
onboarding buddy. This will help make sure that these issues are still
relevant, help you get any context, underatnding known pitfalls, and to
confirm a high level approach or design (if relevant.) On-boarding buddies
should be prepared to do some design work before someone joins the team.
The descriptions that follow provide some basic background and attempt to
describe the user stories and the potential impact of these project.
E2E Test Systems
~~~~~~~~~~~~~~~~
Tendermint's E2E framework makes it possible to run small test networks with
different Tendermint configurations, and make sure that the system works. The
tests run Tendermint in a separate binary, and the system provides some very
high level protection against making changes that could break Tendermint in
otherwise difficult to detect ways.
Working on the E2E system is a good place to get introduced to the Tendermint
codebase, particularly for developers who are newer to Go, as the E2E
system (generator, runner, etc.) is distinct from the rest of Tendermint and
comparatively quite small, so it may be easier to begin making changes in this
area. At the same time, because the E2E system exercises *all* of Tendermint,
work in this area is a good way to get introduced to various components of the
system.
Configurable E2E Workloads
++++++++++++++++++++++++++
All E2E tests use the same workload (e.g. generated transactions, submitted to
different nodes in the network,) which has been tuned empirically to provide a
gentle but consistent parallel load that all E2E tests can pass. Ideally, the
workload generator could be configurable to have different shapes of work
(bursty, different transaction sizes, weighted to different nodes, etc.) and
even perhaps further parameterized within a basic shape, which would make it
possible to use our existing test infrastructure to answer different questions
about the performance or capability of the system.
The work would involve adding a new parameter to the E2E test manifest, and
creating an option (e.g. "legacy") for the current load generation model,
extract configurations options for the current load generation, and then
prototype implementations of alternate load generation, and also run some
preliminary using the tools.
Byzantine E2E Workloads
+++++++++++++++++++++++
There are two main kinds of integration tests in Tendermint: the E2E test
framework, and then a collection of integration tests that masquerade as
unit-tests. While some of this expansion of test scope is (potentially)
inevitable, the masquerading unit tests (e.g ``consensus.byzantine_test.go``)
end up being difficult to understand, difficult to maintain, and unreliable.
One solution to this, would be to modify the E2E ABCI application to allow it
to inject byzantine behavior, and then have this be a configurable aspect of
a test network to be able to provoke Byzantine behavior in a "real" system and
then observe that evidence is constructed. This would make it possible to
remove the legacy tests entirely once the new tests have proven themselves.
Abstract Orchestration Framework
++++++++++++++++++++++++++++++++
The orchestration of e2e test processes is presently done using docker
compose, which works well, but has proven a bit limiting as all processes need
to run on a single machine, and the log aggregation functions are confusing at
best.
This project would replace the current orchestration with something more
generic, potentially maintaining the current system, but also allowing the e2e
tests to manage processes using k8s. There are a few "local" k8s frameworks
(e.g. kind and k3s,) which might be able to be useful for our current testing
model, but hopefully, we could use this new implementation with other k8s
systems for more flexible distribute test orchestration.
Improve Operationalize Experience of ``run-multiple.sh``
++++++++++++++++++++++++++++++++++++++++++++++++++++++++
The e2e test runner currently runs a single test, and in most cases we manage
the test cases using a shell script that ensure cleanup of entire test
suites. This is a bit difficult to maintain and makes reproduction of test
cases more awkward than it should be. The e2e ``runner`` itself should provide
equivalent functionality to ``run-multiple.sh``: ensure cleanup of test cases,
collect and process output, and be able to manage entire suites of cases.
It might also be useful to implement an e2e test orchestrator that runs all
tendermint instances in a single process, using "real" networks for faster
feedback and iteration during development.
In addition to being a bit easier to maintain, having a more capable runner
implementation would make it easier to collect data from test runs, improve
debugability and reporting.
Fan-Out For CI E2E Tests
++++++++++++++++++++++++
While there are some parallelism in the execution of e2e tests, each e2e test
job must build a tendermint e2e image, which takes about 5 minutes of CPU time
per-task, which given the size of each of the runs.
We'd like to be able to reduce the amount of overhead per-e2e tests while
keeping the cycle time for working with the tests very low, while also
maintaining a reasonable level of test coverage. This is an impossible
tradeoff, in some ways, and the percentage of overhead at the moment is large
enough that we can make some material progress with a moderate amount of time.
Most of this work has to do with modifying github actions configuration and
e2e artifact (docker) building to reduce redundant work. Eventually, when we
can drop the requirement for CGo storage engines, it will be possible to move
(cross) compile tendermint locally, and then inject the binary into the docker
container, which would reduce a lot of the build-time complexity, although we
can move more in this direction or have runtime flags to disable CGo
dependencies for local development.
Remove Panics
~~~~~~~~~~~~~
There are lots of places in the code base which can panic, and would not be
particularly well handled. While in some cases, panics are the right answer,
in many cases the panics were just added to simplify downstream error
checking, and could easily be converted to errors.
The `Don't Panic RFC
<https://github.com/tendermint/tendermint/blob/master/docs/rfc/rfc-008-do-not-panic.MD>`_
covers some of the background and approach.
While the changes are in this project are relatively rote, this will provide
exposure to lots of different areas of the codebase as well as insight into
how different areas of the codebase interact with eachother, as well as
experience with the test suites and infrastructure.
Implement more Expressive ABCI Applications
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Tendermint maintains two very simple ABCI applications (a KV application used
for basic testing, and slightly more advanced test application used in the
end-to-end tests). Writing an application would provide a new engineer with
useful experiences using Tendermint that mirrors the expierence of downstream
users.
This is more of an exploratory project, but could include providing common
interfaces on top of Tendermint consensus for other well known protocols or
tools (e.g. ``etcd``) or a DNS server or some other tool.
Self-Regulating Reactors
~~~~~~~~~~~~~~~~~~~~~~~~
Currently reactors (the internal processes that are responsible for the higher
level behavior of Tendermint) can be started and stopped, but have no
provision for being paused. These additional semantics may allow Tendermint to
pause reactors (and avoid processing their messhages, etc.) and allow better
coordination in the future.
While this is a big project, it's possible to break this apart into many
smaller projects: make p2p channels pauseable, add pause/UN-pause hooks to the
service implementation and machinery, and finally to modify the reactor
implementations to take advantage of these additional semantics
This project would give an engineer some exposure to the p2p layer of the
code, as well as to various aspects of the reactor implementations.
Metrics
~~~~~~~
Tendermint has a metrics system that is relatively underutilized, and figuring
out ways to capture and organize the metrics to provide value to users might
provide an interesting set of projects for new engineers on Tendermint.
Convert Logs to Metrics
+++++++++++++++++++++++
Because the tendermint logs tend to be quite verbose and not particularly
actionable, most users largely ignore the logging or run at very low
verbosity. While the log statements in the code do describe useful events,
taken as a whole the system is not particularly tractable, and particularly at
the Debug level, not useful. One solution to this problem is to identify log
messages that might be (e.g. increment a counter for certian kinds of errors)
One approach might be to look at various logging statements, particularly
debug statements or errors that are logged but not returned, and see if
they're convertable to counters or other metrics.
Expose Metrics to Tests
+++++++++++++++++++++++
The existing Tendermint test suites replace the metrics infrastructure with
no-op implementations, which means that tests can neither verify that metrics
are ever recorded, nor can tests use metrics to observe events in the
system. Writing an implementation, for testing, that makes it possible to
record metrics and provides an API for introspecting this data, as well as
potentially writing tests that take advantage of this type, could be useful.
Logging Metrics
+++++++++++++++
In some systems, the logging system itself can provide some interesting
insights for operators: having metrics that track the number of messages at
different levels as well as the total number of messages, can act as a canary
for the system as a whole.
This should be achievable by adding an interceptor layer within the logging
package itself that can add metrics to the existing system.
docs/tendermint-core/consensus/proposer-based-timestamps.md
+4 -3
View File
@@ -13,14 +13,15 @@ order: 3
The PBTS algorithm defines a way for a Tendermint blockchain to create block
timestamps that are within a reasonable bound of the clocks of the validators on
the network. This replaces the original BFTTime algorithm for timestamp
assignment that relied on the timestamps included in precommit messages.
assignment that computed a timestamp using the timestamps included in precommit
messages.
## Algorithm Parameters
The functionality of the PBTS algorithm is governed by two parameters within
Tendermint. These two parameters are [consensus
parameters](https://github.com/tendermint/tendermint/blob/master/spec/abci/apps.md#L291),
meaning they are configured by the ABCI application and are expected to be the
meaning they are configured by the ABCI application and are therefore the same
same across all nodes on the network.
### `Precision`
@@ -51,7 +52,7 @@ useful for the protocols and applications built on top of Tendermint.
The following protocols and application features require a reliable source of time:
* Tendermint Light Clients [rely on correspondence between their known time](https://github.com/tendermint/tendermint/blob/master/spec/light-client/verification/README.md#definitions-1) and the block time for block verification.
* Tendermint Evidence validity is determined [either in terms of heights or in terms of time](https://github.com/tendermint/tendermint/blob/master/spec/consensus/evidence.md#verification).
* Tendermint Evidence expiration is determined [either in terms of heights or in terms of time](https://github.com/tendermint/tendermint/blob/master/spec/consensus/evidence.md#verification).
* Unbonding of staked assets in the Cosmos Hub [occurs after a period of 21
days](https://github.com/cosmos/governance/blob/master/params-change/Staking.md#unbondingtime).
* IBC packets can use either a [timestamp or a height to timeout packet
docs/tendermint-core/subscription.md
+205 -51
View File
@@ -2,74 +2,228 @@
order: 7
---
# Subscribing to events via Websocket
# Subscribing to Events
Tendermint emits different events, which you can subscribe to via
[Websocket](https://en.wikipedia.org/wiki/WebSocket). This can be useful
for third-party applications (for analysis) or for inspecting state.
A Tendermint node emits events about important state transitions during
consensus. These events can be queried by clients via the [RPC interface][rpc]
on nodes that enable it. The [list of supported event types][event-types] can
be found in the tendermint/types Go package.
[List of events](https://godoc.org/github.com/tendermint/tendermint/types#pkg-constants)
In Tendermint v0.36 there are two APIs to query events:
To connect to a node via websocket from the CLI, you can use a tool such as
[wscat](https://github.com/websockets/wscat) and run:
- The [**legacy streaming API**](#legacy-streaming-api), comprising the
`subscribe`, `unsubscribe`, and `unsubscribe_all` RPC methods over websocket.
- The [**event log API**](#event-log-api), comprising the `events` RPC method.
The legacy streaming API is deprecated in Tendermint v0.36, and will be removed
in Tendermint v0.37. Clients are strongly encouraged to migrate to the new
event log API as soon as is practical.
[rpc]: https://docs.tendermint.com/master/rpc
[event-types]: https://godoc.org/github.com/tendermint/tendermint/types#EventNewBlockValue
## Filter Queries
Event requests take a [filter query][query] parameter. A filter query is a
string that describes a subset of available event items to return. An empty
query matches all events; otherwise a query comprises one or more *terms*
comparing event metadata to target values.
For example, to select new block events, use the term:
```
tm.event = 'NewBlock'
```
Multiple terms can be combined with `AND` (case matters), for example to match
the transaction event with a given hash, use the query:
```
tm.event = 'Tx' AND tx.hash = 'EA7B33F'
```
Operands may be strings in single quotes (`'Tx'`), numbers (`45`), dates, or
timestamps.
The comparison operators include `=`, `<`, `<=`, `>`, `>=`, and `CONTAINS` (for
substring match). In addition, the `EXISTS` operator checks for the presence
of an attribute regardless of its value.
### Attributes
Tendermint implicitly defines a string-valued `tm.event` attribute for all
event types. Transaction items (type `Tx`) are also assigned `tx.hash`
(string), giving the hash of the transaction, and and `tx.height` (number)
giving the height of the block containing the transaction. For `NewBlock` and
`NewBlockHeader` events, Tendermint defines a `block.height` attribute giving
the height of the block.
Additional attributes can be provided by the application as [ABCI `Event`
records][abci-event] in response to the `FinalizeBlock` request. The full name
of the attribute in the query is formed by combining the `type` and attribute
`key` with a period.
For example, given the events
```go
[]abci.Event{{
Type: "reward",
Attributes: []abci.EventAttribute{
{Key: "address", Value: "cosmos1xyz012pdq"},
{Key: "amount", Value: "45.62"},
{Key: "balance", Value: "100.390001"},
},
}}
```
a query may refer to the names `reward.address`, `reward.amount`, and `reward.balance`, as in:
```
reward.address EXISTS AND reward.balance > 45
```
Certain application-specific metadata are also indexed for offline queries.
See [Indexing transactions](../app-dev/indexing-transactions.md) for more details.
[query]: https://godoc.org/github.com/tendermint/tendermint/internal/pubsub/query/syntax
[abci-event]: https://github.com/tendermint/tendermint/blob/master/proto/tendermint/abci/types.proto#L397
## Event Log API
Starting in Tendermint v0.36, when the `rpc.event-log-window-size`
configuration is enabled, the node maintains maintains a log of all events
within this operator-defined time window. This API supersedes the websocket
subscription API described below.
Clients can query these events can by long-polling the `/events` RPC method,
which returns the most recent items from the log that match the [request
parameters][reqevents]. Each item returned includes a cursor that marks its
location in the log. Cursors can be passed via the `before` and `after`
parameters to fetch events earlier in the log.
For example, this request:
```json
{
"jsonrpc": "2.0",
"id": 1,
"method": "events",
"params": {
"filter": {
"query": "tm.event = 'Tx' AND app.key = 'applesauce'"
},
"maxItems": 1,
"after": ""
}
}
```
will return a result similar to the following:
```json
{
"jsonrpc": "2.0",
"id": 1,
"result": {
"items": [
{
"cursor": "16ee3d5e65be53d8-03d5",
"event": "Tx",
"data": {
"type": "tendermint/event/Tx",
"value": {
"height": 70,
"tx": "YXBwbGVzYXVjZT1zeXJ1cA==",
"result": {
"events": [
{
"type": "app",
"attributes": [
{
"key": "creator",
"value": "Cosmoshi Netowoko",
"index": true
},
{
"key": "key",
"value": "applesauce",
"index": true
},
{
"key": "index_key",
"value": "index is working",
"index": true
},
{
"key": "noindex_key",
"value": "index is working",
"index": false
}
]
}
]
}
}
}
}
],
"more": false,
"oldest": "16ee3d4c471c3b00-0001",
"newest": "16ee3d5f2e05a4e0-0400"
}
}
```
The `"items"` array gives the matching items (up to the requested
`"maxResults"`) in decreasing time order (i.e., newest to oldest). In this
case, there is only one result, but if there are additional results that were
not returned, the `"more"` flag will be true. Calling `/events` again with the
same query and `"after"` set to the cursor of the newest result (in this
example, `"16ee3d5e65be53d8-03d5"`) will fetch newer results.
Go clients can use the [`eventstream`][eventstream] package to simplify the use
of this method. The `eventstream.Stream` automatically handles polling for new
events, updating the cursor, and reporting any missed events.
[reqevents]: https://pkg.go.dev/github.com/tendermint/tendermint@master/rpc/coretypes#RequestEvents
[eventstream]: https://godoc.org/github.com/tendermint/tendermint/rpc/client/eventstream
## Legacy Streaming API
- **Note:** This API is deprecated in Tendermint v0.36, and will be removed in
Tendermint v0.37. New clients and existing use should use the [event log
API](#event-log-api) instead. See [ADR 075][adr075] for more details.
To subscribe to events in the streaming API, you must connect to the node RPC
service using a [websocket][ws]. From the command line you can use a tool such
as [wscat][wscat], for example:
```sh
wscat ws://127.0.0.1:26657/websocket
```
You can subscribe to any of the events above by calling the `subscribe` RPC
method via Websocket along with a valid query.
[ws]: https://en.wikipedia.org/wiki/WebSocket
[wscat]: https://github.com/websockets/wscat
To subscribe to events, call the `subscribe` JSON-RPC method method passing in
a [filter query][query] for the events you wish to receive:
```json
{
"jsonrpc": "2.0",
"method": "subscribe",
"id": 0,
"id": 1,
"params": {
"query": "tm.event='NewBlock'"
}
}
```
Check out [API docs](https://docs.tendermint.com/master/rpc/) for
more information on query syntax and other options.
The subscribe method returns an initial response confirming the subscription,
then sends additional JSON-RPC response messages containing the matching events
as they are published. The subscription continues until either the client
explicitly cancels the subscription (by calling `unsubscribe` or
`unsubscribe_all`) or until the websocket connection is terminated.
You can also use tags, given you had included them into DeliverTx
response, to query transaction results. See [Indexing
transactions](../app-dev/indexing-transactions.md) for details.
## ValidatorSetUpdates
When validator set changes, ValidatorSetUpdates event is published. The
event carries a list of pubkey/power pairs. The list is the same
Tendermint receives from ABCI application (see [EndBlock
section](https://github.com/tendermint/tendermint/blob/master/spec/abci/abci.md#endblock) in
the ABCI spec).
Response:
```json
{
"jsonrpc": "2.0",
"id": 0,
"result": {
"query": "tm.event='ValidatorSetUpdates'",
"data": {
"type": "tendermint/event/ValidatorSetUpdates",
"value": {
"validator_updates": [
{
"address": "09EAD022FD25DE3A02E64B0FE9610B1417183EE4",
"pub_key": {
"type": "tendermint/PubKeyEd25519",
"value": "ww0z4WaZ0Xg+YI10w43wTWbBmM3dpVza4mmSQYsd0ck="
},
"voting_power": "10",
"proposer_priority": "0"
}
]
}
}
}
}
```
[adr075]: https://tinyurl.com/adr075
go.mod
+44 -36
View File
@@ -16,12 +16,12 @@ require (
github.com/gorilla/websocket v1.5.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/lib/pq v1.10.5
github.com/lib/pq v1.10.6
github.com/libp2p/go-buffer-pool v0.0.2
github.com/mroth/weightedrand v0.4.1
github.com/oasisprotocol/curve25519-voi v0.0.0-20210609091139-0a56a4bca00b
github.com/ory/dockertest v3.3.5+incompatible
github.com/prometheus/client_golang v1.12.1
github.com/prometheus/client_golang v1.12.2
github.com/rs/cors v1.8.2
github.com/rs/zerolog v1.26.1
github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa
@@ -32,26 +32,34 @@ require (
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
golang.org/x/net v0.0.0-20220412020605-290c469a71a5
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
google.golang.org/grpc v1.46.0
google.golang.org/grpc v1.46.2
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
pgregory.net/rapid v0.4.7
)
require (
github.com/creachadair/atomicfile v0.2.5
github.com/creachadair/atomicfile v0.2.6
github.com/creachadair/taskgroup v0.3.2
github.com/golangci/golangci-lint v1.45.2
github.com/golangci/golangci-lint v1.46.0
github.com/google/go-cmp v0.5.8
github.com/vektra/mockery/v2 v2.12.1
github.com/vektra/mockery/v2 v2.12.2
gotest.tools v2.2.0+incompatible
)
require github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect
require (
github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0 // indirect
github.com/firefart/nonamedreturns v1.0.1 // indirect
github.com/lufeee/execinquery v1.0.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.0 // indirect
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect
golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e // indirect
)
require (
4d63.com/gochecknoglobals v0.1.0 // indirect
github.com/Antonboom/errname v0.1.5 // indirect
github.com/Antonboom/nilnil v0.1.0 // indirect
github.com/Antonboom/errname v0.1.6 // indirect
github.com/Antonboom/nilnil v0.1.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/DataDog/zstd v1.4.1 // indirect
github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
@@ -64,18 +72,18 @@ require (
github.com/ashanbrown/makezero v1.1.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bkielbasa/cyclop v1.2.0 // indirect
github.com/blizzy78/varnamelen v0.6.1 // indirect
github.com/blizzy78/varnamelen v0.8.0 // indirect
github.com/bombsimon/wsl/v3 v3.3.0 // indirect
github.com/breml/bidichk v0.2.2 // indirect
github.com/breml/errchkjson v0.2.3 // indirect
github.com/breml/bidichk v0.2.3 // indirect
github.com/breml/errchkjson v0.3.0 // indirect
github.com/butuzov/ireturn v0.1.1 // indirect
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/charithe/durationcheck v0.0.9 // indirect
github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af // indirect
github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 // indirect
github.com/containerd/continuity v0.2.1 // indirect
github.com/creachadair/tomledit v0.0.19
github.com/creachadair/tomledit v0.0.22
github.com/daixiang0/gci v0.3.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/denis-tingaikin/go-header v0.4.3 // indirect
@@ -92,9 +100,9 @@ require (
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fatih/structtag v1.2.0 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/fzipp/gocyclo v0.4.0 // indirect
github.com/go-critic/go-critic v0.6.2 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/fzipp/gocyclo v0.5.1 // indirect
github.com/go-critic/go-critic v0.6.3 // indirect
github.com/go-toolsmith/astcast v1.0.0 // indirect
github.com/go-toolsmith/astcopy v1.0.0 // indirect
github.com/go-toolsmith/astequal v1.0.1 // indirect
@@ -108,7 +116,7 @@ require (
github.com/golang/snappy v0.0.3 // indirect
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 // indirect
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 // indirect
github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe // indirect
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a // indirect
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 // indirect
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca // indirect
@@ -134,10 +142,10 @@ require (
github.com/julz/importas v0.1.0 // indirect
github.com/kisielk/errcheck v1.6.0 // indirect
github.com/kisielk/gotool v1.0.0 // indirect
github.com/kulti/thelper v0.5.1 // indirect
github.com/kulti/thelper v0.6.2 // indirect
github.com/kunwardeep/paralleltest v1.0.3 // indirect
github.com/kyoh86/exportloopref v0.1.8 // indirect
github.com/ldez/gomoddirectives v0.2.2 // indirect
github.com/ldez/gomoddirectives v0.2.3 // indirect
github.com/ldez/tagliatelle v0.3.1 // indirect
github.com/leonklingele/grouper v1.1.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
@@ -148,37 +156,37 @@ require (
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mbilski/exhaustivestruct v1.2.0 // indirect
github.com/mgechev/revive v1.1.4 // indirect
github.com/mgechev/revive v1.2.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moricho/tparallel v0.2.1 // indirect
github.com/nakabonne/nestif v0.3.1 // indirect
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
github.com/nishanths/exhaustive v0.7.11 // indirect
github.com/nishanths/predeclared v0.2.1 // indirect
github.com/nishanths/predeclared v0.2.2 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runc v1.0.3 // indirect
github.com/pelletier/go-toml v1.9.4 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polyfloyd/go-errorlint v0.0.0-20211125173453-6d6d39c5bb8b // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.34.0
github.com/prometheus/procfs v0.7.3 // indirect
github.com/quasilyte/go-ruleguard v0.3.15 // indirect
github.com/quasilyte/gogrep v0.0.0-20220103110004-ffaa07af02e3 // indirect
github.com/quasilyte/go-ruleguard v0.3.16-0.20220213074421-6aa060fab41a // indirect
github.com/quasilyte/gogrep v0.0.0-20220120141003-628d8b3623b5 // indirect
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 // indirect
github.com/ryancurrah/gomodguard v1.2.3 // indirect
github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect
github.com/sanposhiho/wastedassign/v2 v2.0.6 // indirect
github.com/securego/gosec/v2 v2.10.0 // indirect
github.com/securego/gosec/v2 v2.11.0 // indirect
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/sivchari/containedctx v1.0.2 // indirect
github.com/sivchari/tenv v1.4.7 // indirect
github.com/sivchari/tenv v1.5.0 // indirect
github.com/sonatard/noctx v0.0.1 // indirect
github.com/sourcegraph/go-diff v0.6.1 // indirect
github.com/spf13/afero v1.8.2 // indirect
@@ -194,28 +202,28 @@ require (
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
github.com/tetafro/godot v1.4.11 // indirect
github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 // indirect
github.com/tomarrell/wrapcheck/v2 v2.5.0 // indirect
github.com/tomarrell/wrapcheck/v2 v2.6.1 // indirect
github.com/tommy-muehle/go-mnd/v2 v2.5.0 // indirect
github.com/ultraware/funlen v0.0.3 // indirect
github.com/ultraware/whitespace v0.0.5 // indirect
github.com/uudashr/gocognit v1.0.5 // indirect
github.com/yagipy/maintidx v1.0.0 // indirect
github.com/yeya24/promlinter v0.1.1-0.20210918184747-d757024714a1 // indirect
github.com/yeya24/promlinter v0.2.0 // indirect
gitlab.com/bosi/decorder v0.2.1 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.10 // indirect
golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
google.golang.org/protobuf v1.28.0 // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
honnef.co/go/tools v0.2.2 // indirect
mvdan.cc/gofumpt v0.3.0 // indirect
honnef.co/go/tools v0.3.1 // indirect
mvdan.cc/gofumpt v0.3.1 // indirect
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 // indirect
go.sum
+98 -79
View File
@@ -46,7 +46,6 @@ cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJW
cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/firestore v1.6.0/go.mod h1:afJwI0vaXwAG54kI7A//lP/lSPDkQORQuMkv56TxEPU=
cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
@@ -62,17 +61,17 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Antonboom/errname v0.1.5 h1:IM+A/gz0pDhKmlt5KSNTVAvfLMb+65RxavBXpRtCUEg=
github.com/Antonboom/errname v0.1.5/go.mod h1:DugbBstvPFQbv/5uLcRRzfrNqKE9tVdVCqWCLp6Cifo=
github.com/Antonboom/nilnil v0.1.0 h1:DLDavmg0a6G/F4Lt9t7Enrbgb3Oph6LnDE6YVsmTt74=
github.com/Antonboom/nilnil v0.1.0/go.mod h1:PhHLvRPSghY5Y7mX4TW+BHZQYo1A8flE5H20D3IPZBo=
github.com/Antonboom/errname v0.1.6 h1:LzIJZlyLOCSu51o3/t2n9Ck7PcoP9wdbrdaW6J8fX24=
github.com/Antonboom/errname v0.1.6/go.mod h1:7lz79JAnuoMNDAWE9MeeIr1/c/VpSUWatBv2FH9NYpI=
github.com/Antonboom/nilnil v0.1.1 h1:PHhrh5ANKFWRBh7TdYmyyq2gyT2lotnvFvvFbylF81Q=
github.com/Antonboom/nilnil v0.1.1/go.mod h1:L1jBqoWM7AOeTD+tSquifKSesRHs4ZdaxvZR+xdJEaI=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -83,6 +82,8 @@ github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM=
github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0 h1:LAPPhJ4KR5Z8aKVZF5S48csJkxL5RMKmE/98fMs1u5M=
github.com/GaijinEntertainment/go-exhaustruct/v2 v2.1.0/go.mod h1:LGOGuvEgCfCQsy3JF2tRmpGDpzA53iZfyGEWSPwQ6/4=
github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
@@ -148,14 +149,14 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/bkielbasa/cyclop v1.2.0 h1:7Jmnh0yL2DjKfw28p86YTd/B4lRGcNuu12sKE35sM7A=
github.com/bkielbasa/cyclop v1.2.0/go.mod h1:qOI0yy6A7dYC4Zgsa72Ppm9kONl0RoIlPbzot9mhmeI=
github.com/blizzy78/varnamelen v0.6.1 h1:kttPCLzXFa+0nt++Cw9fb7GrSSM4KkyIAoX/vXsbuqA=
github.com/blizzy78/varnamelen v0.6.1/go.mod h1:zy2Eic4qWqjrxa60jG34cfL0VXcSwzUrIx68eJPb4Q8=
github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M=
github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k=
github.com/bombsimon/wsl/v3 v3.3.0 h1:Mka/+kRLoQJq7g2rggtgQsjuI/K5Efd87WX96EWFxjM=
github.com/bombsimon/wsl/v3 v3.3.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
github.com/breml/bidichk v0.2.2 h1:w7QXnpH0eCBJm55zGCTJveZEkQBt6Fs5zThIdA6qQ9Y=
github.com/breml/bidichk v0.2.2/go.mod h1:zbfeitpevDUGI7V91Uzzuwrn4Vls8MoBMrwtt78jmso=
github.com/breml/errchkjson v0.2.3 h1:97eGTmR/w0paL2SwfRPI1jaAZHaH/fXnxWTw2eEIqE0=
github.com/breml/errchkjson v0.2.3/go.mod h1:jZEATw/jF69cL1iy7//Yih8yp/mXp2CBoBr9GJwCAsY=
github.com/breml/bidichk v0.2.3 h1:qe6ggxpTfA8E75hdjWPZ581sY3a2lnl0IRxLQFelECI=
github.com/breml/bidichk v0.2.3/go.mod h1:8u2C6DnAy0g2cEq+k/A2+tr9O1s+vHGxWn0LTc70T2A=
github.com/breml/errchkjson v0.3.0 h1:YdDqhfqMT+I1vIxPSas44P+9Z9HzJwCeAzjB8PxP1xw=
github.com/breml/errchkjson v0.3.0/go.mod h1:9Cogkyv9gcT8HREpzi3TiqBxCqDzo8awa92zSDFcofU=
github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c=
github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y=
@@ -187,8 +188,8 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/charithe/durationcheck v0.0.9 h1:mPP4ucLrf/rKZiIG/a9IPXHGlh8p4CzgpyTy6EEutYk=
github.com/charithe/durationcheck v0.0.9/go.mod h1:SSbRIBVfMjCi/kEB6K65XEA83D6prSM8ap1UCpNKtgg=
github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af h1:spmv8nSH9h5oCQf40jt/ufBCt9j0/58u4G+rkeMqXGI=
github.com/chavacava/garif v0.0.0-20210405164556-e8a0a408d6af/go.mod h1:Qjyv4H3//PWVzTeCezG2b9IRn6myJxJSr4TD/xo6ojU=
github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4 h1:tFXjAxje9thrTF4h57Ckik+scJjTWdwAtZqZPtOT48M=
github.com/chavacava/garif v0.0.0-20220316182200-5cad0b5181d4/go.mod h1:W8EnPSQ8Nv4fUjc/v1/8tHFqhuOJXnRub0dTfuAQktU=
github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -227,12 +228,13 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creachadair/atomicfile v0.2.5 h1:wkOlpsjyJOvJ3Hd8juHKdirJnCSIPacvtY21/3nYjAo=
github.com/creachadair/atomicfile v0.2.5/go.mod h1:BRq8Une6ckFneYXZQ+kO7p1ZZP3I2fzVzf28JxrIkBc=
github.com/creachadair/atomicfile v0.2.6 h1:FgYxYvGcqREApTY8Nxg8msM6P/KVKK3ob5h9FaRUTNg=
github.com/creachadair/atomicfile v0.2.6/go.mod h1:BRq8Une6ckFneYXZQ+kO7p1ZZP3I2fzVzf28JxrIkBc=
github.com/creachadair/command v0.0.0-20220426235536-a748effdf6a1/go.mod h1:bAM+qFQb/KwWyCc9MLC4U1jvn3XyakqP5QRkds5T6cY=
github.com/creachadair/taskgroup v0.3.2 h1:zlfutDS+5XG40AOxcHDSThxKzns8Tnr9jnr6VqkYlkM=
github.com/creachadair/taskgroup v0.3.2/go.mod h1:wieWwecHVzsidg2CsUnFinW1faVN4+kq+TDlRJQ0Wbk=
github.com/creachadair/tomledit v0.0.19 h1:zbpfUtYFYFdpRjwJY9HJlto1iZ4M5YwYB6qqc37F6UM=
github.com/creachadair/tomledit v0.0.19/go.mod h1:gvtfnSZLa+YNQD28vaPq0Nk12bRxEhmUdBzAWn+EGF4=
github.com/creachadair/tomledit v0.0.22 h1:lRtepmrwhzDq+g1gv5ftVn5itgo7CjYbm6abKTToqJ4=
github.com/creachadair/tomledit v0.0.22/go.mod h1:cIu/4x5L855oSRejIqr+WRFh+mv9g4fWLiUFaApYn/Y=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
@@ -299,6 +301,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
github.com/firefart/nonamedreturns v1.0.1 h1:fSvcq6ZpK/uBAgJEGMvzErlzyM4NELLqqdTofVjVNag=
github.com/firefart/nonamedreturns v1.0.1/go.mod h1:D3dpIBojGGNh5UfElmwPu73SwDCm+VKhHYqwlNOk2uQ=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
@@ -309,14 +313,15 @@ github.com/frankban/quicktest v1.14.2 h1:SPb1KFFmM+ybpEjPUhCCkZOM5xlovT5UbrMvWnX
github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM=
github.com/fzipp/gocyclo v0.4.0 h1:IykTnjwh2YLyYkGa0y92iTTEQcnyAz0r9zOo15EbJ7k=
github.com/fzipp/gocyclo v0.4.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
github.com/fzipp/gocyclo v0.5.1 h1:L66amyuYogbxl0j2U+vGqJXusPF2IkduvXLnYD5TFgw=
github.com/fzipp/gocyclo v0.5.1/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-critic/go-critic v0.6.2 h1:L5SDut1N4ZfsWZY0sH4DCrsHLHnhuuWak2wa165t9gs=
github.com/go-critic/go-critic v0.6.2/go.mod h1:td1s27kfmLpe5G/DPjlnFI7o1UCzePptwU7Az0V5iCM=
github.com/go-critic/go-critic v0.6.3 h1:abibh5XYBTASawfTQ0rA7dVtQT+6KzpGqb/J+DxRDaw=
github.com/go-critic/go-critic v0.6.3/go.mod h1:c6b3ZP1MQ7o6lPR7Rv3lEf7pYQUmAcx8ABHgdZCQt/k=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -418,12 +423,12 @@ github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5
github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe h1:6RGUuS7EGotKx6J5HIP8ZtyMdiDscjMLfRBSPuzVVeo=
github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe/go.mod h1:gjqyPShc/m8pEMpk0a3SeagVb0kaqvhscv+i9jI5ZhQ=
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
github.com/golangci/golangci-lint v1.45.2 h1:9I3PzkvscJkFAQpTQi5Ga0V4qWdJERajX1UZ7QqkW+I=
github.com/golangci/golangci-lint v1.45.2/go.mod h1:f20dpzMmUTRp+oYnX0OGjV1Au3Jm2JeI9yLqHq1/xsI=
github.com/golangci/golangci-lint v1.46.0 h1:uz9AtEcIP63FH+FIyuAXcQGVQO4vCUavEsMTJpPeD4s=
github.com/golangci/golangci-lint v1.46.0/go.mod h1:IJpcNOUfx/XLRwE95FHQ6QtbhYwwqcm0H5QkwUfF4ZE=
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
@@ -655,15 +660,15 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kulti/thelper v0.5.1 h1:Uf4CUekH0OvzQTFPrWkstJvXgm6pnNEtQu3HiqEkpB0=
github.com/kulti/thelper v0.5.1/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5exl2U=
github.com/kulti/thelper v0.6.2 h1:K4xulKkwOCnT1CDms6Ex3uG1dvSMUUQe9zxgYQgbRXs=
github.com/kulti/thelper v0.6.2/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dqWHZ6I=
github.com/kunwardeep/paralleltest v1.0.3 h1:UdKIkImEAXjR1chUWLn+PNXqWUGs//7tzMeWuP7NhmI=
github.com/kunwardeep/paralleltest v1.0.3/go.mod h1:vLydzomDFpk7yu5UX02RmP0H8QfRPOV/oFhWN85Mjb4=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/kyoh86/exportloopref v0.1.8 h1:5Ry/at+eFdkX9Vsdw3qU4YkvGtzuVfzT4X7S77LoN/M=
github.com/kyoh86/exportloopref v0.1.8/go.mod h1:1tUcJeiioIs7VWe5gcOObrux3lb66+sBqGZrRkMwPgg=
github.com/ldez/gomoddirectives v0.2.2 h1:p9/sXuNFArS2RLc+UpYZSI4KQwGMEDWC/LbtF5OPFVg=
github.com/ldez/gomoddirectives v0.2.2/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdBr6g8G04uz6d0=
github.com/ldez/gomoddirectives v0.2.3 h1:y7MBaisZVDYmKvt9/l1mjNCiSA1BVn34U0ObUcJwlhA=
github.com/ldez/gomoddirectives v0.2.3/go.mod h1:cpgBogWITnCfRq2qGoDkKMEVSaarhdBr6g8G04uz6d0=
github.com/ldez/tagliatelle v0.3.1 h1:3BqVVlReVUZwafJUwQ+oxbx2BEX2vUG4Yu/NOfMiKiM=
github.com/ldez/tagliatelle v0.3.1/go.mod h1:8s6WJQwEYHbKZDsp/LjArytKOG8qaMrKQQ3mFukHs88=
github.com/leonklingele/grouper v1.1.0 h1:tC2y/ygPbMFSBOs3DcyaEMKnnwH7eYKzohOtRrf0SAg=
@@ -673,11 +678,12 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs=
github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/lufeee/execinquery v1.0.0 h1:1XUTuLIVPDlFvUU3LXmmZwHDsolsxXnY67lzhpeqe0I=
github.com/lufeee/execinquery v1.0.0/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -718,8 +724,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwgOdMUQePUo=
github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg=
github.com/mgechev/revive v1.1.4 h1:sZOjY6GU35Kr9jKa/wsKSHgrFz8eASIB5i3tqWZMp0A=
github.com/mgechev/revive v1.1.4/go.mod h1:ZZq2bmyssGh8MSPz3VVziqRNIMYTJXzP8MUKG90vZ9A=
github.com/mgechev/revive v1.2.1 h1:GjFml7ZsoR0IrQ2E2YIvWFNS5GPDV7xNwvA5GM1HZC4=
github.com/mgechev/revive v1.2.1/go.mod h1:+Ro3wqY4vakcYNtkBWdZC7dBg1xSB6sp054wWwmeFm0=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
@@ -738,8 +744,9 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
@@ -777,8 +784,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA
github.com/nishanths/exhaustive v0.7.11 h1:xV/WU3Vdwh5BUH4N06JNUznb6d5zhRPOnlgCrpNYNKA=
github.com/nishanths/exhaustive v0.7.11/go.mod h1:gX+MP7DWMKJmNa1HfMozK+u04hQd3na9i0hyqf3/dOI=
github.com/nishanths/predeclared v0.0.0-20190419143655-18a43bb90ffc/go.mod h1:62PewwiQTlm/7Rj+cxVYqZvDIUc+JjZq6GHAC1fsObQ=
github.com/nishanths/predeclared v0.2.1 h1:1TXtjmy4f3YCFjTxRd8zcFHOmoUir+gp0ESzjFzG2sw=
github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB9sbB1usJ+xjQE=
github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk=
github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@@ -832,10 +839,12 @@ github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.0.0-beta.8 h1:dy81yyLYJDwMTifq24Oi/IslOslRrDSb3jwDggjz3Z0=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pelletier/go-toml/v2 v2.0.0-beta.8/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
github.com/pelletier/go-toml/v2 v2.0.0 h1:P7Bq0SaI8nsexyay5UAyDo+ICWy5MQPgEZ5+l8JQTKo=
github.com/pelletier/go-toml/v2 v2.0.0/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA=
@@ -864,8 +873,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -878,8 +888,9 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE=
github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -892,19 +903,23 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T
github.com/pseudomuto/protoc-gen-doc v1.3.2/go.mod h1:y5+P6n3iGrbKG+9O04V5ld71in3v/bX88wUwgt+U8EA=
github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q=
github.com/quasilyte/go-ruleguard v0.3.1-0.20210203134552-1b5a410e1cc8/go.mod h1:KsAh3x0e7Fkpgs+Q9pNLS5XpFSvYCEVl5gP9Pp1xp30=
github.com/quasilyte/go-ruleguard v0.3.15 h1:iWYzp1z72IlXTioET0+XI6SjQdPfMGfuAiZiKznOt7g=
github.com/quasilyte/go-ruleguard v0.3.15/go.mod h1:NhuWhnlVEM1gT1A4VJHYfy9MuYSxxwHgxWoPsn9llB4=
github.com/quasilyte/go-ruleguard v0.3.16-0.20220213074421-6aa060fab41a h1:sWFavxtIctGrVs5SYZ5Ml1CvrDAs8Kf5kx2PI3C41dA=
github.com/quasilyte/go-ruleguard v0.3.16-0.20220213074421-6aa060fab41a/go.mod h1:VMX+OnnSw4LicdiEGtRSD/1X8kW7GuEscjYNr4cOIT4=
github.com/quasilyte/go-ruleguard/dsl v0.3.0/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/quasilyte/go-ruleguard/dsl v0.3.12-0.20220101150716-969a394a9451/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/quasilyte/go-ruleguard/dsl v0.3.12/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/quasilyte/go-ruleguard/dsl v0.3.17/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/quasilyte/go-ruleguard/dsl v0.3.16/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/quasilyte/go-ruleguard/dsl v0.3.19/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/quasilyte/go-ruleguard/rules v0.0.0-20201231183845-9e62ed36efe1/go.mod h1:7JTjp89EGyU1d6XfBiXihJNG37wB2VRkd125Q1u7Plc=
github.com/quasilyte/go-ruleguard/rules v0.0.0-20211022131956-028d6511ab71/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50=
github.com/quasilyte/gogrep v0.0.0-20220103110004-ffaa07af02e3 h1:P4QPNn+TK49zJjXKERt/vyPbv/mCHB/zQ4flDYOMN+M=
github.com/quasilyte/gogrep v0.0.0-20220103110004-ffaa07af02e3/go.mod h1:wSEyW6O61xRV6zb6My3HxrQ5/8ke7NE2OayqCHa3xRM=
github.com/quasilyte/gogrep v0.0.0-20220120141003-628d8b3623b5 h1:PDWGei+Rf2bBiuZIbZmM20J2ftEy9IeUCHA8HbQqed8=
github.com/quasilyte/gogrep v0.0.0-20220120141003-628d8b3623b5/go.mod h1:wSEyW6O61xRV6zb6My3HxrQ5/8ke7NE2OayqCHa3xRM=
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY=
github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs=
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/go-dbus v0.0.0-20121104212943-b7232d34b1d5/go.mod h1:+u151txRmLpwxBmpYn9z3d1sdJdjRPQpsXuYeY9jNls=
github.com/remyoudompheng/go-liblzma v0.0.0-20190506200333-81bf2d431b96/go.mod h1:90HvCY7+oHHUKkbeMCiHt1WuFR2/hPJ9QrljDG+v6ls=
github.com/remyoudompheng/go-misc v0.0.0-20190427085024-2d6ac652a50e/go.mod h1:80FQABjoFzZ2M5uEa6FUaJYEmqU2UOKojlFVak1UAwI=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -925,7 +940,6 @@ github.com/ryancurrah/gomodguard v1.2.3/go.mod h1:rYbA/4Tg5c54mV1sv4sQTP5WOPBcoL
github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw=
github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE=
github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM=
github.com/sagikazarmark/crypt v0.5.0/go.mod h1:l+nzl7KWh51rpzp2h7t4MZWyiEWdhNpOAnclKvg+mdA=
@@ -933,12 +947,12 @@ github.com/sanposhiho/wastedassign/v2 v2.0.6 h1:+6/hQIHKNJAUixEj6EmOngGIisyeI+T3
github.com/sanposhiho/wastedassign/v2 v2.0.6/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
github.com/securego/gosec/v2 v2.10.0 h1:l6BET4EzWtyUXCpY2v7N92v0DDCas0L7ngg3bpqbr8g=
github.com/securego/gosec/v2 v2.10.0/go.mod h1:PVq8Ewh/nCN8l/kKC6zrGXSr7m2NmEK6ITIAWMtIaA0=
github.com/securego/gosec/v2 v2.11.0 h1:+PDkpzR41OI2jrw1q6AdXZCbsNGNGT7pQjal0H0cArI=
github.com/securego/gosec/v2 v2.11.0/go.mod h1:SX8bptShuG8reGC0XS09+a4H2BoWSJi+fscA+Pulbpo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
github.com/shirou/gopsutil/v3 v3.22.4/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM=
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -950,8 +964,8 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sivchari/containedctx v1.0.2 h1:0hLQKpgC53OVF1VT7CeoFHk9YKstur1XOgfYIc1yrHI=
github.com/sivchari/containedctx v1.0.2/go.mod h1:PwZOeqm4/DLoJOqMSIJs3aKqXRX4YO+uXww087KZ7Bw=
github.com/sivchari/tenv v1.4.7 h1:FdTpgRlTue5eb5nXIYgS/lyVXSjugU8UUVDwhP1NLU8=
github.com/sivchari/tenv v1.4.7/go.mod h1:5nF+bITvkebQVanjU6IuMbvIot/7ReNsUV7I5NbprB0=
github.com/sivchari/tenv v1.5.0 h1:wxW0mFpKI6DIb3s6m1jCDYvkWXCskrimXMuGd0K/kSQ=
github.com/sivchari/tenv v1.5.0/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa h1:YJfZp12Z3AFhSBeXOlv4BO55RMwPn2NoQeDsrdWnBtY=
@@ -989,13 +1003,14 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/spf13/viper v1.9.0/go.mod h1:+i6ajR7OX2XaiBkrcZJFK21htRk7eDeLg7+O6bhUPP4=
github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM=
github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU=
github.com/spf13/viper v1.11.0 h1:7OX/1FS6n7jHD1zGrZTM7WtY13ZELRyosK4k93oPr44=
github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUsX7Zk=
github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0=
github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc=
github.com/stbenjam/no-sprintf-host-port v0.1.1/go.mod h1:TLhvtIvONRzdmkFiio4O8LHsN9N74I+PhRquPsxpL0I=
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
@@ -1033,13 +1048,13 @@ github.com/tetafro/godot v1.4.11 h1:BVoBIqAf/2QdbFmSwAWnaIqDivZdOV0ZRwEm6jivLKw=
github.com/tetafro/godot v1.4.11/go.mod h1:LR3CJpxDVGlYOWn3ZZg1PgNZdTUvzsZWu8xaEohUpn8=
github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144 h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro=
github.com/timakin/bodyclose v0.0.0-20210704033933-f49887972144/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20200427203606-3cfed13b9966/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tomarrell/wrapcheck/v2 v2.5.0 h1:g27SGGHNoQdvHz4KZA9o4v09RcWzylR+b1yueE5ECiw=
github.com/tomarrell/wrapcheck/v2 v2.5.0/go.mod h1:68bQ/eJg55BROaRTbMjC7vuhL2OgfoG8bLp9ZyoBfyY=
github.com/tomarrell/wrapcheck/v2 v2.6.1 h1:Cf4a/iwuMp9s7kKrh74GTgijRVim0wEpKjgAsT7Wctw=
github.com/tomarrell/wrapcheck/v2 v2.6.1/go.mod h1:Eo+Opt6pyMW1b6cNllOcDSSoHO0aTJ+iF6BfCUbHltA=
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
github.com/tommy-muehle/go-mnd/v2 v2.5.0 h1:iAj0a8e6+dXSL7Liq0aXPox36FiN1dBbjA6lt9fl65s=
github.com/tommy-muehle/go-mnd/v2 v2.5.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw=
@@ -1059,8 +1074,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/vektra/mockery/v2 v2.12.1 h1:BAJk2fGjVg/P9Fi+BxZD1/ZeKTOclpeAb/SKCc12zXc=
github.com/vektra/mockery/v2 v2.12.1/go.mod h1:8vf4KDDUptfkyypzdHLuE7OE2xA7Gdt60WgIS8PgD+U=
github.com/vektra/mockery/v2 v2.12.2 h1:JbRx9F+XcCJiDTyCm3V5lXYwl56m5ZouV6I9eZa1Dj0=
github.com/vektra/mockery/v2 v2.12.2/go.mod h1:8vf4KDDUptfkyypzdHLuE7OE2xA7Gdt60WgIS8PgD+U=
github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
@@ -1069,8 +1084,8 @@ github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1z
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM=
github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk=
github.com/yeya24/promlinter v0.1.1-0.20210918184747-d757024714a1 h1:YAaOqqMTstELMMGblt6yJ/fcOt4owSYuw3IttMnKfAM=
github.com/yeya24/promlinter v0.1.1-0.20210918184747-d757024714a1/go.mod h1:rs5vtZzeBHqqMwXqFScncpCF6u06lezhZepno9AB1Oc=
github.com/yeya24/promlinter v0.2.0 h1:xFKDQ82orCU5jQujdaD8stOHiv8UN68BSdn2a8u8Y3o=
github.com/yeya24/promlinter v0.2.0/go.mod h1:u54lkmBOZrpEbQQ6gox2zWKKLKu2SGe+2KOiextY+IA=
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
@@ -1152,7 +1167,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220313003712-b769efc7c000/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA=
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1168,7 +1183,10 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 h1:FR+oGxGfbQu1d+jglI3rCkjAjUnhRSZcUxr+DqlDLNo=
golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e h1:qyrTQ++p1afMkO4DPEeLGq/3oTsdlvdH4vqZUBWzUKM=
golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@@ -1385,11 +1403,9 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1400,14 +1416,17 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
golang.org/x/sys v0.0.0-20220403020550-483a9cbc67c0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
@@ -1435,6 +1454,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190228203856-589c23e65e65/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -1508,7 +1528,6 @@ golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4X
golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201028025901-8cd080b735b3/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201114224030-61ea331ec02b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
@@ -1528,8 +1547,9 @@ golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.9-0.20211228192929-ee1ca4ffc4da/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a h1:ofrrl6c6NG5/IOSx/R1cyiQxxjqlur0h/TvbUhkH0II=
golang.org/x/tools v0.1.11-0.20220316014157-77aa08bb151a/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1703,8 +1723,8 @@ google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ5
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ=
google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1733,7 +1753,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.66.3/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4=
@@ -1765,10 +1784,10 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.2.2 h1:MNh1AVMyVX23VUHE2O27jm6lNj3vjO5DexS4A1xvnzk=
honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY=
mvdan.cc/gofumpt v0.3.0 h1:kTojdZo9AcEYbQYhGuLf/zszYthRdhDNDUi2JKTxas4=
mvdan.cc/gofumpt v0.3.0/go.mod h1:0+VyGZWleeIj5oostkOex+nDBA0eyavuDnDusAJ8ylo=
honnef.co/go/tools v0.3.1 h1:1kJlrWJLkaGXgcaeosRXViwviqjI7nkBvU2+sZW0AYc=
honnef.co/go/tools v0.3.1/go.mod h1:vlRD9XErLMGT+mDuofSr0mMMquscM/1nQqtRSsh6m70=
mvdan.cc/gofumpt v0.3.1 h1:avhhrOmv0IuvQVK7fvwV91oFSGAk5/6Po8GXTzICeu8=
mvdan.cc/gofumpt v0.3.1/go.mod h1:w3ymliuxvzVx8DAutBnVyDqYb1Niy/yCJt/lk821YCE=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
internal/blocksync/pool.go
+44 -16
View File
@@ -200,16 +200,20 @@ func (pool *BlockPool) IsCaughtUp() bool {
return pool.height >= (pool.maxPeerHeight - 1)
}
// PeekTwoBlocks returns blocks at pool.height and pool.height+1.
// We need to see the second block's Commit to validate the first block.
// So we peek two blocks at a time.
// PeekTwoBlocks returns blocks at pool.height and pool.height+1. We need to
// see the second block's Commit to validate the first block. So we peek two
// blocks at a time. We return an extended commit, containing vote extensions
// and their associated signatures, as this is critical to consensus in ABCI++
// as we switch from block sync to consensus mode.
//
// The caller will verify the commit.
func (pool *BlockPool) PeekTwoBlocks() (first *types.Block, second *types.Block) {
func (pool *BlockPool) PeekTwoBlocks() (first, second *types.Block, firstExtCommit *types.ExtendedCommit) {
pool.mtx.RLock()
defer pool.mtx.RUnlock()
if r := pool.requesters[pool.height]; r != nil {
first = r.getBlock()
firstExtCommit = r.getExtendedCommit()
}
if r := pool.requesters[pool.height+1]; r != nil {
second = r.getBlock()
@@ -218,7 +222,8 @@ func (pool *BlockPool) PeekTwoBlocks() (first *types.Block, second *types.Block)
}
// PopRequest pops the first block at pool.height.
// It must have been validated by 'second'.Commit from PeekTwoBlocks().
// It must have been validated by the second Commit from PeekTwoBlocks.
// TODO(thane): (?) and its corresponding ExtendedCommit.
func (pool *BlockPool) PopRequest() {
pool.mtx.Lock()
defer pool.mtx.Unlock()
@@ -262,16 +267,25 @@ func (pool *BlockPool) RedoRequest(height int64) types.NodeID {
return peerID
}
// AddBlock validates that the block comes from the peer it was expected from and calls the requester to store it.
// AddBlock validates that the block comes from the peer it was expected from
// and calls the requester to store it.
//
// This requires an extended commit at the same height as the supplied block -
// the block contains the last commit, but we need the latest commit in case we
// need to switch over from block sync to consensus at this height. If the
// height of the extended commit and the height of the block do not match, we
// do not add the block and return an error.
// TODO: ensure that blocks come in order for each peer.
func (pool *BlockPool) AddBlock(peerID types.NodeID, block *types.Block, blockSize int) {
func (pool *BlockPool) AddBlock(peerID types.NodeID, block *types.Block, extCommit *types.ExtendedCommit, blockSize int) error {
pool.mtx.Lock()
defer pool.mtx.Unlock()
if extCommit != nil && block.Height != extCommit.Height {
return fmt.Errorf("heights don't match, not adding block (block height: %d, commit height: %d)", block.Height, extCommit.Height)
}
requester := pool.requesters[block.Height]
if requester == nil {
pool.logger.Error("peer sent us a block we didn't expect",
"peer", peerID, "curHeight", pool.height, "blockHeight", block.Height)
diff := pool.height - block.Height
if diff < 0 {
diff *= -1
@@ -279,10 +293,10 @@ func (pool *BlockPool) AddBlock(peerID types.NodeID, block *types.Block, blockSi
if diff > maxDiffBetweenCurrentAndReceivedBlockHeight {
pool.sendError(errors.New("peer sent us a block we didn't expect with a height too far ahead/behind"), peerID)
}
return
return fmt.Errorf("peer sent us a block we didn't expect (peer: %s, current height: %d, block height: %d)", peerID, pool.height, block.Height)
}
if requester.setBlock(block, peerID) {
if requester.setBlock(block, extCommit, peerID) {
atomic.AddInt32(&pool.numPending, -1)
peer := pool.peers[peerID]
if peer != nil {
@@ -290,9 +304,11 @@ func (pool *BlockPool) AddBlock(peerID types.NodeID, block *types.Block, blockSi
}
} else {
err := errors.New("requester is different or block already exists")
pool.logger.Error(err.Error(), "peer", peerID, "requester", requester.getPeerID(), "blockHeight", block.Height)
pool.sendError(err, peerID)
return fmt.Errorf("%w (peer: %s, requester: %s, block height: %d)", err, peerID, requester.getPeerID(), block.Height)
}
return nil
}
// MaxPeerHeight returns the highest reported height.
@@ -456,6 +472,7 @@ func (pool *BlockPool) debug() string {
} else {
str += fmt.Sprintf("H(%v):", h)
str += fmt.Sprintf("B?(%v) ", pool.requesters[h].block != nil)
str += fmt.Sprintf("C?(%v) ", pool.requesters[h].extCommit != nil)
}
}
return str
@@ -544,9 +561,10 @@ type bpRequester struct {
gotBlockCh chan struct{}
redoCh chan types.NodeID // redo may send multitime, add peerId to identify repeat
mtx sync.Mutex
peerID types.NodeID
block *types.Block
mtx sync.Mutex
peerID types.NodeID
block *types.Block
extCommit *types.ExtendedCommit
}
func newBPRequester(logger log.Logger, pool *BlockPool, height int64) *bpRequester {
@@ -572,13 +590,16 @@ func (bpr *bpRequester) OnStart(ctx context.Context) error {
func (*bpRequester) OnStop() {}
// Returns true if the peer matches and block doesn't already exist.
func (bpr *bpRequester) setBlock(block *types.Block, peerID types.NodeID) bool {
func (bpr *bpRequester) setBlock(block *types.Block, extCommit *types.ExtendedCommit, peerID types.NodeID) bool {
bpr.mtx.Lock()
if bpr.block != nil || bpr.peerID != peerID {
bpr.mtx.Unlock()
return false
}
bpr.block = block
if extCommit != nil {
bpr.extCommit = extCommit
}
bpr.mtx.Unlock()
select {
@@ -594,6 +615,12 @@ func (bpr *bpRequester) getBlock() *types.Block {
return bpr.block
}
func (bpr *bpRequester) getExtendedCommit() *types.ExtendedCommit {
bpr.mtx.Lock()
defer bpr.mtx.Unlock()
return bpr.extCommit
}
func (bpr *bpRequester) getPeerID() types.NodeID {
bpr.mtx.Lock()
defer bpr.mtx.Unlock()
@@ -611,6 +638,7 @@ func (bpr *bpRequester) reset() {
bpr.peerID = ""
bpr.block = nil
bpr.extCommit = nil
}
// Tells bpRequester to pick another peer and try again.
internal/blocksync/pool_test.go
+6 -3
View File
@@ -43,7 +43,10 @@ func (p testPeer) runInputRoutine() {
// Request desired, pretend like we got the block immediately.
func (p testPeer) simulateInput(input inputData) {
block := &types.Block{Header: types.Header{Height: input.request.Height}}
input.pool.AddBlock(input.request.PeerID, block, 123)
extCommit := &types.ExtendedCommit{
Height: input.request.Height,
}
_ = input.pool.AddBlock(input.request.PeerID, block, extCommit, 123)
// TODO: uncommenting this creates a race which is detected by:
// https://github.com/golang/go/blob/2bd767b1022dd3254bcec469f0ee164024726486/src/testing/testing.go#L854-L856
// see: https://github.com/tendermint/tendermint/issues/3390#issue-418379890
@@ -110,7 +113,7 @@ func TestBlockPoolBasic(t *testing.T) {
if !pool.IsRunning() {
return
}
first, second := pool.PeekTwoBlocks()
first, second, _ := pool.PeekTwoBlocks()
if first != nil && second != nil {
pool.PopRequest()
} else {
@@ -164,7 +167,7 @@ func TestBlockPoolTimeout(t *testing.T) {
if !pool.IsRunning() {
return
}
first, second := pool.PeekTwoBlocks()
first, second, _ := pool.PeekTwoBlocks()
if first != nil && second != nil {
pool.PopRequest()
} else {
internal/blocksync/reactor.go
+126 -45
View File
@@ -76,7 +76,7 @@ type Reactor struct {
stateStore sm.Store
blockExec *sm.BlockExecutor
store *store.BlockStore
store sm.BlockStore
pool *BlockPool
consReactor consensusReactor
blockSync *atomicBool
@@ -185,25 +185,39 @@ func (r *Reactor) OnStop() {
// Otherwise, we'll respond saying we do not have it.
func (r *Reactor) respondToPeer(ctx context.Context, msg *bcproto.BlockRequest, peerID types.NodeID, blockSyncCh *p2p.Channel) error {
block := r.store.LoadBlock(msg.Height)
if block != nil {
blockProto, err := block.ToProto()
if err != nil {
r.logger.Error("failed to convert msg to protobuf", "err", err)
return err
}
if block == nil {
r.logger.Info("peer requesting a block we do not have", "peer", peerID, "height", msg.Height)
return blockSyncCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: &bcproto.BlockResponse{Block: blockProto},
Message: &bcproto.NoBlockResponse{Height: msg.Height},
})
}
r.logger.Info("peer requesting a block we do not have", "peer", peerID, "height", msg.Height)
state, err := r.stateStore.Load()
if err != nil {
return fmt.Errorf("loading state: %w", err)
}
var extCommit *types.ExtendedCommit
if state.ConsensusParams.ABCI.VoteExtensionsEnabled(msg.Height) {
extCommit = r.store.LoadBlockExtendedCommit(msg.Height)
if extCommit == nil {
return fmt.Errorf("found block in store with no extended commit: %v", block)
}
}
blockProto, err := block.ToProto()
if err != nil {
return fmt.Errorf("failed to convert block to protobuf: %w", err)
}
return blockSyncCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: &bcproto.NoBlockResponse{Height: msg.Height},
To: peerID,
Message: &bcproto.BlockResponse{
Block: blockProto,
ExtCommit: extCommit.ToProto(),
},
})
}
// handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
@@ -236,8 +250,21 @@ func (r *Reactor) handleMessage(ctx context.Context, envelope *p2p.Envelope, blo
"err", err)
return err
}
var extCommit *types.ExtendedCommit
if msg.ExtCommit != nil {
var err error
extCommit, err = types.ExtendedCommitFromProto(msg.ExtCommit)
if err != nil {
r.logger.Error("failed to convert extended commit from proto",
"peer", envelope.From,
"err", err)
return err
}
}
r.pool.AddBlock(envelope.From, block, block.Size())
if err := r.pool.AddBlock(envelope.From, block, extCommit, block.Size()); err != nil {
r.logger.Error("failed to add block", "err", err)
}
case *bcproto.StatusRequest:
return blockSyncCh.Send(ctx, p2p.Envelope{
@@ -425,6 +452,8 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool, blockSyncCh
lastRate = 0.0
didProcessCh = make(chan struct{}, 1)
initialCommitHasExtensions = (r.initialState.LastBlockHeight > 0 && r.store.LoadBlockExtendedCommit(r.initialState.LastBlockHeight) != nil)
)
defer trySyncTicker.Stop()
@@ -448,6 +477,35 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool, blockSyncCh
)
switch {
// The case statement below is a bit confusing, so here is a breakdown
// of its logic and purpose:
//
// If VoteExtensions are enabled we cannot switch to consensus without
// the vote extension data for the previous height, i.e. state.LastBlockHeight.
//
// If extensions were required during state.LastBlockHeight and we have
// sync'd at least one block, then we are guaranteed to have extensions.
// BlockSync requires that the blocks it fetches have extensions if
// extensions were enabled during the height.
//
// If extensions were required during state.LastBlockHeight and we have
// not sync'd any blocks, then we can only transition to Consensus
// if we already had extensions for the initial height.
// If any of these conditions is not met, we continue the loop, looking
// for extensions.
case state.ConsensusParams.ABCI.VoteExtensionsEnabled(state.LastBlockHeight) &&
(blocksSynced == 0 && !initialCommitHasExtensions):
r.logger.Info(
"no extended commit yet",
"height", height,
"last_block_height", state.LastBlockHeight,
"initial_height", state.InitialHeight,
"max_peer_height", r.pool.MaxPeerHeight(),
"timeout_in", syncTimeout-time.Since(lastAdvance),
)
continue
case r.pool.IsCaughtUp():
r.logger.Info("switching to consensus reactor", "height", height)
@@ -490,15 +548,20 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool, blockSyncCh
// TODO: Uncouple from request routine.
// see if there are any blocks to sync
first, second := r.pool.PeekTwoBlocks()
if first == nil || second == nil {
// we need both to sync the first block
first, second, extCommit := r.pool.PeekTwoBlocks()
if first != nil && extCommit == nil &&
state.ConsensusParams.ABCI.VoteExtensionsEnabled(first.Height) {
// See https://github.com/tendermint/tendermint/pull/8433#discussion_r866790631
panic(fmt.Errorf("peeked first block without extended commit at height %d - possible node store corruption", first.Height))
} else if first == nil || second == nil {
// we need to have fetched two consecutive blocks in order to
// perform blocksync verification
continue
} else {
// try again quickly next loop
didProcessCh <- struct{}{}
}
// try again quickly next loop
didProcessCh <- struct{}{}
firstParts, err := first.MakePartSet(types.BlockPartSizeBytes)
if err != nil {
r.logger.Error("failed to make ",
@@ -517,8 +580,20 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool, blockSyncCh
// NOTE: We can probably make this more efficient, but note that calling
// first.Hash() doesn't verify the tx contents, so MakePartSet() is
// currently necessary.
if err = state.Validators.VerifyCommitLight(chainID, firstID, first.Height, second.LastCommit); err != nil {
err = fmt.Errorf("invalid last commit: %w", err)
// TODO(sergio): Should we also validate against the extended commit?
err = state.Validators.VerifyCommitLight(chainID, firstID, first.Height, second.LastCommit)
if err == nil {
// validate the block before we persist it
err = r.blockExec.ValidateBlock(ctx, state, first)
}
if err == nil && state.ConsensusParams.ABCI.VoteExtensionsEnabled(first.Height) {
// if vote extensions were required at this height, ensure they exist.
err = extCommit.EnsureExtensions()
}
// If either of the checks failed we log the error and request for a new block
// at that height
if err != nil {
r.logger.Error(
err.Error(),
"last_commit", second.LastCommit,
@@ -545,37 +620,43 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool, blockSyncCh
return
}
}
return
}
r.pool.PopRequest()
// TODO: batch saves so we do not persist to disk every block
if state.ConsensusParams.ABCI.VoteExtensionsEnabled(first.Height) {
r.store.SaveBlockWithExtendedCommit(first, firstParts, extCommit)
} else {
r.pool.PopRequest()
// TODO: batch saves so we do not persist to disk every block
// We use LastCommit here instead of extCommit. extCommit is not
// guaranteed to be populated by the peer if extensions are not enabled.
// Currently, the peer should provide an extCommit even if the vote extension data are absent
// but this may change so using second.LastCommit is safer.
r.store.SaveBlock(first, firstParts, second.LastCommit)
}
var err error
// TODO: Same thing for app - but we would need a way to get the hash
// without persisting the state.
state, err = r.blockExec.ApplyBlock(ctx, state, firstID, first)
if err != nil {
panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
}
// TODO: Same thing for app - but we would need a way to get the hash
// without persisting the state.
state, err = r.blockExec.ApplyBlock(ctx, state, firstID, first)
if err != nil {
// TODO: This is bad, are we zombie?
panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
}
r.metrics.RecordConsMetrics(first)
r.metrics.RecordConsMetrics(first)
blocksSynced++
blocksSynced++
if blocksSynced%100 == 0 {
lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
r.logger.Info(
"block sync rate",
"height", r.pool.height,
"max_peer_height", r.pool.MaxPeerHeight(),
"blocks/s", lastRate,
)
if blocksSynced%100 == 0 {
lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
r.logger.Info(
"block sync rate",
"height", r.pool.height,
"max_peer_height", r.pool.MaxPeerHeight(),
"blocks/s", lastRate,
)
lastHundred = time.Now()
}
lastHundred = time.Now()
}
}
}
internal/blocksync/reactor_test.go
+121 -56
View File
@@ -40,8 +40,6 @@ type reactorTestSuite struct {
blockSyncChannels map[types.NodeID]*p2p.Channel
peerChans map[types.NodeID]chan p2p.PeerUpdate
peerUpdates map[types.NodeID]*p2p.PeerUpdates
blockSync bool
}
func setup(
@@ -69,7 +67,6 @@ func setup(
blockSyncChannels: make(map[types.NodeID]*p2p.Channel, numNodes),
peerChans: make(map[types.NodeID]chan p2p.PeerUpdate, numNodes),
peerUpdates: make(map[types.NodeID]*p2p.PeerUpdates, numNodes),
blockSync: true,
}
chDesc := &p2p.ChannelDescriptor{ID: BlockSyncChannel, MessageType: new(bcproto.Message)}
@@ -97,21 +94,19 @@ func setup(
return rts
}
func (rts *reactorTestSuite) addNode(
func makeReactor(
ctx context.Context,
t *testing.T,
nodeID types.NodeID,
genDoc *types.GenesisDoc,
privVal types.PrivValidator,
maxBlockHeight int64,
) {
t.Helper()
channelCreator p2p.ChannelCreator,
peerEvents p2p.PeerEventSubscriber) *Reactor {
logger := log.NewNopLogger()
rts.nodes = append(rts.nodes, nodeID)
rts.app[nodeID] = proxy.New(abciclient.NewLocalClient(logger, &abci.BaseApplication{}), logger, proxy.NopMetrics())
require.NoError(t, rts.app[nodeID].Start(ctx))
app := proxy.New(abciclient.NewLocalClient(logger, &abci.BaseApplication{}), logger, proxy.NopMetrics())
require.NoError(t, app.Start(ctx))
blockDB := dbm.NewMemDB()
stateDB := dbm.NewMemDB()
@@ -139,7 +134,7 @@ func (rts *reactorTestSuite) addNode(
blockExec := sm.NewBlockExecutor(
stateStore,
log.NewNopLogger(),
rts.app[nodeID],
app,
mp,
sm.EmptyEvidencePool{},
blockStore,
@@ -147,40 +142,35 @@ func (rts *reactorTestSuite) addNode(
sm.NopMetrics(),
)
for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
lastCommit := types.NewCommit(blockHeight-1, 0, types.BlockID{}, nil)
return NewReactor(
logger,
stateStore,
blockExec,
blockStore,
nil,
channelCreator,
peerEvents,
true,
consensus.NopMetrics(),
nil, // eventbus, can be nil
)
}
if blockHeight > 1 {
lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
lastBlock := blockStore.LoadBlock(blockHeight - 1)
func (rts *reactorTestSuite) addNode(
ctx context.Context,
t *testing.T,
nodeID types.NodeID,
genDoc *types.GenesisDoc,
privVal types.PrivValidator,
maxBlockHeight int64,
) {
t.Helper()
vote, err := factory.MakeVote(
ctx,
privVal,
lastBlock.Header.ChainID, 0,
lastBlock.Header.Height, 0, 2,
lastBlockMeta.BlockID,
time.Now(),
)
require.NoError(t, err)
lastCommit = types.NewCommit(
vote.Height,
vote.Round,
lastBlockMeta.BlockID,
[]types.CommitSig{vote.CommitSig()},
)
}
logger := log.NewNopLogger()
thisBlock := sf.MakeBlock(state, blockHeight, lastCommit)
thisParts, err := thisBlock.MakePartSet(types.BlockPartSizeBytes)
require.NoError(t, err)
blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()}
state, err = blockExec.ApplyBlock(ctx, state, blockID, thisBlock)
require.NoError(t, err)
blockStore.SaveBlock(thisBlock, thisParts, lastCommit)
}
rts.nodes = append(rts.nodes, nodeID)
rts.app[nodeID] = proxy.New(abciclient.NewLocalClient(logger, &abci.BaseApplication{}), logger, proxy.NopMetrics())
require.NoError(t, rts.app[nodeID].Start(ctx))
rts.peerChans[nodeID] = make(chan p2p.PeerUpdate)
rts.peerUpdates[nodeID] = p2p.NewPeerUpdates(rts.peerChans[nodeID], 1)
@@ -189,21 +179,64 @@ func (rts *reactorTestSuite) addNode(
chCreator := func(ctx context.Context, chdesc *p2p.ChannelDescriptor) (*p2p.Channel, error) {
return rts.blockSyncChannels[nodeID], nil
}
rts.reactors[nodeID] = NewReactor(
rts.logger.With("nodeID", nodeID),
stateStore,
blockExec,
blockStore,
nil,
chCreator,
func(ctx context.Context) *p2p.PeerUpdates { return rts.peerUpdates[nodeID] },
rts.blockSync,
consensus.NopMetrics(),
nil, // eventbus, can be nil
)
require.NoError(t, rts.reactors[nodeID].Start(ctx))
require.True(t, rts.reactors[nodeID].IsRunning())
peerEvents := func(ctx context.Context) *p2p.PeerUpdates { return rts.peerUpdates[nodeID] }
reactor := makeReactor(ctx, t, nodeID, genDoc, privVal, chCreator, peerEvents)
lastExtCommit := &types.ExtendedCommit{}
state, err := reactor.stateStore.Load()
require.NoError(t, err)
for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
block, blockID, partSet, seenExtCommit := makeNextBlock(ctx, t, state, privVal, blockHeight, lastExtCommit)
state, err = reactor.blockExec.ApplyBlock(ctx, state, blockID, block)
require.NoError(t, err)
reactor.store.SaveBlockWithExtendedCommit(block, partSet, seenExtCommit)
lastExtCommit = seenExtCommit
}
rts.reactors[nodeID] = reactor
require.NoError(t, reactor.Start(ctx))
require.True(t, reactor.IsRunning())
}
func makeNextBlock(ctx context.Context,
t *testing.T,
state sm.State,
signer types.PrivValidator,
height int64,
lc *types.ExtendedCommit) (*types.Block, types.BlockID, *types.PartSet, *types.ExtendedCommit) {
lastExtCommit := lc.Clone()
block := sf.MakeBlock(state, height, lastExtCommit.ToCommit())
partSet, err := block.MakePartSet(types.BlockPartSizeBytes)
require.NoError(t, err)
blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: partSet.Header()}
// Simulate a commit for the current height
vote, err := factory.MakeVote(
ctx,
signer,
block.Header.ChainID,
0,
block.Header.Height,
0,
2,
blockID,
time.Now(),
)
require.NoError(t, err)
seenExtCommit := &types.ExtendedCommit{
Height: vote.Height,
Round: vote.Round,
BlockID: blockID,
ExtendedSignatures: []types.ExtendedCommitSig{vote.ExtendedCommitSig()},
}
return block, blockID, partSet, seenExtCommit
}
func (rts *reactorTestSuite) start(ctx context.Context, t *testing.T) {
@@ -411,3 +444,35 @@ func TestReactor_BadBlockStopsPeer(t *testing.T) {
len(rts.reactors[newNode.NodeID].pool.peers),
)
}
/*
func TestReactorReceivesNoExtendedCommit(t *testing.T) {
blockDB := dbm.NewMemDB()
stateDB := dbm.NewMemDB()
stateStore := sm.NewStore(stateDB)
blockStore := store.NewBlockStore(blockDB)
blockExec := sm.NewBlockExecutor(
stateStore,
log.NewNopLogger(),
rts.app[nodeID],
mp,
sm.EmptyEvidencePool{},
blockStore,
eventbus,
sm.NopMetrics(),
)
NewReactor(
log.NewNopLogger(),
stateStore,
blockExec,
blockStore,
nil,
chCreator,
func(ctx context.Context) *p2p.PeerUpdates { return rts.peerUpdates[nodeID] },
rts.blockSync,
consensus.NopMetrics(),
nil, // eventbus, can be nil
)
}
*/
internal/consensus/byzantine_test.go
+5 -5
View File
@@ -178,22 +178,22 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
lazyNodeState.decideProposal = func(ctx context.Context, height int64, round int32) {
require.NotNil(t, lazyNodeState.privValidator)
var commit *types.Commit
var extCommit *types.ExtendedCommit
switch {
case lazyNodeState.Height == lazyNodeState.state.InitialHeight:
// We're creating a proposal for the first block.
// The commit is empty, but not nil.
commit = types.NewCommit(0, 0, types.BlockID{}, nil)
extCommit = &types.ExtendedCommit{}
case lazyNodeState.LastCommit.HasTwoThirdsMajority():
// Make the commit from LastCommit
commit = lazyNodeState.LastCommit.MakeCommit()
extCommit = lazyNodeState.LastCommit.MakeExtendedCommit()
default: // This shouldn't happen.
lazyNodeState.logger.Error("enterPropose: Cannot propose anything: No commit for the previous block")
return
}
// omit the last signature in the commit
commit.Signatures[len(commit.Signatures)-1] = types.NewCommitSigAbsent()
extCommit.ExtendedSignatures[len(extCommit.ExtendedSignatures)-1] = types.NewExtendedCommitSigAbsent()
if lazyNodeState.privValidatorPubKey == nil {
// If this node is a validator & proposer in the current round, it will
@@ -204,7 +204,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
proposerAddr := lazyNodeState.privValidatorPubKey.Address()
block, err := lazyNodeState.blockExec.CreateProposalBlock(
ctx, lazyNodeState.Height, lazyNodeState.state, commit, proposerAddr, lazyNodeState.LastCommit.GetVotes())
ctx, lazyNodeState.Height, lazyNodeState.state, extCommit, proposerAddr)
require.NoError(t, err)
blockParts, err := block.MakePartSet(types.BlockPartSizeBytes)
require.NoError(t, err)
internal/consensus/common_test.go
+12 -6
View File
@@ -160,7 +160,8 @@ func signVote(
blockID types.BlockID) *types.Vote {
var ext []byte
if voteType == tmproto.PrecommitType {
// Only non-nil precommits are allowed to carry vote extensions.
if voteType == tmproto.PrecommitType && !blockID.IsNil() {
ext = []byte("extension")
}
v, err := vs.signVote(ctx, voteType, chainID, blockID, ext)
@@ -526,10 +527,11 @@ func loadPrivValidator(t *testing.T, cfg *config.Config) *privval.FilePV {
}
type makeStateArgs struct {
config *config.Config
logger log.Logger
validators int
application abci.Application
config *config.Config
consensusParams *types.ConsensusParams
logger log.Logger
validators int
application abci.Application
}
func makeState(ctx context.Context, t *testing.T, args makeStateArgs) (*State, []*validatorStub) {
@@ -550,9 +552,13 @@ func makeState(ctx context.Context, t *testing.T, args makeStateArgs) (*State, [
if args.logger == nil {
args.logger = log.NewNopLogger()
}
c := factory.ConsensusParams()
if args.consensusParams != nil {
c = args.consensusParams
}
state, privVals := makeGenesisState(ctx, t, args.config, genesisStateArgs{
Params: factory.ConsensusParams(),
Params: c,
Validators: validators,
})
internal/consensus/metrics.gen.go
+248
View File
@@ -0,0 +1,248 @@
// Code generated by metricsgen. DO NOT EDIT.
package consensus
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "height",
Help: "Height of the chain.",
}, labels).With(labelsAndValues...),
ValidatorLastSignedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validator_last_signed_height",
Help: "Last height signed by this validator if the node is a validator.",
}, append(labels, "validator_address")).With(labelsAndValues...),
Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "rounds",
Help: "Number of rounds.",
}, labels).With(labelsAndValues...),
RoundDuration: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "round_duration",
Help: "Histogram of round duration.",
Buckets: stdprometheus.ExponentialBucketsRange(0.1, 100, 8),
}, labels).With(labelsAndValues...),
Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validators",
Help: "Number of validators.",
}, labels).With(labelsAndValues...),
ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validators_power",
Help: "Total power of all validators.",
}, labels).With(labelsAndValues...),
ValidatorPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validator_power",
Help: "Power of a validator.",
}, append(labels, "validator_address")).With(labelsAndValues...),
ValidatorMissedBlocks: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validator_missed_blocks",
Help: "Amount of blocks missed per validator.",
}, append(labels, "validator_address")).With(labelsAndValues...),
MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "missing_validators",
Help: "Number of validators who did not sign.",
}, labels).With(labelsAndValues...),
MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "missing_validators_power",
Help: "Total power of the missing validators.",
}, labels).With(labelsAndValues...),
ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "byzantine_validators",
Help: "Number of validators who tried to double sign.",
}, labels).With(labelsAndValues...),
ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "byzantine_validators_power",
Help: "Total power of the byzantine validators.",
}, labels).With(labelsAndValues...),
BlockIntervalSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_interval_seconds",
Help: "Time between this and the last block.",
}, labels).With(labelsAndValues...),
NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "num_txs",
Help: "Number of transactions.",
}, labels).With(labelsAndValues...),
BlockSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_size_bytes",
Help: "Size of the block.",
}, labels).With(labelsAndValues...),
TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "total_txs",
Help: "Total number of transactions.",
}, labels).With(labelsAndValues...),
CommittedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "latest_block_height",
Help: "The latest block height.",
}, labels).With(labelsAndValues...),
BlockSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_syncing",
Help: "Whether or not a node is block syncing. 1 if yes, 0 if no.",
}, labels).With(labelsAndValues...),
StateSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "state_syncing",
Help: "Whether or not a node is state syncing. 1 if yes, 0 if no.",
}, labels).With(labelsAndValues...),
BlockParts: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_parts",
Help: "Number of block parts transmitted by each peer.",
}, append(labels, "peer_id")).With(labelsAndValues...),
StepDuration: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "step_duration",
Help: "Histogram of durations for each step in the consensus protocol.",
Buckets: stdprometheus.ExponentialBucketsRange(0.1, 100, 8),
}, append(labels, "step")).With(labelsAndValues...),
BlockGossipReceiveLatency: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_gossip_receive_latency",
Help: "Histogram of time taken to receive a block in seconds, measured between when a new block is first discovered to when the block is completed.",
Buckets: stdprometheus.ExponentialBucketsRange(0.1, 100, 8),
}, labels).With(labelsAndValues...),
BlockGossipPartsReceived: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_gossip_parts_received",
Help: "Number of block parts received by the node, separated by whether the part was relevant to the block the node is trying to gather or not.",
}, append(labels, "matches_current")).With(labelsAndValues...),
QuorumPrevoteDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "quorum_prevote_delay",
Help: "Interval in seconds between the proposal timestamp and the timestamp of the earliest prevote that achieved a quorum.",
}, append(labels, "proposer_address")).With(labelsAndValues...),
FullPrevoteDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "full_prevote_delay",
Help: "Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted.",
}, append(labels, "proposer_address")).With(labelsAndValues...),
ProposalTimestampDifference: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "proposal_timestamp_difference",
Help: "Difference between the timestamp in the proposal message and the local time of the validator at the time it received the message.",
Buckets: []float64{-10, -.5, -.025, 0, .1, .5, 1, 1.5, 2, 10},
}, append(labels, "is_timely")).With(labelsAndValues...),
VoteExtensionReceiveCount: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "vote_extension_receive_count",
Help: "Number of vote extensions received labeled by application response status.",
}, append(labels, "status")).With(labelsAndValues...),
ProposalReceiveCount: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "proposal_receive_count",
Help: "Total number of proposals received by the node since process start labeled by application response status.",
}, append(labels, "status")).With(labelsAndValues...),
ProposalCreateCount: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "proposal_create_count",
Help: "Total number of proposals created by the node since process start.",
}, labels).With(labelsAndValues...),
RoundVotingPowerPercent: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "round_voting_power_percent",
Help: "A value between 0 and 1.0 representing the percentage of the total voting power per vote type received within a round.",
}, append(labels, "vote_type")).With(labelsAndValues...),
LateVotes: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "late_votes",
Help: "Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in.",
}, append(labels, "vote_type")).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
Height: discard.NewGauge(),
ValidatorLastSignedHeight: discard.NewGauge(),
Rounds: discard.NewGauge(),
RoundDuration: discard.NewHistogram(),
Validators: discard.NewGauge(),
ValidatorsPower: discard.NewGauge(),
ValidatorPower: discard.NewGauge(),
ValidatorMissedBlocks: discard.NewGauge(),
MissingValidators: discard.NewGauge(),
MissingValidatorsPower: discard.NewGauge(),
ByzantineValidators: discard.NewGauge(),
ByzantineValidatorsPower: discard.NewGauge(),
BlockIntervalSeconds: discard.NewHistogram(),
NumTxs: discard.NewGauge(),
BlockSizeBytes: discard.NewHistogram(),
TotalTxs: discard.NewGauge(),
CommittedHeight: discard.NewGauge(),
BlockSyncing: discard.NewGauge(),
StateSyncing: discard.NewGauge(),
BlockParts: discard.NewCounter(),
StepDuration: discard.NewHistogram(),
BlockGossipReceiveLatency: discard.NewHistogram(),
BlockGossipPartsReceived: discard.NewCounter(),
QuorumPrevoteDelay: discard.NewGauge(),
FullPrevoteDelay: discard.NewGauge(),
ProposalTimestampDifference: discard.NewHistogram(),
VoteExtensionReceiveCount: discard.NewCounter(),
ProposalReceiveCount: discard.NewCounter(),
ProposalCreateCount: discard.NewCounter(),
RoundVotingPowerPercent: discard.NewGauge(),
LateVotes: discard.NewCounter(),
}
}
internal/consensus/metrics.go
+82 -230
View File
@@ -5,13 +5,10 @@ import (
"time"
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
cstypes "github.com/tendermint/tendermint/internal/consensus/types"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
"github.com/tendermint/tendermint/types"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
const (
@@ -20,28 +17,30 @@ const (
MetricsSubsystem = "consensus"
)
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics contains metrics exposed by this package.
type Metrics struct {
// Height of the chain.
Height metrics.Gauge
// ValidatorLastSignedHeight of a validator.
ValidatorLastSignedHeight metrics.Gauge
// Last height signed by this validator if the node is a validator.
ValidatorLastSignedHeight metrics.Gauge `metrics_labels:"validator_address"`
// Number of rounds.
Rounds metrics.Gauge
// Histogram of round duration.
RoundDuration metrics.Histogram
RoundDuration metrics.Histogram `metrics_buckettype:"exprange" metrics_bucketsizes:"0.1, 100, 8"`
// Number of validators.
Validators metrics.Gauge
// Total power of all validators.
ValidatorsPower metrics.Gauge
// Power of a validator.
ValidatorPower metrics.Gauge
// Amount of blocks missed by a validator.
ValidatorMissedBlocks metrics.Gauge
ValidatorPower metrics.Gauge `metrics_labels:"validator_address"`
// Amount of blocks missed per validator.
ValidatorMissedBlocks metrics.Gauge `metrics_labels:"validator_address"`
// Number of validators who did not sign.
MissingValidators metrics.Gauge
// Total power of the missing validators.
@@ -61,27 +60,27 @@ type Metrics struct {
// Total number of transactions.
TotalTxs metrics.Gauge
// The latest block height.
CommittedHeight metrics.Gauge
CommittedHeight metrics.Gauge `metrics_name:"latest_block_height"`
// Whether or not a node is block syncing. 1 if yes, 0 if no.
BlockSyncing metrics.Gauge
// Whether or not a node is state syncing. 1 if yes, 0 if no.
StateSyncing metrics.Gauge
// Number of blockparts transmitted by peer.
BlockParts metrics.Counter
// Number of block parts transmitted by each peer.
BlockParts metrics.Counter `metrics_labels:"peer_id"`
// Histogram of step duration.
StepDuration metrics.Histogram
// Histogram of durations for each step in the consensus protocol.
StepDuration metrics.Histogram `metrics_labels:"step" metrics_buckettype:"exprange" metrics_bucketsizes:"0.1, 100, 8"`
stepStart time.Time
// Histogram of time taken to receive a block in seconds, measured between when a new block is first
// discovered to when the block is completed.
BlockGossipReceiveLatency metrics.Histogram
BlockGossipReceiveLatency metrics.Histogram `metrics_buckettype:"exprange" metrics_bucketsizes:"0.1, 100, 8"`
blockGossipStart time.Time
// Number of block parts received by the node, separated by whether the part
// was relevant to the block the node is trying to gather or not.
BlockGossipPartsReceived metrics.Counter
BlockGossipPartsReceived metrics.Counter `metrics_labels:"matches_current"`
// QuroumPrevoteMessageDelay is the interval in seconds between the proposal
// timestamp and the timestamp of the earliest prevote that achieved a quorum
@@ -92,232 +91,50 @@ type Metrics struct {
// be above 2/3 of the total voting power of the network defines the endpoint
// the endpoint of the interval. Subtract the proposal timestamp from this endpoint
// to obtain the quorum delay.
QuorumPrevoteDelay metrics.Gauge
//metrics:Interval in seconds between the proposal timestamp and the timestamp of the earliest prevote that achieved a quorum.
QuorumPrevoteDelay metrics.Gauge `metrics_labels:"proposer_address"`
// FullPrevoteDelay is the interval in seconds between the proposal
// timestamp and the timestamp of the latest prevote in a round where 100%
// of the voting power on the network issued prevotes.
FullPrevoteDelay metrics.Gauge
//metrics:Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted.
FullPrevoteDelay metrics.Gauge `metrics_labels:"proposer_address"`
// ProposalTimestampDifference is the difference between the timestamp in
// the proposal message and the local time of the validator at the time
// that the validator received the message.
ProposalTimestampDifference metrics.Histogram
}
//metrics:Difference between the timestamp in the proposal message and the local time of the validator at the time it received the message.
ProposalTimestampDifference metrics.Histogram `metrics_labels:"is_timely" metrics_bucketsizes:"-10, -.5, -.025, 0, .1, .5, 1, 1.5, 2, 10"`
// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "height",
Help: "Height of the chain.",
}, labels).With(labelsAndValues...),
Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "rounds",
Help: "Number of rounds.",
}, labels).With(labelsAndValues...),
RoundDuration: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "round_duration",
Help: "Time spent in a round.",
Buckets: stdprometheus.ExponentialBucketsRange(0.1, 100, 8),
}, labels).With(labelsAndValues...),
Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validators",
Help: "Number of validators.",
}, labels).With(labelsAndValues...),
ValidatorLastSignedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validator_last_signed_height",
Help: "Last signed height for a validator",
}, append(labels, "validator_address")).With(labelsAndValues...),
ValidatorMissedBlocks: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validator_missed_blocks",
Help: "Total missed blocks for a validator",
}, append(labels, "validator_address")).With(labelsAndValues...),
ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validators_power",
Help: "Total power of all validators.",
}, labels).With(labelsAndValues...),
ValidatorPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validator_power",
Help: "Power of a validator",
}, append(labels, "validator_address")).With(labelsAndValues...),
MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "missing_validators",
Help: "Number of validators who did not sign.",
}, labels).With(labelsAndValues...),
MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "missing_validators_power",
Help: "Total power of the missing validators.",
}, labels).With(labelsAndValues...),
ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "byzantine_validators",
Help: "Number of validators who tried to double sign.",
}, labels).With(labelsAndValues...),
ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "byzantine_validators_power",
Help: "Total power of the byzantine validators.",
}, labels).With(labelsAndValues...),
BlockIntervalSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_interval_seconds",
Help: "Time between this and the last block.",
}, labels).With(labelsAndValues...),
NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "num_txs",
Help: "Number of transactions.",
}, labels).With(labelsAndValues...),
BlockSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_size_bytes",
Help: "Size of the block.",
}, labels).With(labelsAndValues...),
TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "total_txs",
Help: "Total number of transactions.",
}, labels).With(labelsAndValues...),
CommittedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "latest_block_height",
Help: "The latest block height.",
}, labels).With(labelsAndValues...),
BlockSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_syncing",
Help: "Whether or not a node is block syncing. 1 if yes, 0 if no.",
}, labels).With(labelsAndValues...),
StateSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "state_syncing",
Help: "Whether or not a node is state syncing. 1 if yes, 0 if no.",
}, labels).With(labelsAndValues...),
BlockParts: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_parts",
Help: "Number of blockparts transmitted by peer.",
}, append(labels, "peer_id")).With(labelsAndValues...),
BlockGossipReceiveLatency: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_gossip_receive_latency",
Help: "Difference in seconds between when the validator learns of a new block" +
"and when the validator receives the last piece of the block.",
Buckets: stdprometheus.ExponentialBucketsRange(0.1, 100, 8),
}, labels).With(labelsAndValues...),
BlockGossipPartsReceived: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_gossip_parts_received",
Help: "Number of block parts received by the node, labeled by whether the " +
"part was relevant to the block the node was currently gathering or not.",
}, append(labels, "matches_current")).With(labelsAndValues...),
StepDuration: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "step_duration",
Help: "Time spent per step.",
Buckets: stdprometheus.ExponentialBucketsRange(0.1, 100, 8),
}, append(labels, "step")).With(labelsAndValues...),
QuorumPrevoteDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "quorum_prevote_delay",
Help: "Difference in seconds between the proposal timestamp and the timestamp " +
"of the latest prevote that achieved a quorum in the prevote step.",
}, append(labels, "proposer_address")).With(labelsAndValues...),
FullPrevoteDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "full_prevote_delay",
Help: "Difference in seconds between the proposal timestamp and the timestamp " +
"of the latest prevote that achieved 100% of the voting power in the prevote step.",
}, append(labels, "proposer_address")).With(labelsAndValues...),
ProposalTimestampDifference: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "proposal_timestamp_difference",
Help: "Difference in seconds between the timestamp in the proposal " +
"message and the local time when the message was received. " +
"Only calculated when a new block is proposed.",
Buckets: []float64{-10, -.5, -.025, 0, .1, .5, 1, 1.5, 2, 10},
}, append(labels, "is_timely")).With(labelsAndValues...),
}
}
// VoteExtensionReceiveCount is the number of vote extensions received by this
// node. The metric is annotated by the status of the vote extension from the
// application, either 'accepted' or 'rejected'.
//metrics:Number of vote extensions received labeled by application response status.
VoteExtensionReceiveCount metrics.Counter `metrics_labels:"status"`
// NopMetrics returns no-op Metrics.
func NopMetrics() *Metrics {
return &Metrics{
Height: discard.NewGauge(),
// ProposalReceiveCount is the total number of proposals received by this node
// since process start.
// The metric is annotated by the status of the proposal from the application,
// either 'accepted' or 'rejected'.
//metrics:Total number of proposals received by the node since process start labeled by application response status.
ProposalReceiveCount metrics.Counter `metrics_labels:"status"`
ValidatorLastSignedHeight: discard.NewGauge(),
// ProposalCreationCount is the total number of proposals created by this node
// since process start.
//metrics:Total number of proposals created by the node since process start.
ProposalCreateCount metrics.Counter
Rounds: discard.NewGauge(),
RoundDuration: discard.NewHistogram(),
StepDuration: discard.NewHistogram(),
// RoundVotingPowerPercent is the percentage of the total voting power received
// with a round. The value begins at 0 for each round and approaches 1.0 as
// additional voting power is observed. The metric is labeled by vote type.
//metrics:A value between 0 and 1.0 representing the percentage of the total voting power per vote type received within a round.
RoundVotingPowerPercent metrics.Gauge `metrics_labels:"vote_type"`
Validators: discard.NewGauge(),
ValidatorsPower: discard.NewGauge(),
ValidatorPower: discard.NewGauge(),
ValidatorMissedBlocks: discard.NewGauge(),
MissingValidators: discard.NewGauge(),
MissingValidatorsPower: discard.NewGauge(),
ByzantineValidators: discard.NewGauge(),
ByzantineValidatorsPower: discard.NewGauge(),
BlockIntervalSeconds: discard.NewHistogram(),
NumTxs: discard.NewGauge(),
BlockSizeBytes: discard.NewHistogram(),
TotalTxs: discard.NewGauge(),
CommittedHeight: discard.NewGauge(),
BlockSyncing: discard.NewGauge(),
StateSyncing: discard.NewGauge(),
BlockParts: discard.NewCounter(),
BlockGossipReceiveLatency: discard.NewHistogram(),
BlockGossipPartsReceived: discard.NewCounter(),
QuorumPrevoteDelay: discard.NewGauge(),
FullPrevoteDelay: discard.NewGauge(),
ProposalTimestampDifference: discard.NewHistogram(),
}
// LateVotes stores the number of votes that were received by this node that
// correspond to earlier heights and rounds than this node is currently
// in.
//metrics:Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in.
LateVotes metrics.Counter `metrics_labels:"vote_type"`
}
// RecordConsMetrics uses for recording the block related metrics during fast-sync.
@@ -336,10 +153,45 @@ func (m *Metrics) MarkBlockGossipComplete() {
m.BlockGossipReceiveLatency.Observe(time.Since(m.blockGossipStart).Seconds())
}
func (m *Metrics) MarkProposalProcessed(accepted bool) {
status := "accepted"
if !accepted {
status = "rejected"
}
m.ProposalReceiveCount.With("status", status).Add(1)
}
func (m *Metrics) MarkVoteExtensionReceived(accepted bool) {
status := "accepted"
if !accepted {
status = "rejected"
}
m.VoteExtensionReceiveCount.With("status", status).Add(1)
}
func (m *Metrics) MarkVoteReceived(vt tmproto.SignedMsgType, power, totalPower int64) {
p := float64(power) / float64(totalPower)
n := strings.ToLower(strings.TrimPrefix(vt.String(), "SIGNED_MSG_TYPE_"))
m.RoundVotingPowerPercent.With("vote_type", n).Add(p)
}
func (m *Metrics) MarkRound(r int32, st time.Time) {
m.Rounds.Set(float64(r))
roundTime := time.Since(st).Seconds()
m.RoundDuration.Observe(roundTime)
pvt := tmproto.PrevoteType
pvn := strings.ToLower(strings.TrimPrefix(pvt.String(), "SIGNED_MSG_TYPE_"))
m.RoundVotingPowerPercent.With("vote_type", pvn).Set(0)
pct := tmproto.PrecommitType
pcn := strings.ToLower(strings.TrimPrefix(pct.String(), "SIGNED_MSG_TYPE_"))
m.RoundVotingPowerPercent.With("vote_type", pcn).Set(0)
}
func (m *Metrics) MarkLateVote(vt tmproto.SignedMsgType) {
n := strings.ToLower(strings.TrimPrefix(vt.String(), "SIGNED_MSG_TYPE_"))
m.LateVotes.With("vote_type", n).Add(1)
}
func (m *Metrics) MarkStep(s cstypes.RoundStepType) {
internal/consensus/msgs.go
+1 -5
View File
@@ -222,11 +222,7 @@ func (*VoteMessage) TypeTag() string { return "tendermint/Vote" }
// ValidateBasic checks whether the vote within the message is well-formed.
func (m *VoteMessage) ValidateBasic() error {
// Here we validate votes with vote extensions, since we require vote
// extensions to be sent in precommit messages during consensus. Prevote
// messages should never have vote extensions, and this is also validated
// here.
return m.Vote.ValidateWithExtension()
return m.Vote.ValidateBasic()
}
// String returns a string representation.
internal/consensus/msgs_test.go
+1 -1
View File
@@ -67,7 +67,7 @@ func TestMsgToProto(t *testing.T) {
pv := types.NewMockPV()
vote, err := factory.MakeVote(ctx, pv, factory.DefaultTestChainID,
0, 1, 0, 2, types.BlockID{}, time.Now())
0, 1, 0, 2, bi, time.Now())
require.NoError(t, err)
pbVote := vote.ToProto()
internal/consensus/reactor.go
+17 -8
View File
@@ -216,6 +216,8 @@ func (r *Reactor) OnStart(ctx context.Context) error {
if err := r.state.Start(ctx); err != nil {
return err
}
} else if err := r.state.updateStateFromStore(); err != nil {
return err
}
go r.updateRoundStateRoutine(ctx)
@@ -794,15 +796,22 @@ func (r *Reactor) gossipVotesRoutine(ctx context.Context, ps *PeerState, voteCh
// catchup logic -- if peer is lagging by more than 1, send Commit
blockStoreBase := r.state.blockStore.Base()
if blockStoreBase > 0 && prs.Height != 0 && rs.Height >= prs.Height+2 && prs.Height >= blockStoreBase {
// Load the block commit for prs.Height, which contains precommit
// Load the block's extended commit for prs.Height, which contains precommit
// signatures for prs.Height.
if commit := r.state.blockStore.LoadBlockCommit(prs.Height); commit != nil {
if ok, err := r.pickSendVote(ctx, ps, commit, voteCh); err != nil {
return
} else if ok {
logger.Debug("picked Catchup commit to send", "height", prs.Height)
continue
}
var ec *types.ExtendedCommit
if r.state.state.ConsensusParams.ABCI.VoteExtensionsEnabled(prs.Height) {
ec = r.state.blockStore.LoadBlockExtendedCommit(prs.Height)
} else {
ec = r.state.blockStore.LoadBlockCommit(prs.Height).WrappedExtendedCommit()
}
if ec == nil {
continue
}
if ok, err := r.pickSendVote(ctx, ps, ec, voteCh); err != nil {
return
} else if ok {
logger.Debug("picked Catchup commit to send", "height", prs.Height)
continue
}
}
internal/consensus/reactor_test.go
+113
View File
@@ -32,6 +32,7 @@ import (
"github.com/tendermint/tendermint/internal/test/factory"
"github.com/tendermint/tendermint/libs/log"
tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
"github.com/tendermint/tendermint/types"
)
@@ -600,6 +601,118 @@ func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) {
wg.Wait()
}
// TestSwitchToConsensusVoteExtensions tests that the SwitchToConsensus correctly
// checks for vote extension data when required.
func TestSwitchToConsensusVoteExtensions(t *testing.T) {
for _, testCase := range []struct {
name string
storedHeight int64
initialRequiredHeight int64
includeExtensions bool
shouldPanic bool
}{
{
name: "no vote extensions but not required",
initialRequiredHeight: 0,
storedHeight: 2,
includeExtensions: false,
shouldPanic: false,
},
{
name: "no vote extensions but required this height",
initialRequiredHeight: 2,
storedHeight: 2,
includeExtensions: false,
shouldPanic: true,
},
{
name: "no vote extensions and required in future",
initialRequiredHeight: 3,
storedHeight: 2,
includeExtensions: false,
shouldPanic: false,
},
{
name: "no vote extensions and required previous height",
initialRequiredHeight: 1,
storedHeight: 2,
includeExtensions: false,
shouldPanic: true,
},
{
name: "vote extensions and required previous height",
initialRequiredHeight: 1,
storedHeight: 2,
includeExtensions: true,
shouldPanic: false,
},
} {
t.Run(testCase.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
defer cancel()
cs, vs := makeState(ctx, t, makeStateArgs{validators: 1})
validator := vs[0]
validator.Height = testCase.storedHeight
cs.state.LastBlockHeight = testCase.storedHeight
cs.state.LastValidators = cs.state.Validators.Copy()
cs.state.ConsensusParams.ABCI.VoteExtensionsEnableHeight = testCase.initialRequiredHeight
propBlock, err := cs.createProposalBlock(ctx)
require.NoError(t, err)
// Consensus is preparing to do the next height after the stored height.
cs.Height = testCase.storedHeight + 1
propBlock.Height = testCase.storedHeight
blockParts, err := propBlock.MakePartSet(types.BlockPartSizeBytes)
require.NoError(t, err)
var voteSet *types.VoteSet
if testCase.includeExtensions {
voteSet = types.NewExtendedVoteSet(cs.state.ChainID, testCase.storedHeight, 0, tmproto.PrecommitType, cs.state.Validators)
} else {
voteSet = types.NewVoteSet(cs.state.ChainID, testCase.storedHeight, 0, tmproto.PrecommitType, cs.state.Validators)
}
signedVote := signVote(ctx, t, validator, tmproto.PrecommitType, cs.state.ChainID, types.BlockID{
Hash: propBlock.Hash(),
PartSetHeader: blockParts.Header(),
})
if !testCase.includeExtensions {
signedVote.Extension = nil
signedVote.ExtensionSignature = nil
}
added, err := voteSet.AddVote(signedVote)
require.NoError(t, err)
require.True(t, added)
if testCase.includeExtensions {
cs.blockStore.SaveBlockWithExtendedCommit(propBlock, blockParts, voteSet.MakeExtendedCommit())
} else {
cs.blockStore.SaveBlock(propBlock, blockParts, voteSet.MakeExtendedCommit().ToCommit())
}
reactor := NewReactor(
log.NewNopLogger(),
cs,
nil,
nil,
cs.eventBus,
true,
NopMetrics(),
)
if testCase.shouldPanic {
assert.Panics(t, func() {
reactor.SwitchToConsensus(ctx, cs.state, false)
})
} else {
reactor.SwitchToConsensus(ctx, cs.state, false)
}
})
}
}
func TestReactorRecordsVotesAndBlockParts(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
internal/consensus/replay_test.go
+41 -29
View File
@@ -297,7 +297,7 @@ type simulatorTestSuite struct {
GenesisState sm.State
Config *config.Config
Chain []*types.Block
Commits []*types.Commit
ExtCommits []*types.ExtendedCommit
CleanupFunc cleanupFunc
Mempool mempool.Mempool
@@ -578,11 +578,11 @@ func setupSimulator(ctx context.Context, t *testing.T) *simulatorTestSuite {
}
ensureNewRound(t, newRoundCh, height+1, 0)
sim.Chain = make([]*types.Block, 0)
sim.Commits = make([]*types.Commit, 0)
sim.Chain = []*types.Block{}
sim.ExtCommits = []*types.ExtendedCommit{}
for i := 1; i <= numBlocks; i++ {
sim.Chain = append(sim.Chain, css[0].blockStore.LoadBlock(int64(i)))
sim.Commits = append(sim.Commits, css[0].blockStore.LoadBlockCommit(int64(i)))
sim.ExtCommits = append(sim.ExtCommits, css[0].blockStore.LoadBlockExtendedCommit(int64(i)))
}
return sim
@@ -679,7 +679,7 @@ func testHandshakeReplay(
testValidatorsChange bool,
) {
var chain []*types.Block
var commits []*types.Commit
var extCommits []*types.ExtendedCommit
var store *mockBlockStore
var stateDB dbm.DB
var genesisState sm.State
@@ -699,7 +699,7 @@ func testHandshakeReplay(
genesisState = sim.GenesisState
cfg = sim.Config
chain = append([]*types.Block{}, sim.Chain...) // copy chain
commits = sim.Commits
extCommits = sim.ExtCommits
store = newMockBlockStore(t, cfg, genesisState.ConsensusParams)
} else { // test single node
testConfig, err := ResetConfig(t.TempDir(), fmt.Sprintf("%s_%v_s", t.Name(), mode))
@@ -718,7 +718,7 @@ func testHandshakeReplay(
err = wal.Start(ctx)
require.NoError(t, err)
t.Cleanup(func() { cancel(); wal.Wait() })
chain, commits = makeBlockchainFromWAL(t, wal)
chain, extCommits = makeBlockchainFromWAL(t, wal)
pubKey, err := privVal.GetPubKey(ctx)
require.NoError(t, err)
stateDB, genesisState, store = stateAndStore(t, cfg, pubKey, kvstore.ProtocolVersion)
@@ -726,7 +726,7 @@ func testHandshakeReplay(
}
stateStore := sm.NewStore(stateDB)
store.chain = chain
store.commits = commits
store.extCommits = extCommits
state := genesisState.Copy()
// run the chain through state.ApplyBlock to build up the tendermint state
@@ -1034,7 +1034,7 @@ func (app *badApp) Commit(context.Context) (*abci.ResponseCommit, error) {
//--------------------------
// utils for making blocks
func makeBlockchainFromWAL(t *testing.T, wal WAL) ([]*types.Block, []*types.Commit) {
func makeBlockchainFromWAL(t *testing.T, wal WAL) ([]*types.Block, []*types.ExtendedCommit) {
t.Helper()
var height int64
@@ -1047,10 +1047,10 @@ func makeBlockchainFromWAL(t *testing.T, wal WAL) ([]*types.Block, []*types.Comm
// log.Notice("Build a blockchain by reading from the WAL")
var (
blocks []*types.Block
commits []*types.Commit
thisBlockParts *types.PartSet
thisBlockCommit *types.Commit
blocks []*types.Block
extCommits []*types.ExtendedCommit
thisBlockParts *types.PartSet
thisBlockExtCommit *types.ExtendedCommit
)
dec := NewWALDecoder(gr)
@@ -1082,12 +1082,12 @@ func makeBlockchainFromWAL(t *testing.T, wal WAL) ([]*types.Block, []*types.Comm
require.Equal(t, block.Height, height+1,
"read bad block from wal. got height %d, expected %d", block.Height, height+1)
commitHeight := thisBlockCommit.Height
commitHeight := thisBlockExtCommit.Height
require.Equal(t, commitHeight, height+1,
"commit doesnt match. got height %d, expected %d", commitHeight, height+1)
blocks = append(blocks, block)
commits = append(commits, thisBlockCommit)
extCommits = append(extCommits, thisBlockExtCommit)
height++
}
case *types.PartSetHeader:
@@ -1097,8 +1097,12 @@ func makeBlockchainFromWAL(t *testing.T, wal WAL) ([]*types.Block, []*types.Comm
require.NoError(t, err)
case *types.Vote:
if p.Type == tmproto.PrecommitType {
thisBlockCommit = types.NewCommit(p.Height, p.Round,
p.BlockID, []types.CommitSig{p.CommitSig()})
thisBlockExtCommit = &types.ExtendedCommit{
Height: p.Height,
Round: p.Round,
BlockID: p.BlockID,
ExtendedSignatures: []types.ExtendedCommitSig{p.ExtendedCommitSig()},
}
}
}
}
@@ -1113,12 +1117,12 @@ func makeBlockchainFromWAL(t *testing.T, wal WAL) ([]*types.Block, []*types.Comm
require.NoError(t, err)
require.Equal(t, block.Height, height+1, "read bad block from wal. got height %d, expected %d", block.Height, height+1)
commitHeight := thisBlockCommit.Height
commitHeight := thisBlockExtCommit.Height
require.Equal(t, commitHeight, height+1, "commit does not match. got height %d, expected %d", commitHeight, height+1)
blocks = append(blocks, block)
commits = append(commits, thisBlockCommit)
return blocks, commits
extCommits = append(extCommits, thisBlockExtCommit)
return blocks, extCommits
}
func readPieceFromWAL(msg *TimedWALMessage) interface{} {
@@ -1162,14 +1166,16 @@ func stateAndStore(
// mock block store
type mockBlockStore struct {
cfg *config.Config
params types.ConsensusParams
chain []*types.Block
commits []*types.Commit
base int64
t *testing.T
cfg *config.Config
params types.ConsensusParams
chain []*types.Block
extCommits []*types.ExtendedCommit
base int64
t *testing.T
}
var _ sm.BlockStore = &mockBlockStore{}
// TODO: NewBlockStore(db.NewMemDB) ...
func newMockBlockStore(t *testing.T, cfg *config.Config, params types.ConsensusParams) *mockBlockStore {
return &mockBlockStore{
@@ -1198,20 +1204,26 @@ func (bs *mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
}
}
func (bs *mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil }
func (bs *mockBlockStore) SaveBlockWithExtendedCommit(block *types.Block, blockParts *types.PartSet, seenCommit *types.ExtendedCommit) {
}
func (bs *mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
}
func (bs *mockBlockStore) LoadBlockCommit(height int64) *types.Commit {
return bs.commits[height-1]
return bs.extCommits[height-1].ToCommit()
}
func (bs *mockBlockStore) LoadSeenCommit() *types.Commit {
return bs.commits[len(bs.commits)-1]
return bs.extCommits[len(bs.extCommits)-1].ToCommit()
}
func (bs *mockBlockStore) LoadBlockExtendedCommit(height int64) *types.ExtendedCommit {
return bs.extCommits[height-1]
}
func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) {
pruned := uint64(0)
for i := int64(0); i < height-1; i++ {
bs.chain[i] = nil
bs.commits[i] = nil
bs.extCommits[i] = nil
pruned++
}
bs.base = height
internal/consensus/state.go
+139 -51
View File
@@ -122,9 +122,8 @@ type State struct {
// store blocks and commits
blockStore sm.BlockStore
stateStore sm.Store
initialStatePopulated bool
skipBootstrapping bool
stateStore sm.Store
skipBootstrapping bool
// create and execute blocks
blockExec *sm.BlockExecutor
@@ -248,9 +247,6 @@ func NewState(
}
func (cs *State) updateStateFromStore() error {
if cs.initialStatePopulated {
return nil
}
state, err := cs.stateStore.Load()
if err != nil {
return fmt.Errorf("loading state: %w", err)
@@ -259,6 +255,15 @@ func (cs *State) updateStateFromStore() error {
return nil
}
eq, err := state.Equals(cs.state)
if err != nil {
return fmt.Errorf("comparing state: %w", err)
}
// if the new state is equivalent to the old state, we should not trigger a state update.
if eq {
return nil
}
// We have no votes, so reconstruct LastCommit from SeenCommit.
if state.LastBlockHeight > 0 {
cs.reconstructLastCommit(state)
@@ -266,7 +271,6 @@ func (cs *State) updateStateFromStore() error {
cs.updateToState(state)
cs.initialStatePopulated = true
return nil
}
@@ -692,27 +696,54 @@ func (cs *State) sendInternalMessage(ctx context.Context, mi msgInfo) {
}
}
// Reconstruct LastCommit from SeenCommit, which we saved along with the block,
// (which happens even before saving the state)
// Reconstruct the LastCommit from either SeenCommit or the ExtendedCommit. SeenCommit
// and ExtendedCommit are saved along with the block. If VoteExtensions are required
// the method will panic on an absent ExtendedCommit or an ExtendedCommit without
// extension data.
func (cs *State) reconstructLastCommit(state sm.State) {
extensionsEnabled := cs.state.ConsensusParams.ABCI.VoteExtensionsEnabled(state.LastBlockHeight)
if !extensionsEnabled {
votes, err := cs.votesFromSeenCommit(state)
if err != nil {
panic(fmt.Sprintf("failed to reconstruct last commit; %s", err))
}
cs.LastCommit = votes
return
}
votes, err := cs.votesFromExtendedCommit(state)
if err != nil {
panic(fmt.Sprintf("failed to reconstruct last extended commit; %s", err))
}
cs.LastCommit = votes
}
func (cs *State) votesFromExtendedCommit(state sm.State) (*types.VoteSet, error) {
ec := cs.blockStore.LoadBlockExtendedCommit(state.LastBlockHeight)
if ec == nil {
return nil, fmt.Errorf("extended commit for height %v not found", state.LastBlockHeight)
}
vs := ec.ToExtendedVoteSet(state.ChainID, state.LastValidators)
if !vs.HasTwoThirdsMajority() {
return nil, errors.New("extended commit does not have +2/3 majority")
}
return vs, nil
}
func (cs *State) votesFromSeenCommit(state sm.State) (*types.VoteSet, error) {
commit := cs.blockStore.LoadSeenCommit()
if commit == nil || commit.Height != state.LastBlockHeight {
commit = cs.blockStore.LoadBlockCommit(state.LastBlockHeight)
}
if commit == nil {
panic(fmt.Sprintf(
"failed to reconstruct last commit; commit for height %v not found",
state.LastBlockHeight,
))
return nil, fmt.Errorf("commit for height %v not found", state.LastBlockHeight)
}
lastPrecommits := types.CommitToVoteSet(state.ChainID, commit, state.LastValidators)
if !lastPrecommits.HasTwoThirdsMajority() {
panic("failed to reconstruct last commit; does not have +2/3 maj")
vs := commit.ToVoteSet(state.ChainID, state.LastValidators)
if !vs.HasTwoThirdsMajority() {
return nil, errors.New("commit does not have +2/3 majority")
}
cs.LastCommit = lastPrecommits
return vs, nil
}
// Updates State and increments height to match that of state.
@@ -814,7 +845,11 @@ func (cs *State) updateToState(state sm.State) {
cs.ValidRound = -1
cs.ValidBlock = nil
cs.ValidBlockParts = nil
cs.Votes = cstypes.NewHeightVoteSet(state.ChainID, height, validators)
if state.ConsensusParams.ABCI.VoteExtensionsEnabled(height) {
cs.Votes = cstypes.NewExtendedHeightVoteSet(state.ChainID, height, validators)
} else {
cs.Votes = cstypes.NewHeightVoteSet(state.ChainID, height, validators)
}
cs.CommitRound = -1
cs.LastValidators = state.LastValidators
cs.TriggeredTimeoutPrecommit = false
@@ -1334,6 +1369,7 @@ func (cs *State) defaultDecideProposal(ctx context.Context, height int64, round
} else if block == nil {
return
}
cs.metrics.ProposalCreateCount.Add(1)
blockParts, err = block.MakePartSet(types.BlockPartSizeBytes)
if err != nil {
cs.logger.Error("unable to create proposal block part set", "error", err)
@@ -1400,16 +1436,17 @@ func (cs *State) createProposalBlock(ctx context.Context) (*types.Block, error)
return nil, errors.New("entered createProposalBlock with privValidator being nil")
}
var commit *types.Commit
// TODO(sergio): wouldn't it be easier if CreateProposalBlock accepted cs.LastCommit directly?
var lastExtCommit *types.ExtendedCommit
switch {
case cs.Height == cs.state.InitialHeight:
// We're creating a proposal for the first block.
// The commit is empty, but not nil.
commit = types.NewCommit(0, 0, types.BlockID{}, nil)
lastExtCommit = &types.ExtendedCommit{}
case cs.LastCommit.HasTwoThirdsMajority():
// Make the commit from LastCommit
commit = cs.LastCommit.MakeCommit()
lastExtCommit = cs.LastCommit.MakeExtendedCommit()
default: // This shouldn't happen.
cs.logger.Error("propose step; cannot propose anything without commit for the previous block")
@@ -1425,7 +1462,7 @@ func (cs *State) createProposalBlock(ctx context.Context) (*types.Block, error)
proposerAddr := cs.privValidatorPubKey.Address()
ret, err := cs.blockExec.CreateProposalBlock(ctx, cs.Height, cs.state, commit, proposerAddr, cs.LastCommit.GetVotes())
ret, err := cs.blockExec.CreateProposalBlock(ctx, cs.Height, cs.state, lastExtCommit, proposerAddr)
if err != nil {
panic(err)
}
@@ -1531,6 +1568,7 @@ func (cs *State) defaultDoPrevote(ctx context.Context, height int64, round int32
if err != nil {
panic(fmt.Sprintf("ProcessProposal: %v", err))
}
cs.metrics.MarkProposalProcessed(isAppValid)
// Vote nil if the Application rejected the block
if !isAppValid {
@@ -1922,9 +1960,12 @@ func (cs *State) finalizeCommit(ctx context.Context, height int64) {
if cs.blockStore.Height() < block.Height {
// NOTE: the seenCommit is local justification to commit this block,
// but may differ from the LastCommit included in the next block
precommits := cs.Votes.Precommits(cs.CommitRound)
seenCommit := precommits.MakeCommit()
cs.blockStore.SaveBlock(block, blockParts, seenCommit)
seenExtendedCommit := cs.Votes.Precommits(cs.CommitRound).MakeExtendedCommit()
if cs.state.ConsensusParams.ABCI.VoteExtensionsEnabled(block.Height) {
cs.blockStore.SaveBlockWithExtendedCommit(block, blockParts, seenExtendedCommit)
} else {
cs.blockStore.SaveBlock(block, blockParts, seenExtendedCommit.ToCommit())
}
} else {
// Happens during replay if we already saved the block but didn't commit
logger.Debug("calling finalizeCommit on already stored block", "height", block.Height)
@@ -2026,7 +2067,7 @@ func (cs *State) RecordMetrics(height int64, block *types.Block) {
for i, val := range cs.LastValidators.Validators {
commitSig := block.LastCommit.Signatures[i]
if commitSig.Absent() {
if commitSig.BlockIDFlag == types.BlockIDFlagAbsent {
missingValidators++
missingValidatorsPower += val.VotingPower
}
@@ -2036,7 +2077,7 @@ func (cs *State) RecordMetrics(height int64, block *types.Block) {
"validator_address", val.Address.String(),
}
cs.metrics.ValidatorPower.With(label...).Set(float64(val.VotingPower))
if commitSig.ForBlock() {
if commitSig.BlockIDFlag == types.BlockIDFlagCommit {
cs.metrics.ValidatorLastSignedHeight.With(label...).Set(float64(height))
} else {
cs.metrics.ValidatorMissedBlocks.With(label...).Add(float64(1))
@@ -2297,6 +2338,10 @@ func (cs *State) addVote(
"cs_height", cs.Height,
)
if vote.Height < cs.Height || (vote.Height == cs.Height && vote.Round < cs.Round) {
cs.metrics.MarkLateVote(vote.Type)
}
// A precommit for the previous height?
// These come in while we wait timeoutCommit
if vote.Height+1 == cs.Height && vote.Type == tmproto.PrecommitType {
@@ -2335,10 +2380,45 @@ func (cs *State) addVote(
return
}
// Verify VoteExtension if precommit
if vote.Type == tmproto.PrecommitType {
if err = cs.blockExec.VerifyVoteExtension(ctx, vote); err != nil {
return false, err
// Check to see if the chain is configured to extend votes.
if cs.state.ConsensusParams.ABCI.VoteExtensionsEnabled(cs.Height) {
// The chain is configured to extend votes, check that the vote is
// not for a nil block and verify the extensions signature against the
// corresponding public key.
var myAddr []byte
if cs.privValidatorPubKey != nil {
myAddr = cs.privValidatorPubKey.Address()
}
// Verify VoteExtension if precommit and not nil
// https://github.com/tendermint/tendermint/issues/8487
if vote.Type == tmproto.PrecommitType && !vote.BlockID.IsNil() &&
!bytes.Equal(vote.ValidatorAddress, myAddr) { // Skip the VerifyVoteExtension call if the vote was issued by this validator.
// The core fields of the vote message were already validated in the
// consensus reactor when the vote was received.
// Here, we verify the signature of the vote extension included in the vote
// message.
_, val := cs.state.Validators.GetByIndex(vote.ValidatorIndex)
if err := vote.VerifyExtension(cs.state.ChainID, val.PubKey); err != nil {
return false, err
}
err := cs.blockExec.VerifyVoteExtension(ctx, vote)
cs.metrics.MarkVoteExtensionReceived(err == nil)
if err != nil {
return false, err
}
}
} else {
// Vote extensions are not enabled on the network.
// strip the extension data from the vote in case any is present.
//
// TODO punish a peer if it sent a vote with an extension when the feature
// is disabled on the network.
// https://github.com/tendermint/tendermint/issues/8565
if stripped := vote.StripExtension(); stripped {
cs.logger.Error("vote included extension data but vote extensions are not enabled", "peer", peerID)
}
}
@@ -2348,6 +2428,11 @@ func (cs *State) addVote(
// Either duplicate, or error upon cs.Votes.AddByIndex()
return
}
if vote.Round == cs.Round {
vals := cs.state.Validators
_, val := vals.GetByIndex(vote.ValidatorIndex)
cs.metrics.MarkVoteReceived(vote.Type, val.VotingPower, vals.TotalVotingPower())
}
if err := cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote}); err != nil {
return added, err
@@ -2482,18 +2567,18 @@ func (cs *State) signVote(
// If the signedMessageType is for precommit,
// use our local precommit Timeout as the max wait time for getting a singed commit. The same goes for prevote.
timeout := cs.voteTimeout(cs.Round)
switch msgType {
case tmproto.PrecommitType:
// if the signedMessage type is for a precommit, add VoteExtension
ext, err := cs.blockExec.ExtendVote(ctx, vote)
if err != nil {
return nil, err
timeout := time.Second
if msgType == tmproto.PrecommitType && !vote.BlockID.IsNil() {
timeout = cs.voteTimeout(cs.Round)
// if the signedMessage type is for a non-nil precommit, add
// VoteExtension
if cs.state.ConsensusParams.ABCI.VoteExtensionsEnabled(cs.Height) {
ext, err := cs.blockExec.ExtendVote(ctx, vote)
if err != nil {
return nil, err
}
vote.Extension = ext
}
vote.Extension = ext
default:
timeout = time.Second
}
v := vote.ToProto()
@@ -2533,14 +2618,17 @@ func (cs *State) signAddVote(
// TODO: pass pubKey to signVote
vote, err := cs.signVote(ctx, msgType, hash, header)
if err == nil {
cs.sendInternalMessage(ctx, msgInfo{&VoteMessage{vote}, "", tmtime.Now()})
cs.logger.Debug("signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote)
return vote
if err != nil {
cs.logger.Error("failed signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err)
return nil
}
cs.logger.Error("failed signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err)
return nil
if !cs.state.ConsensusParams.ABCI.VoteExtensionsEnabled(vote.Height) {
// The signer will sign the extension, make sure to remove the data on the way out
vote.StripExtension()
}
cs.sendInternalMessage(ctx, msgInfo{&VoteMessage{vote}, "", tmtime.Now()})
cs.logger.Debug("signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote)
return vote
}
// updatePrivValidatorPubKey get's the private validator public key and
internal/consensus/state_test.go
+225 -74
View File
@@ -18,6 +18,7 @@ import (
"github.com/tendermint/tendermint/internal/eventbus"
tmpubsub "github.com/tendermint/tendermint/internal/pubsub"
tmquery "github.com/tendermint/tendermint/internal/pubsub/query"
"github.com/tendermint/tendermint/internal/test/factory"
tmbytes "github.com/tendermint/tendermint/libs/bytes"
"github.com/tendermint/tendermint/libs/log"
tmrand "github.com/tendermint/tendermint/libs/rand"
@@ -1950,7 +1951,7 @@ func TestFinalizeBlockCalled(t *testing.T) {
expectCalled bool
}{
{
name: "finalze block called when block committed",
name: "finalize block called when block committed",
voteNil: false,
expectCalled: true,
},
@@ -1970,11 +1971,15 @@ func TestFinalizeBlockCalled(t *testing.T) {
Status: abci.ResponseProcessProposal_ACCEPT,
}, nil)
m.On("PrepareProposal", mock.Anything, mock.Anything).Return(&abci.ResponsePrepareProposal{}, nil)
m.On("VerifyVoteExtension", mock.Anything, mock.Anything).Return(&abci.ResponseVerifyVoteExtension{
Status: abci.ResponseVerifyVoteExtension_ACCEPT,
}, nil)
// We only expect VerifyVoteExtension to be called on non-nil precommits.
// https://github.com/tendermint/tendermint/issues/8487
if !testCase.voteNil {
m.On("ExtendVote", mock.Anything, mock.Anything).Return(&abci.ResponseExtendVote{}, nil)
m.On("VerifyVoteExtension", mock.Anything, mock.Anything).Return(&abci.ResponseVerifyVoteExtension{
Status: abci.ResponseVerifyVoteExtension_ACCEPT,
}, nil)
}
m.On("FinalizeBlock", mock.Anything, mock.Anything).Return(&abci.ResponseFinalizeBlock{}, nil).Maybe()
m.On("ExtendVote", mock.Anything, mock.Anything).Return(&abci.ResponseExtendVote{}, nil)
m.On("Commit", mock.Anything).Return(&abci.ResponseCommit{}, nil).Maybe()
cs1, vss := makeState(ctx, t, makeStateArgs{config: config, application: m})
@@ -2022,77 +2027,101 @@ func TestFinalizeBlockCalled(t *testing.T) {
}
}
// TestExtendVoteCalled tests that the vote extension methods are called at the
// correct point in the consensus algorithm.
func TestExtendVoteCalled(t *testing.T) {
config := configSetup(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// TestExtendVoteCalledWhenEnabled tests that the vote extension methods are called at the
// correct point in the consensus algorithm when vote extensions are enabled.
func TestExtendVoteCalledWhenEnabled(t *testing.T) {
for _, testCase := range []struct {
name string
enabled bool
}{
{
name: "enabled",
enabled: true,
},
{
name: "disabled",
enabled: false,
},
} {
t.Run(testCase.name, func(t *testing.T) {
config := configSetup(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
m := abcimocks.NewApplication(t)
m.On("ProcessProposal", mock.Anything, mock.Anything).Return(&abci.ResponseProcessProposal{Status: abci.ResponseProcessProposal_ACCEPT}, nil)
m.On("PrepareProposal", mock.Anything, mock.Anything).Return(&abci.ResponsePrepareProposal{}, nil)
m.On("ExtendVote", mock.Anything, mock.Anything).Return(&abci.ResponseExtendVote{
VoteExtension: []byte("extension"),
}, nil)
m.On("VerifyVoteExtension", mock.Anything, mock.Anything).Return(&abci.ResponseVerifyVoteExtension{
Status: abci.ResponseVerifyVoteExtension_ACCEPT,
}, nil)
m.On("Commit", mock.Anything).Return(&abci.ResponseCommit{}, nil).Maybe()
m.On("FinalizeBlock", mock.Anything, mock.Anything).Return(&abci.ResponseFinalizeBlock{}, nil).Maybe()
cs1, vss := makeState(ctx, t, makeStateArgs{config: config, application: m})
height, round := cs1.Height, cs1.Round
m := abcimocks.NewApplication(t)
m.On("ProcessProposal", mock.Anything, mock.Anything).Return(&abci.ResponseProcessProposal{Status: abci.ResponseProcessProposal_ACCEPT}, nil)
m.On("PrepareProposal", mock.Anything, mock.Anything).Return(&abci.ResponsePrepareProposal{}, nil)
if testCase.enabled {
m.On("ExtendVote", mock.Anything, mock.Anything).Return(&abci.ResponseExtendVote{
VoteExtension: []byte("extension"),
}, nil)
m.On("VerifyVoteExtension", mock.Anything, mock.Anything).Return(&abci.ResponseVerifyVoteExtension{
Status: abci.ResponseVerifyVoteExtension_ACCEPT,
}, nil)
}
m.On("Commit", mock.Anything).Return(&abci.ResponseCommit{}, nil).Maybe()
m.On("FinalizeBlock", mock.Anything, mock.Anything).Return(&abci.ResponseFinalizeBlock{}, nil).Maybe()
c := factory.ConsensusParams()
if !testCase.enabled {
c.ABCI.VoteExtensionsEnableHeight = 0
}
cs1, vss := makeState(ctx, t, makeStateArgs{config: config, application: m, consensusParams: c})
height, round := cs1.Height, cs1.Round
proposalCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryCompleteProposal)
newRoundCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryNewRound)
pv1, err := cs1.privValidator.GetPubKey(ctx)
require.NoError(t, err)
addr := pv1.Address()
voteCh := subscribeToVoter(ctx, t, cs1, addr)
proposalCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryCompleteProposal)
newRoundCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryNewRound)
pv1, err := cs1.privValidator.GetPubKey(ctx)
require.NoError(t, err)
addr := pv1.Address()
voteCh := subscribeToVoter(ctx, t, cs1, addr)
startTestRound(ctx, cs1, cs1.Height, round)
ensureNewRound(t, newRoundCh, height, round)
ensureNewProposal(t, proposalCh, height, round)
startTestRound(ctx, cs1, cs1.Height, round)
ensureNewRound(t, newRoundCh, height, round)
ensureNewProposal(t, proposalCh, height, round)
m.AssertNotCalled(t, "ExtendVote", mock.Anything, mock.Anything)
m.AssertNotCalled(t, "ExtendVote", mock.Anything, mock.Anything)
rs := cs1.GetRoundState()
rs := cs1.GetRoundState()
blockID := types.BlockID{
Hash: rs.ProposalBlock.Hash(),
PartSetHeader: rs.ProposalBlockParts.Header(),
}
signAddVotes(ctx, t, cs1, tmproto.PrevoteType, config.ChainID(), blockID, vss[1:]...)
ensurePrevoteMatch(t, voteCh, height, round, blockID.Hash)
blockID := types.BlockID{
Hash: rs.ProposalBlock.Hash(),
PartSetHeader: rs.ProposalBlockParts.Header(),
}
signAddVotes(ctx, t, cs1, tmproto.PrevoteType, config.ChainID(), blockID, vss[1:]...)
ensurePrevoteMatch(t, voteCh, height, round, blockID.Hash)
ensurePrecommit(t, voteCh, height, round)
ensurePrecommit(t, voteCh, height, round)
m.AssertCalled(t, "ExtendVote", ctx, &abci.RequestExtendVote{
Height: height,
Hash: blockID.Hash,
})
if testCase.enabled {
m.AssertCalled(t, "ExtendVote", ctx, &abci.RequestExtendVote{
Height: height,
Hash: blockID.Hash,
})
} else {
m.AssertNotCalled(t, "ExtendVote", mock.Anything, mock.Anything)
}
m.AssertCalled(t, "VerifyVoteExtension", ctx, &abci.RequestVerifyVoteExtension{
Hash: blockID.Hash,
ValidatorAddress: addr,
Height: height,
VoteExtension: []byte("extension"),
})
signAddVotes(ctx, t, cs1, tmproto.PrecommitType, config.ChainID(), blockID, vss[1:]...)
ensureNewRound(t, newRoundCh, height+1, 0)
m.AssertExpectations(t)
signAddVotes(ctx, t, cs1, tmproto.PrecommitType, config.ChainID(), blockID, vss[1:]...)
ensureNewRound(t, newRoundCh, height+1, 0)
m.AssertExpectations(t)
// Only 3 of the vote extensions are seen, as consensus proceeds as soon as the +2/3 threshold
// is observed by the consensus engine.
for _, pv := range vss[:3] {
pv, err := pv.GetPubKey(ctx)
require.NoError(t, err)
addr := pv.Address()
m.AssertCalled(t, "VerifyVoteExtension", ctx, &abci.RequestVerifyVoteExtension{
Hash: blockID.Hash,
ValidatorAddress: addr,
Height: height,
VoteExtension: []byte("extension"),
// Only 3 of the vote extensions are seen, as consensus proceeds as soon as the +2/3 threshold
// is observed by the consensus engine.
for _, pv := range vss[1:3] {
pv, err := pv.GetPubKey(ctx)
require.NoError(t, err)
addr := pv.Address()
if testCase.enabled {
m.AssertCalled(t, "VerifyVoteExtension", ctx, &abci.RequestVerifyVoteExtension{
Hash: blockID.Hash,
ValidatorAddress: addr,
Height: height,
VoteExtension: []byte("extension"),
})
} else {
m.AssertNotCalled(t, "VerifyVoteExtension", mock.Anything, mock.Anything)
}
}
})
}
@@ -2117,6 +2146,7 @@ func TestVerifyVoteExtensionNotCalledOnAbsentPrecommit(t *testing.T) {
m.On("FinalizeBlock", mock.Anything, mock.Anything).Return(&abci.ResponseFinalizeBlock{}, nil).Maybe()
cs1, vss := makeState(ctx, t, makeStateArgs{config: config, application: m})
height, round := cs1.Height, cs1.Round
cs1.state.ConsensusParams.ABCI.VoteExtensionsEnableHeight = cs1.Height
proposalCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryCompleteProposal)
newRoundCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryNewRound)
@@ -2134,7 +2164,7 @@ func TestVerifyVoteExtensionNotCalledOnAbsentPrecommit(t *testing.T) {
Hash: rs.ProposalBlock.Hash(),
PartSetHeader: rs.ProposalBlockParts.Header(),
}
signAddVotes(ctx, t, cs1, tmproto.PrevoteType, config.ChainID(), blockID, vss[2:]...)
signAddVotes(ctx, t, cs1, tmproto.PrevoteType, config.ChainID(), blockID, vss...)
ensurePrevoteMatch(t, voteCh, height, round, blockID.Hash)
ensurePrecommit(t, voteCh, height, round)
@@ -2144,13 +2174,6 @@ func TestVerifyVoteExtensionNotCalledOnAbsentPrecommit(t *testing.T) {
Hash: blockID.Hash,
})
m.AssertCalled(t, "VerifyVoteExtension", mock.Anything, &abci.RequestVerifyVoteExtension{
Hash: blockID.Hash,
ValidatorAddress: addr,
Height: height,
VoteExtension: []byte("extension"),
})
m.On("Commit", mock.Anything).Return(&abci.ResponseCommit{}, nil).Maybe()
signAddVotes(ctx, t, cs1, tmproto.PrecommitType, config.ChainID(), blockID, vss[2:]...)
ensureNewRound(t, newRoundCh, height+1, 0)
@@ -2262,6 +2285,134 @@ func TestPrepareProposalReceivesVoteExtensions(t *testing.T) {
}
}
// TestVoteExtensionEnableHeight tests that 'ExtensionRequireHeight' correctly
// enforces that vote extensions be present in consensus for heights greater than
// or equal to the configured value.
func TestVoteExtensionEnableHeight(t *testing.T) {
for _, testCase := range []struct {
name string
enableHeight int64
hasExtension bool
expectExtendCalled bool
expectVerifyCalled bool
expectSuccessfulRound bool
}{
{
name: "extension present but not enabled",
hasExtension: true,
enableHeight: 0,
expectExtendCalled: false,
expectVerifyCalled: false,
expectSuccessfulRound: true,
},
{
name: "extension absent but not required",
hasExtension: false,
enableHeight: 0,
expectExtendCalled: false,
expectVerifyCalled: false,
expectSuccessfulRound: true,
},
{
name: "extension present and required",
hasExtension: true,
enableHeight: 1,
expectExtendCalled: true,
expectVerifyCalled: true,
expectSuccessfulRound: true,
},
{
name: "extension absent but required",
hasExtension: false,
enableHeight: 1,
expectExtendCalled: true,
expectVerifyCalled: false,
expectSuccessfulRound: false,
},
{
name: "extension absent but required in future height",
hasExtension: false,
enableHeight: 2,
expectExtendCalled: false,
expectVerifyCalled: false,
expectSuccessfulRound: true,
},
} {
t.Run(testCase.name, func(t *testing.T) {
config := configSetup(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
numValidators := 3
m := abcimocks.NewApplication(t)
m.On("ProcessProposal", mock.Anything, mock.Anything).Return(&abci.ResponseProcessProposal{
Status: abci.ResponseProcessProposal_ACCEPT,
}, nil)
m.On("PrepareProposal", mock.Anything, mock.Anything).Return(&abci.ResponsePrepareProposal{}, nil)
if testCase.expectExtendCalled {
m.On("ExtendVote", mock.Anything, mock.Anything).Return(&abci.ResponseExtendVote{}, nil)
}
if testCase.expectVerifyCalled {
m.On("VerifyVoteExtension", mock.Anything, mock.Anything).Return(&abci.ResponseVerifyVoteExtension{
Status: abci.ResponseVerifyVoteExtension_ACCEPT,
}, nil).Times(numValidators - 1)
}
m.On("FinalizeBlock", mock.Anything, mock.Anything).Return(&abci.ResponseFinalizeBlock{}, nil).Maybe()
m.On("Commit", mock.Anything).Return(&abci.ResponseCommit{}, nil).Maybe()
c := factory.ConsensusParams()
c.ABCI.VoteExtensionsEnableHeight = testCase.enableHeight
cs1, vss := makeState(ctx, t, makeStateArgs{config: config, application: m, validators: numValidators, consensusParams: c})
cs1.state.ConsensusParams.ABCI.VoteExtensionsEnableHeight = testCase.enableHeight
height, round := cs1.Height, cs1.Round
timeoutCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryTimeoutPropose)
proposalCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryCompleteProposal)
newRoundCh := subscribe(ctx, t, cs1.eventBus, types.EventQueryNewRound)
pv1, err := cs1.privValidator.GetPubKey(ctx)
require.NoError(t, err)
addr := pv1.Address()
voteCh := subscribeToVoter(ctx, t, cs1, addr)
startTestRound(ctx, cs1, cs1.Height, round)
ensureNewRound(t, newRoundCh, height, round)
ensureNewProposal(t, proposalCh, height, round)
rs := cs1.GetRoundState()
blockID := types.BlockID{
Hash: rs.ProposalBlock.Hash(),
PartSetHeader: rs.ProposalBlockParts.Header(),
}
// sign all of the votes
signAddVotes(ctx, t, cs1, tmproto.PrevoteType, config.ChainID(), blockID, vss[1:]...)
ensurePrevoteMatch(t, voteCh, height, round, rs.ProposalBlock.Hash())
var ext []byte
if testCase.hasExtension {
ext = []byte("extension")
}
for _, vs := range vss[1:] {
vote, err := vs.signVote(ctx, tmproto.PrecommitType, config.ChainID(), blockID, ext)
if !testCase.hasExtension {
vote.ExtensionSignature = nil
}
require.NoError(t, err)
addVotes(cs1, vote)
}
if testCase.expectSuccessfulRound {
ensurePrecommit(t, voteCh, height, round)
height++
ensureNewRound(t, newRoundCh, height, round)
} else {
ensureNoNewTimeout(t, timeoutCh, cs1.state.ConsensusParams.Timeout.VoteTimeout(round).Nanoseconds())
}
m.AssertExpectations(t)
})
}
}
// 4 vals, 3 Nil Precommits at P0
// What we want:
// P0 waits for timeoutPrecommit before starting next round
internal/consensus/types/height_vote_set.go
+21 -5
View File
@@ -38,9 +38,10 @@ We let each peer provide us with up to 2 unexpected "catchup" rounds.
One for their LastCommit round, and another for the official commit round.
*/
type HeightVoteSet struct {
chainID string
height int64
valSet *types.ValidatorSet
chainID string
height int64
valSet *types.ValidatorSet
extensionsEnabled bool
mtx sync.Mutex
round int32 // max tracked round
@@ -50,7 +51,17 @@ type HeightVoteSet struct {
func NewHeightVoteSet(chainID string, height int64, valSet *types.ValidatorSet) *HeightVoteSet {
hvs := &HeightVoteSet{
chainID: chainID,
chainID: chainID,
extensionsEnabled: false,
}
hvs.Reset(height, valSet)
return hvs
}
func NewExtendedHeightVoteSet(chainID string, height int64, valSet *types.ValidatorSet) *HeightVoteSet {
hvs := &HeightVoteSet{
chainID: chainID,
extensionsEnabled: true,
}
hvs.Reset(height, valSet)
return hvs
@@ -108,7 +119,12 @@ func (hvs *HeightVoteSet) addRound(round int32) {
}
// log.Debug("addRound(round)", "round", round)
prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, tmproto.PrevoteType, hvs.valSet)
precommits := types.NewVoteSet(hvs.chainID, hvs.height, round, tmproto.PrecommitType, hvs.valSet)
var precommits *types.VoteSet
if hvs.extensionsEnabled {
precommits = types.NewExtendedVoteSet(hvs.chainID, hvs.height, round, tmproto.PrecommitType, hvs.valSet)
} else {
precommits = types.NewVoteSet(hvs.chainID, hvs.height, round, tmproto.PrecommitType, hvs.valSet)
}
hvs.roundVoteSets[round] = RoundVoteSet{
Prevotes: prevotes,
Precommits: precommits,
internal/consensus/types/height_vote_set_test.go
+1 -1
View File
@@ -27,7 +27,7 @@ func TestPeerCatchupRounds(t *testing.T) {
valSet, privVals := factory.ValidatorSet(ctx, t, 10, 1)
chainID := cfg.ChainID()
hvs := NewHeightVoteSet(chainID, 1, valSet)
hvs := NewExtendedHeightVoteSet(chainID, 1, valSet)
vote999_0 := makeVoteHR(ctx, t, 1, 0, 999, privVals, chainID)
added, err := hvs.AddVote(vote999_0, "peer1")
internal/eventlog/eventlog.go
+9 -9
View File
@@ -24,9 +24,9 @@ import (
// any number of readers.
type Log struct {
// These values do not change after construction.
windowSize time.Duration
maxItems int
numItemsGauge gauge
windowSize time.Duration
maxItems int
metrics *Metrics
// Protects access to the fields below. Lock to modify the values of these
// fields, or to read or snapshot the values.
@@ -45,14 +45,14 @@ func New(opts LogSettings) (*Log, error) {
return nil, errors.New("window size must be positive")
}
lg := &Log{
windowSize: opts.WindowSize,
maxItems: opts.MaxItems,
numItemsGauge: discard{},
ready: make(chan struct{}),
source: opts.Source,
windowSize: opts.WindowSize,
maxItems: opts.MaxItems,
metrics: NopMetrics(),
ready: make(chan struct{}),
source: opts.Source,
}
if opts.Metrics != nil {
lg.numItemsGauge = opts.Metrics.numItemsGauge
lg.metrics = opts.Metrics
}
return lg, nil
}
internal/eventlog/metrics.gen.go
+30
View File
@@ -0,0 +1,30 @@
// Code generated by metricsgen. DO NOT EDIT.
package eventlog
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
numItems: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "num_items",
Help: "Number of items currently resident in the event log.",
}, labels).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
numItems: discard.NewGauge(),
}
}
internal/eventlog/metrics.go
+6 -31
View File
@@ -1,39 +1,14 @@
package eventlog
import (
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
import "github.com/go-kit/kit/metrics"
// gauge is the subset of the Prometheus gauge interface used here.
type gauge interface {
Set(float64)
}
const MetricsSubsystem = "eventlog"
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics define the metrics exported by the eventlog package.
type Metrics struct {
numItemsGauge gauge
}
// discard is a no-op implementation of the gauge interface.
type discard struct{}
func (discard) Set(float64) {}
const eventlogSubsystem = "eventlog"
// PrometheusMetrics returns a collection of eventlog metrics for Prometheus.
func PrometheusMetrics(ns string, fields ...string) *Metrics {
var labels []string
for i := 0; i < len(fields); i += 2 {
labels = append(labels, fields[i])
}
return &Metrics{
numItemsGauge: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: ns,
Subsystem: eventlogSubsystem,
Name: "num_items",
Help: "Number of items currently resident in the event log.",
}, labels).With(fields...),
}
// Number of items currently resident in the event log.
numItems metrics.Gauge
}
internal/eventlog/prune.go
+2 -2
View File
@@ -12,7 +12,7 @@ func (lg *Log) checkPrune(head *logEntry, size int, age time.Duration) error {
const windowSlop = 30 * time.Second
if age < (lg.windowSize+windowSlop) && (lg.maxItems <= 0 || size <= lg.maxItems) {
lg.numItemsGauge.Set(float64(lg.numItems))
lg.metrics.numItems.Set(float64(lg.numItems))
return nil // no pruning is needed
}
@@ -46,7 +46,7 @@ func (lg *Log) checkPrune(head *logEntry, size int, age time.Duration) error {
lg.mu.Lock()
defer lg.mu.Unlock()
lg.numItems = newState.size
lg.numItemsGauge.Set(float64(newState.size))
lg.metrics.numItems.Set(float64(newState.size))
lg.oldestCursor = newState.oldest
lg.head = newState.head
return err
internal/evidence/metrics.gen.go
+30
View File
@@ -0,0 +1,30 @@
// Code generated by metricsgen. DO NOT EDIT.
package evidence
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
NumEvidence: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "num_evidence",
Help: "Number of pending evidence in the evidence pool.",
}, labels).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
NumEvidence: discard.NewGauge(),
}
}
internal/evidence/metrics.go
+3 -30
View File
@@ -2,9 +2,6 @@ package evidence
import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
const (
@@ -13,35 +10,11 @@ const (
MetricsSubsystem = "evidence_pool"
)
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics contains metrics exposed by this package.
// see MetricsProvider for descriptions.
type Metrics struct {
// Number of evidence in the evidence pool
// Number of pending evidence in the evidence pool.
NumEvidence metrics.Gauge
}
// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
NumEvidence: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "num_evidence",
Help: "Number of pending evidence in evidence pool.",
}, labels).With(labelsAndValues...),
}
}
// NopMetrics returns no-op Metrics.
func NopMetrics() *Metrics {
return &Metrics{
NumEvidence: discard.NewGauge(),
}
}
internal/evidence/pool_test.go
+19 -15
View File
@@ -249,8 +249,8 @@ func TestEvidencePoolUpdate(t *testing.T) {
evidenceChainID,
)
require.NoError(t, err)
lastCommit := makeCommit(height, val.PrivKey.PubKey().Address())
block := types.MakeBlock(height+1, []types.Tx{}, lastCommit, []types.Evidence{ev})
lastExtCommit := makeExtCommit(height, val.PrivKey.PubKey().Address())
block := types.MakeBlock(height+1, []types.Tx{}, lastExtCommit.ToCommit(), []types.Evidence{ev})
// update state (partially)
state.LastBlockHeight = height + 1
@@ -568,8 +568,8 @@ func initializeBlockStore(db dbm.DB, state sm.State, valAddr []byte) (*store.Blo
blockStore := store.NewBlockStore(db)
for i := int64(1); i <= state.LastBlockHeight; i++ {
lastCommit := makeCommit(i-1, valAddr)
block := sf.MakeBlock(state, i, lastCommit)
lastCommit := makeExtCommit(i-1, valAddr)
block := sf.MakeBlock(state, i, lastCommit.ToCommit())
block.Header.Time = defaultEvidenceTime.Add(time.Duration(i) * time.Minute)
block.Header.Version = version.Consensus{Block: version.BlockProtocol, App: 1}
@@ -579,22 +579,26 @@ func initializeBlockStore(db dbm.DB, state sm.State, valAddr []byte) (*store.Blo
return nil, err
}
seenCommit := makeCommit(i, valAddr)
blockStore.SaveBlock(block, partSet, seenCommit)
seenCommit := makeExtCommit(i, valAddr)
blockStore.SaveBlockWithExtendedCommit(block, partSet, seenCommit)
}
return blockStore, nil
}
func makeCommit(height int64, valAddr []byte) *types.Commit {
commitSigs := []types.CommitSig{{
BlockIDFlag: types.BlockIDFlagCommit,
ValidatorAddress: valAddr,
Timestamp: defaultEvidenceTime,
Signature: []byte("Signature"),
}}
return types.NewCommit(height, 0, types.BlockID{}, commitSigs)
func makeExtCommit(height int64, valAddr []byte) *types.ExtendedCommit {
return &types.ExtendedCommit{
Height: height,
ExtendedSignatures: []types.ExtendedCommitSig{{
CommitSig: types.CommitSig{
BlockIDFlag: types.BlockIDFlagCommit,
ValidatorAddress: valAddr,
Timestamp: defaultEvidenceTime,
Signature: []byte("Signature"),
},
ExtensionSignature: []byte("Extended Signature"),
}},
}
}
func defaultTestPool(ctx context.Context, t *testing.T, height int64) (*evidence.Pool, types.MockPV, *eventbus.EventBus) {
internal/evidence/verify_test.go
+18 -12
View File
@@ -233,9 +233,10 @@ func TestVerifyLightClientAttack_Equivocation(t *testing.T) {
// we are simulating a duplicate vote attack where all the validators in the conflictingVals set
// except the last validator vote twice
blockID := factory.MakeBlockIDWithHash(conflictingHeader.Hash())
voteSet := types.NewVoteSet(evidenceChainID, 10, 1, tmproto.SignedMsgType(2), conflictingVals)
commit, err := factory.MakeCommit(ctx, blockID, 10, 1, voteSet, conflictingPrivVals[:4], defaultEvidenceTime)
voteSet := types.NewExtendedVoteSet(evidenceChainID, 10, 1, tmproto.SignedMsgType(2), conflictingVals)
extCommit, err := factory.MakeExtendedCommit(ctx, blockID, 10, 1, voteSet, conflictingPrivVals[:4], defaultEvidenceTime)
require.NoError(t, err)
commit := extCommit.ToCommit()
ev := &types.LightClientAttackEvidence{
ConflictingBlock: &types.LightBlock{
@@ -252,10 +253,11 @@ func TestVerifyLightClientAttack_Equivocation(t *testing.T) {
}
trustedBlockID := makeBlockID(trustedHeader.Hash(), 1000, []byte("partshash"))
trustedVoteSet := types.NewVoteSet(evidenceChainID, 10, 1, tmproto.SignedMsgType(2), conflictingVals)
trustedCommit, err := factory.MakeCommit(ctx, trustedBlockID, 10, 1,
trustedVoteSet := types.NewExtendedVoteSet(evidenceChainID, 10, 1, tmproto.SignedMsgType(2), conflictingVals)
trustedExtCommit, err := factory.MakeExtendedCommit(ctx, trustedBlockID, 10, 1,
trustedVoteSet, conflictingPrivVals, defaultEvidenceTime)
require.NoError(t, err)
trustedCommit := trustedExtCommit.ToCommit()
trustedSignedHeader := &types.SignedHeader{
Header: trustedHeader,
@@ -334,9 +336,10 @@ func TestVerifyLightClientAttack_Amnesia(t *testing.T) {
// we are simulating an amnesia attack where all the validators in the conflictingVals set
// except the last validator vote twice. However this time the commits are of different rounds.
blockID := makeBlockID(conflictingHeader.Hash(), 1000, []byte("partshash"))
voteSet := types.NewVoteSet(evidenceChainID, height, 0, tmproto.SignedMsgType(2), conflictingVals)
commit, err := factory.MakeCommit(ctx, blockID, height, 0, voteSet, conflictingPrivVals, defaultEvidenceTime)
voteSet := types.NewExtendedVoteSet(evidenceChainID, height, 0, tmproto.SignedMsgType(2), conflictingVals)
extCommit, err := factory.MakeExtendedCommit(ctx, blockID, height, 0, voteSet, conflictingPrivVals, defaultEvidenceTime)
require.NoError(t, err)
commit := extCommit.ToCommit()
ev := &types.LightClientAttackEvidence{
ConflictingBlock: &types.LightBlock{
@@ -353,10 +356,11 @@ func TestVerifyLightClientAttack_Amnesia(t *testing.T) {
}
trustedBlockID := makeBlockID(trustedHeader.Hash(), 1000, []byte("partshash"))
trustedVoteSet := types.NewVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), conflictingVals)
trustedCommit, err := factory.MakeCommit(ctx, trustedBlockID, height, 1,
trustedVoteSet := types.NewExtendedVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), conflictingVals)
trustedExtCommit, err := factory.MakeExtendedCommit(ctx, trustedBlockID, height, 1,
trustedVoteSet, conflictingPrivVals, defaultEvidenceTime)
require.NoError(t, err)
trustedCommit := trustedExtCommit.ToCommit()
trustedSignedHeader := &types.SignedHeader{
Header: trustedHeader,
@@ -549,9 +553,10 @@ func makeLunaticEvidence(
})
blockID := factory.MakeBlockIDWithHash(conflictingHeader.Hash())
voteSet := types.NewVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), conflictingVals)
commit, err := factory.MakeCommit(ctx, blockID, height, 1, voteSet, conflictingPrivVals, defaultEvidenceTime)
voteSet := types.NewExtendedVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), conflictingVals)
extCommit, err := factory.MakeExtendedCommit(ctx, blockID, height, 1, voteSet, conflictingPrivVals, defaultEvidenceTime)
require.NoError(t, err)
commit := extCommit.ToCommit()
ev = &types.LightClientAttackEvidence{
ConflictingBlock: &types.LightBlock{
@@ -577,9 +582,10 @@ func makeLunaticEvidence(
}
trustedBlockID := factory.MakeBlockIDWithHash(trustedHeader.Hash())
trustedVals, privVals := factory.ValidatorSet(ctx, t, totalVals, defaultVotingPower)
trustedVoteSet := types.NewVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), trustedVals)
trustedCommit, err := factory.MakeCommit(ctx, trustedBlockID, height, 1, trustedVoteSet, privVals, defaultEvidenceTime)
trustedVoteSet := types.NewExtendedVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), trustedVals)
trustedExtCommit, err := factory.MakeExtendedCommit(ctx, trustedBlockID, height, 1, trustedVoteSet, privVals, defaultEvidenceTime)
require.NoError(t, err)
trustedCommit := trustedExtCommit.ToCommit()
trusted = &types.LightBlock{
SignedHeader: &types.SignedHeader{
internal/inspect/rpc/rpc.go
+2 -1
View File
@@ -125,7 +125,8 @@ func serverRPCConfig(r *config.RPCConfig) *server.Config {
// If necessary adjust global WriteTimeout to ensure it's greater than
// TimeoutBroadcastTxCommit.
// See https://github.com/tendermint/tendermint/issues/3435
if cfg.WriteTimeout <= r.TimeoutBroadcastTxCommit {
// Note we don't need to adjust anything if the timeout is already unlimited.
if cfg.WriteTimeout > 0 && cfg.WriteTimeout <= r.TimeoutBroadcastTxCommit {
cfg.WriteTimeout = r.TimeoutBroadcastTxCommit + 1*time.Second
}
return cfg
internal/mempool/metrics.gen.go
+67
View File
@@ -0,0 +1,67 @@
// Code generated by metricsgen. DO NOT EDIT.
package mempool
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
Size: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "size",
Help: "Number of uncommitted transactions in the mempool.",
}, labels).With(labelsAndValues...),
TxSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "tx_size_bytes",
Help: "Histogram of transaction sizes in bytes.",
Buckets: stdprometheus.ExponentialBuckets(1, 3, 7),
}, labels).With(labelsAndValues...),
FailedTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "failed_txs",
Help: "Number of failed transactions.",
}, labels).With(labelsAndValues...),
RejectedTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "rejected_txs",
Help: "Number of rejected transactions.",
}, labels).With(labelsAndValues...),
EvictedTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "evicted_txs",
Help: "Number of evicted transactions.",
}, labels).With(labelsAndValues...),
RecheckTimes: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "recheck_times",
Help: "Number of times transactions are rechecked in the mempool.",
}, labels).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
Size: discard.NewGauge(),
TxSizeBytes: discard.NewHistogram(),
FailedTxs: discard.NewCounter(),
RejectedTxs: discard.NewCounter(),
EvictedTxs: discard.NewCounter(),
RecheckTimes: discard.NewCounter(),
}
}
internal/mempool/metrics.go
+7 -72
View File
@@ -2,9 +2,6 @@ package mempool
import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
const (
@@ -13,14 +10,16 @@ const (
MetricsSubsystem = "mempool"
)
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics contains metrics exposed by this package.
// see MetricsProvider for descriptions.
type Metrics struct {
// Size of the mempool.
// Number of uncommitted transactions in the mempool.
Size metrics.Gauge
// Histogram of transaction sizes, in bytes.
TxSizeBytes metrics.Histogram
// Histogram of transaction sizes in bytes.
TxSizeBytes metrics.Histogram `metrics_buckettype:"exp" metrics_bucketsizes:"1,3,7"`
// Number of failed transactions.
FailedTxs metrics.Counter
@@ -29,80 +28,16 @@ type Metrics struct {
// transactions that passed CheckTx but failed to make it into the mempool
// due to resource limits, e.g. mempool is full and no lower priority
// transactions exist in the mempool.
//metrics:Number of rejected transactions.
RejectedTxs metrics.Counter
// EvictedTxs defines the number of evicted transactions. These are valid
// transactions that passed CheckTx and existed in the mempool but were later
// evicted to make room for higher priority valid transactions that passed
// CheckTx.
//metrics:Number of evicted transactions.
EvictedTxs metrics.Counter
// Number of times transactions are rechecked in the mempool.
RecheckTimes metrics.Counter
}
// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
Size: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "size",
Help: "Size of the mempool (number of uncommitted transactions).",
}, labels).With(labelsAndValues...),
TxSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "tx_size_bytes",
Help: "Transaction sizes in bytes.",
Buckets: stdprometheus.ExponentialBuckets(1, 3, 17),
}, labels).With(labelsAndValues...),
FailedTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "failed_txs",
Help: "Number of failed transactions.",
}, labels).With(labelsAndValues...),
RejectedTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "rejected_txs",
Help: "Number of rejected transactions.",
}, labels).With(labelsAndValues...),
EvictedTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "evicted_txs",
Help: "Number of evicted transactions.",
}, labels).With(labelsAndValues...),
RecheckTimes: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "recheck_times",
Help: "Number of times transactions are rechecked in the mempool.",
}, labels).With(labelsAndValues...),
}
}
// NopMetrics returns no-op Metrics.
func NopMetrics() *Metrics {
return &Metrics{
Size: discard.NewGauge(),
TxSizeBytes: discard.NewHistogram(),
FailedTxs: discard.NewCounter(),
RejectedTxs: discard.NewCounter(),
EvictedTxs: discard.NewCounter(),
RecheckTimes: discard.NewCounter(),
}
}
internal/mempool/reactor.go
+19 -30
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"runtime/debug"
"sync"
"time"
"github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/internal/libs/clist"
@@ -22,13 +21,6 @@ var (
_ p2p.Wrapper = (*protomem.Message)(nil)
)
// PeerManager defines the interface contract required for getting necessary
// peer information. This should eventually be replaced with a message-oriented
// approach utilizing the p2p stack.
type PeerManager interface {
GetHeight(types.NodeID) int64
}
// Reactor implements a service that contains mempool of txs that are broadcasted
// amongst peers. It maintains a map from peer ID to counter, to prevent gossiping
// txs to the peers you received it from.
@@ -40,9 +32,8 @@ type Reactor struct {
mempool *TxMempool
ids *IDs
getPeerHeight func(types.NodeID) int64
peerEvents p2p.PeerEventSubscriber
chCreator p2p.ChannelCreator
peerEvents p2p.PeerEventSubscriber
chCreator p2p.ChannelCreator
// observePanic is a function for observing panics that were recovered in methods on
// Reactor. observePanic is called with the recovered value.
@@ -59,18 +50,16 @@ func NewReactor(
txmp *TxMempool,
chCreator p2p.ChannelCreator,
peerEvents p2p.PeerEventSubscriber,
getPeerHeight func(types.NodeID) int64,
) *Reactor {
r := &Reactor{
logger: logger,
cfg: cfg,
mempool: txmp,
ids: NewMempoolIDs(),
chCreator: chCreator,
peerEvents: peerEvents,
getPeerHeight: getPeerHeight,
peerRoutines: make(map[types.NodeID]context.CancelFunc),
observePanic: defaultObservePanic,
logger: logger,
cfg: cfg,
mempool: txmp,
ids: NewMempoolIDs(),
chCreator: chCreator,
peerEvents: peerEvents,
peerRoutines: make(map[types.NodeID]context.CancelFunc),
observePanic: defaultObservePanic,
}
r.BaseService = *service.NewBaseService(logger, "Mempool", r)
@@ -153,6 +142,15 @@ func (r *Reactor) handleMempoolMessage(ctx context.Context, envelope *p2p.Envelo
// problem.
continue
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
// Do not propagate context
// cancellation errors, but do
// not continue to check
// transactions from this
// message if we are shutting down.
return nil
}
logger.Error("checktx failed for tx",
"tx", fmt.Sprintf("%X", types.Tx(tx).Hash()),
"err", err)
@@ -318,15 +316,6 @@ func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID, m
memTx := nextGossipTx.Value.(*WrappedTx)
if r.getPeerHeight != nil {
height := r.getPeerHeight(peerID)
if height > 0 && height < memTx.height-1 {
// allow for a lag of one block
time.Sleep(PeerCatchupSleepIntervalMS * time.Millisecond)
continue
}
}
// NOTE: Transaction batching was disabled due to:
// https://github.com/tendermint/tendermint/issues/5796
if ok := r.mempool.txStore.TxHasPeer(memTx.hash, peerMempoolID); !ok {
internal/mempool/reactor_test.go
-1
View File
@@ -85,7 +85,6 @@ func setupReactors(ctx context.Context, t *testing.T, logger log.Logger, numNode
mempool,
chCreator,
func(ctx context.Context) *p2p.PeerUpdates { return rts.peerUpdates[nodeID] },
rts.network.Nodes[nodeID].PeerManager.GetHeight,
)
rts.nodes = append(rts.nodes, nodeID)
internal/p2p/channel.go
+1
View File
@@ -46,6 +46,7 @@ type Wrapper interface {
type PeerError struct {
NodeID types.NodeID
Err error
Fatal bool
}
func (pe PeerError) Error() string { return fmt.Sprintf("peer=%q: %s", pe.NodeID, pe.Err.Error()) }
internal/p2p/metrics.gen.go
+86
View File
@@ -0,0 +1,86 @@
// Code generated by metricsgen. DO NOT EDIT.
package p2p
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
Peers: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peers",
Help: "Number of peers.",
}, labels).With(labelsAndValues...),
PeerReceiveBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peer_receive_bytes_total",
Help: "Number of bytes per channel received from a given peer.",
}, append(labels, "peer_id", "chID", "message_type")).With(labelsAndValues...),
PeerSendBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peer_send_bytes_total",
Help: "Number of bytes per channel sent to a given peer.",
}, append(labels, "peer_id", "chID", "message_type")).With(labelsAndValues...),
PeerPendingSendBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peer_pending_send_bytes",
Help: "Number of bytes pending being sent to a given peer.",
}, append(labels, "peer_id")).With(labelsAndValues...),
RouterPeerQueueRecv: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_peer_queue_recv",
Help: "The time taken to read off of a peer's queue before sending on the connection.",
}, labels).With(labelsAndValues...),
RouterPeerQueueSend: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_peer_queue_send",
Help: "The time taken to send on a peer's queue which will later be read and sent on the connection.",
}, labels).With(labelsAndValues...),
RouterChannelQueueSend: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_channel_queue_send",
Help: "The time taken to send on a p2p channel's queue which will later be consued by the corresponding reactor/service.",
}, labels).With(labelsAndValues...),
PeerQueueDroppedMsgs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_channel_queue_dropped_msgs",
Help: "The number of messages dropped from a peer's queue for a specific p2p Channel.",
}, append(labels, "ch_id")).With(labelsAndValues...),
PeerQueueMsgSize: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peer_queue_msg_size",
Help: "The size of messages sent over a peer's queue for a specific p2p Channel.",
}, append(labels, "ch_id")).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
Peers: discard.NewGauge(),
PeerReceiveBytesTotal: discard.NewCounter(),
PeerSendBytesTotal: discard.NewCounter(),
PeerPendingSendBytes: discard.NewGauge(),
RouterPeerQueueRecv: discard.NewHistogram(),
RouterPeerQueueSend: discard.NewHistogram(),
RouterChannelQueueSend: discard.NewHistogram(),
PeerQueueDroppedMsgs: discard.NewCounter(),
PeerQueueMsgSize: discard.NewGauge(),
}
}
internal/p2p/metrics.go
+25 -106
View File
@@ -7,9 +7,6 @@ import (
"sync"
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
const (
@@ -25,140 +22,55 @@ var (
valueToLabelRegexp = regexp.MustCompile(`\*?(\w+)\.(.*)`)
)
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics contains metrics exposed by this package.
type Metrics struct {
// Number of peers.
Peers metrics.Gauge
// Number of bytes received from a given peer.
PeerReceiveBytesTotal metrics.Counter
// Number of bytes sent to a given peer.
PeerSendBytesTotal metrics.Counter
// Pending bytes to be sent to a given peer.
PeerPendingSendBytes metrics.Gauge
// Number of bytes per channel received from a given peer.
PeerReceiveBytesTotal metrics.Counter `metrics_labels:"peer_id, chID, message_type"`
// Number of bytes per channel sent to a given peer.
PeerSendBytesTotal metrics.Counter `metrics_labels:"peer_id, chID, message_type"`
// Number of bytes pending being sent to a given peer.
PeerPendingSendBytes metrics.Gauge `metrics_labels:"peer_id"`
// RouterPeerQueueRecv defines the time taken to read off of a peer's queue
// before sending on the connection.
//metrics:The time taken to read off of a peer's queue before sending on the connection.
RouterPeerQueueRecv metrics.Histogram
// RouterPeerQueueSend defines the time taken to send on a peer's queue which
// will later be read and sent on the connection (see RouterPeerQueueRecv).
//metrics:The time taken to send on a peer's queue which will later be read and sent on the connection.
RouterPeerQueueSend metrics.Histogram
// RouterChannelQueueSend defines the time taken to send on a p2p channel's
// queue which will later be consued by the corresponding reactor/service.
//metrics:The time taken to send on a p2p channel's queue which will later be consued by the corresponding reactor/service.
RouterChannelQueueSend metrics.Histogram
// PeerQueueDroppedMsgs defines the number of messages dropped from a peer's
// queue for a specific flow (i.e. Channel).
PeerQueueDroppedMsgs metrics.Counter
//metrics:The number of messages dropped from a peer's queue for a specific p2p Channel.
PeerQueueDroppedMsgs metrics.Counter `metrics_labels:"ch_id" metrics_name:"router_channel_queue_dropped_msgs"`
// PeerQueueMsgSize defines the average size of messages sent over a peer's
// queue for a specific flow (i.e. Channel).
PeerQueueMsgSize metrics.Gauge
//metrics:The size of messages sent over a peer's queue for a specific p2p Channel.
PeerQueueMsgSize metrics.Gauge `metrics_labels:"ch_id" metric_name:"router_channel_queue_msg_size"`
}
type metricsLabelCache struct {
mtx *sync.RWMutex
messageLabelNames map[reflect.Type]string
}
// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
Peers: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peers",
Help: "Number of peers.",
}, labels).With(labelsAndValues...),
PeerReceiveBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peer_receive_bytes_total",
Help: "Number of bytes received from a given peer.",
}, append(labels, "peer_id", "chID", "message_type")).With(labelsAndValues...),
PeerSendBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peer_send_bytes_total",
Help: "Number of bytes sent to a given peer.",
}, append(labels, "peer_id", "chID", "message_type")).With(labelsAndValues...),
PeerPendingSendBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "peer_pending_send_bytes",
Help: "Number of pending bytes to be sent to a given peer.",
}, append(labels, "peer_id")).With(labelsAndValues...),
RouterPeerQueueRecv: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_peer_queue_recv",
Help: "The time taken to read off of a peer's queue before sending on the connection.",
}, labels).With(labelsAndValues...),
RouterPeerQueueSend: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_peer_queue_send",
Help: "The time taken to send on a peer's queue which will later be read and sent on the connection (see RouterPeerQueueRecv).",
}, labels).With(labelsAndValues...),
RouterChannelQueueSend: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_channel_queue_send",
Help: "The time taken to send on a p2p channel's queue which will later be consued by the corresponding reactor/service.",
}, labels).With(labelsAndValues...),
PeerQueueDroppedMsgs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_channel_queue_dropped_msgs",
Help: "The number of messages dropped from a peer's queue for a specific p2p Channel.",
}, append(labels, "ch_id")).With(labelsAndValues...),
PeerQueueMsgSize: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "router_channel_queue_msg_size",
Help: "The size of messages sent over a peer's queue for a specific p2p Channel.",
}, append(labels, "ch_id")).With(labelsAndValues...),
mtx: &sync.RWMutex{},
messageLabelNames: map[reflect.Type]string{},
}
}
// NopMetrics returns no-op Metrics.
func NopMetrics() *Metrics {
return &Metrics{
Peers: discard.NewGauge(),
PeerReceiveBytesTotal: discard.NewCounter(),
PeerSendBytesTotal: discard.NewCounter(),
PeerPendingSendBytes: discard.NewGauge(),
RouterPeerQueueRecv: discard.NewHistogram(),
RouterPeerQueueSend: discard.NewHistogram(),
RouterChannelQueueSend: discard.NewHistogram(),
PeerQueueDroppedMsgs: discard.NewCounter(),
PeerQueueMsgSize: discard.NewGauge(),
mtx: &sync.RWMutex{},
messageLabelNames: map[reflect.Type]string{},
}
}
// ValueToMetricLabel is a method that is used to produce a prometheus label value of the golang
// type that is passed in.
// This method uses a map on the Metrics struct so that each label name only needs
// to be produced once to prevent expensive string operations.
func (m *Metrics) ValueToMetricLabel(i interface{}) string {
func (m *metricsLabelCache) ValueToMetricLabel(i interface{}) string {
t := reflect.TypeOf(i)
m.mtx.RLock()
@@ -176,3 +88,10 @@ func (m *Metrics) ValueToMetricLabel(i interface{}) string {
m.messageLabelNames[t] = l
return l
}
func newMetricsLabelCache() *metricsLabelCache {
return &metricsLabelCache{
mtx: &sync.RWMutex{},
messageLabelNames: map[reflect.Type]string{},
}
}
internal/p2p/metrics_test.go
+3 -3
View File
@@ -9,12 +9,12 @@ import (
)
func TestValueToMetricsLabel(t *testing.T) {
m := NopMetrics()
lc := newMetricsLabelCache()
r := &p2p.PexResponse{}
str := m.ValueToMetricLabel(r)
str := lc.ValueToMetricLabel(r)
assert.Equal(t, "p2p_PexResponse", str)
// subsequent calls to the function should produce the same result
str = m.ValueToMetricLabel(r)
str = lc.ValueToMetricLabel(r)
assert.Equal(t, "p2p_PexResponse", str)
}
internal/p2p/peermanager.go
+7 -31
View File
@@ -430,6 +430,13 @@ func (m *PeerManager) PeerRatio() float64 {
return float64(m.store.Size()) / float64(m.options.MaxPeers)
}
func (m *PeerManager) HasMaxPeerCapacity() bool {
m.mtx.Lock()
defer m.mtx.Unlock()
return len(m.connected) >= int(m.options.MaxConnected)
}
// DialNext finds an appropriate peer address to dial, and marks it as dialing.
// If no peer is found, or all connection slots are full, it blocks until one
// becomes available. The caller must call Dialed() or DialFailed() for the
@@ -1027,37 +1034,6 @@ func (m *PeerManager) retryDelay(failures uint32, persistent bool) time.Duration
return delay
}
// GetHeight returns a peer's height, as reported via SetHeight, or 0 if the
// peer or height is unknown.
//
// FIXME: This is a temporary workaround to share state between the consensus
// and mempool reactors, carried over from the legacy P2P stack. Reactors should
// not have dependencies on each other, instead tracking this themselves.
func (m *PeerManager) GetHeight(peerID types.NodeID) int64 {
m.mtx.Lock()
defer m.mtx.Unlock()
peer, _ := m.store.Get(peerID)
return peer.Height
}
// SetHeight stores a peer's height, making it available via GetHeight.
//
// FIXME: This is a temporary workaround to share state between the consensus
// and mempool reactors, carried over from the legacy P2P stack. Reactors should
// not have dependencies on each other, instead tracking this themselves.
func (m *PeerManager) SetHeight(peerID types.NodeID, height int64) error {
m.mtx.Lock()
defer m.mtx.Unlock()
peer, ok := m.store.Get(peerID)
if !ok {
peer = m.newPeerInfo(peerID)
}
peer.Height = height
return m.store.Set(peer)
}
// peerStore stores information about peers. It is not thread-safe, assuming it
// is only used by PeerManager which handles concurrency control. This allows
// the manager to execute multiple operations atomically via its own mutex.
internal/p2p/peermanager_test.go
-35
View File
@@ -1868,38 +1868,3 @@ func TestPeerManager_Advertise_Self(t *testing.T) {
self,
}, peerManager.Advertise(dID, 100))
}
func TestPeerManager_SetHeight_GetHeight(t *testing.T) {
a := p2p.NodeAddress{Protocol: "memory", NodeID: types.NodeID(strings.Repeat("a", 40))}
b := p2p.NodeAddress{Protocol: "memory", NodeID: types.NodeID(strings.Repeat("b", 40))}
db := dbm.NewMemDB()
peerManager, err := p2p.NewPeerManager(selfID, db, p2p.PeerManagerOptions{})
require.NoError(t, err)
// Getting a height should default to 0, for unknown peers and
// for known peers without height.
added, err := peerManager.Add(a)
require.NoError(t, err)
require.True(t, added)
require.EqualValues(t, 0, peerManager.GetHeight(a.NodeID))
require.EqualValues(t, 0, peerManager.GetHeight(b.NodeID))
// Setting a height should work for a known node.
require.NoError(t, peerManager.SetHeight(a.NodeID, 3))
require.EqualValues(t, 3, peerManager.GetHeight(a.NodeID))
// Setting a height should add an unknown node.
require.Equal(t, []types.NodeID{a.NodeID}, peerManager.Peers())
require.NoError(t, peerManager.SetHeight(b.NodeID, 7))
require.EqualValues(t, 7, peerManager.GetHeight(b.NodeID))
require.ElementsMatch(t, []types.NodeID{a.NodeID, b.NodeID}, peerManager.Peers())
// The heights should not be persisted.
peerManager, err = p2p.NewPeerManager(selfID, db, p2p.PeerManagerOptions{})
require.NoError(t, err)
require.ElementsMatch(t, []types.NodeID{a.NodeID, b.NodeID}, peerManager.Peers())
require.Zero(t, peerManager.GetHeight(a.NodeID))
require.Zero(t, peerManager.GetHeight(b.NodeID))
}
internal/p2p/pqueue.go
+4 -1
View File
@@ -70,6 +70,7 @@ var _ queue = (*pqScheduler)(nil)
type pqScheduler struct {
logger log.Logger
metrics *Metrics
lc *metricsLabelCache
size uint
sizes map[uint]uint // cumulative priority sizes
pq *priorityQueue
@@ -88,6 +89,7 @@ type pqScheduler struct {
func newPQScheduler(
logger log.Logger,
m *Metrics,
lc *metricsLabelCache,
chDescs []*ChannelDescriptor,
enqueueBuf, dequeueBuf, capacity uint,
) *pqScheduler {
@@ -117,6 +119,7 @@ func newPQScheduler(
return &pqScheduler{
logger: logger.With("router", "scheduler"),
metrics: m,
lc: lc,
chDescs: chDescsCopy,
capacity: capacity,
chPriorities: chPriorities,
@@ -251,7 +254,7 @@ func (s *pqScheduler) process(ctx context.Context) {
s.metrics.PeerSendBytesTotal.With(
"chID", chIDStr,
"peer_id", string(pqEnv.envelope.To),
"message_type", s.metrics.ValueToMetricLabel(pqEnv.envelope.Message)).Add(float64(pqEnv.size))
"message_type", s.lc.ValueToMetricLabel(pqEnv.envelope.Message)).Add(float64(pqEnv.size))
s.metrics.PeerPendingSendBytes.With(
"peer_id", string(pqEnv.envelope.To)).Add(float64(-pqEnv.size))
select {
internal/p2p/pqueue_test.go
+1 -1
View File
@@ -17,7 +17,7 @@ func TestCloseWhileDequeueFull(t *testing.T) {
chDescs := []*ChannelDescriptor{
{ID: 0x01, Priority: 1},
}
pqueue := newPQScheduler(log.NewNopLogger(), NopMetrics(), chDescs, uint(enqueueLength), 1, 120)
pqueue := newPQScheduler(log.NewNopLogger(), NopMetrics(), newMetricsLabelCache(), chDescs, uint(enqueueLength), 1, 120)
for i := 0; i < enqueueLength; i++ {
pqueue.enqueue() <- Envelope{
internal/p2p/router.go
+20 -5
View File
@@ -148,7 +148,9 @@ type Router struct {
*service.BaseService
logger log.Logger
metrics *Metrics
metrics *Metrics
lc *metricsLabelCache
options RouterOptions
privKey crypto.PrivKey
peerManager *PeerManager
@@ -193,6 +195,7 @@ func NewRouter(
router := &Router{
logger: logger,
metrics: metrics,
lc: newMetricsLabelCache(),
privKey: privKey,
nodeInfoProducer: nodeInfoProducer,
connTracker: newConnTracker(
@@ -226,7 +229,7 @@ func (r *Router) createQueueFactory(ctx context.Context) (func(int) queue, error
size++
}
q := newPQScheduler(r.logger, r.metrics, r.chDescs, uint(size)/2, uint(size)/2, defaultCapacity)
q := newPQScheduler(r.logger, r.metrics, r.lc, r.chDescs, uint(size)/2, uint(size)/2, defaultCapacity)
q.start(ctx)
return q
}, nil
@@ -393,9 +396,21 @@ func (r *Router) routeChannel(
return
}
r.logger.Error("peer error, evicting", "peer", peerError.NodeID, "err", peerError.Err)
shouldEvict := peerError.Fatal || r.peerManager.HasMaxPeerCapacity()
r.logger.Error("peer error",
"peer", peerError.NodeID,
"err", peerError.Err,
"evicting", shouldEvict,
)
if shouldEvict {
r.peerManager.Errored(peerError.NodeID, peerError.Err)
} else {
r.peerManager.processPeerEvent(ctx, PeerUpdate{
NodeID: peerError.NodeID,
Status: PeerStatusBad,
})
}
r.peerManager.Errored(peerError.NodeID, peerError.Err)
case <-ctx.Done():
return
}
@@ -839,7 +854,7 @@ func (r *Router) receivePeer(ctx context.Context, peerID types.NodeID, conn Conn
r.metrics.PeerReceiveBytesTotal.With(
"chID", fmt.Sprint(chID),
"peer_id", string(peerID),
"message_type", r.metrics.ValueToMetricLabel(msg)).Add(float64(proto.Size(msg)))
"message_type", r.lc.ValueToMetricLabel(msg)).Add(float64(proto.Size(msg)))
r.metrics.RouterChannelQueueSend.Observe(time.Since(start).Seconds())
r.logger.Debug("received message", "peer", peerID, "message", msg)
internal/proxy/client_test.go
+3 -3
View File
@@ -58,7 +58,7 @@ func (app *appConnTest) Info(ctx context.Context, req *types.RequestInfo) (*type
var SOCKET = "socket"
func TestEcho(t *testing.T) {
sockPath := fmt.Sprintf("unix:///tmp/echo_%v.sock", tmrand.Str(6))
sockPath := fmt.Sprintf("unix://%s/echo_%v.sock", t.TempDir(), tmrand.Str(6))
logger := log.NewNopLogger()
client, err := abciclient.NewClient(logger, sockPath, SOCKET, true)
if err != nil {
@@ -98,7 +98,7 @@ func TestEcho(t *testing.T) {
func BenchmarkEcho(b *testing.B) {
b.StopTimer() // Initialize
sockPath := fmt.Sprintf("unix:///tmp/echo_%v.sock", tmrand.Str(6))
sockPath := fmt.Sprintf("unix://%s/echo_%v.sock", b.TempDir(), tmrand.Str(6))
logger := log.NewNopLogger()
client, err := abciclient.NewClient(logger, sockPath, SOCKET, true)
if err != nil {
@@ -146,7 +146,7 @@ func TestInfo(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sockPath := fmt.Sprintf("unix:///tmp/echo_%v.sock", tmrand.Str(6))
sockPath := fmt.Sprintf("unix://%s/echo_%v.sock", t.TempDir(), tmrand.Str(6))
logger := log.NewNopLogger()
client, err := abciclient.NewClient(logger, sockPath, SOCKET, true)
if err != nil {
internal/proxy/metrics.gen.go
+32
View File
@@ -0,0 +1,32 @@
// Code generated by metricsgen. DO NOT EDIT.
package proxy
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
MethodTiming: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "method_timing",
Help: "Timing for each ABCI method.",
Buckets: []float64{.0001, .0004, .002, .009, .02, .1, .65, 2, 6, 25},
}, append(labels, "method", "type")).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
MethodTiming: discard.NewHistogram(),
}
}
internal/proxy/metrics.go
+4 -32
View File
@@ -2,9 +2,6 @@ package proxy
import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
const (
@@ -13,35 +10,10 @@ const (
MetricsSubsystem = "abci_connection"
)
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics contains the prometheus metrics exposed by the proxy package.
type Metrics struct {
MethodTiming metrics.Histogram
}
// PrometheusMetrics constructs a Metrics instance that collects metrics samples.
// The resulting metrics will be prefixed with namespace and labeled with the
// defaultLabelsAndValues. defaultLabelsAndValues must be a list of string pairs
// where the first of each pair is the label and the second is the value.
func PrometheusMetrics(namespace string, defaultLabelsAndValues ...string) *Metrics {
defaultLabels := []string{}
for i := 0; i < len(defaultLabelsAndValues); i += 2 {
defaultLabels = append(defaultLabels, defaultLabelsAndValues[i])
}
return &Metrics{
MethodTiming: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "method_timing",
Help: "ABCI Method Timing",
Buckets: []float64{.0001, .0004, .002, .009, .02, .1, .65, 2, 6, 25},
}, append(defaultLabels, []string{"method", "type"}...)).With(defaultLabelsAndValues...),
}
}
// NopMetrics constructs a Metrics instance that discards all samples and is suitable
// for testing.
func NopMetrics() *Metrics {
return &Metrics{
MethodTiming: discard.NewHistogram(),
}
// Timing for each ABCI method.
MethodTiming metrics.Histogram `metrics_bucketsizes:".0001,.0004,.002,.009,.02,.1,.65,2,6,25" metrics_labels:"method, type"`
}
internal/rpc/core/env.go
+2 -1
View File
@@ -236,7 +236,8 @@ func (env *Environment) StartService(ctx context.Context, conf *config.Config) (
// If necessary adjust global WriteTimeout to ensure it's greater than
// TimeoutBroadcastTxCommit.
// See https://github.com/tendermint/tendermint/issues/3435
if cfg.WriteTimeout <= conf.RPC.TimeoutBroadcastTxCommit {
// Note we don't need to adjust anything if the timeout is already unlimited.
if cfg.WriteTimeout > 0 && cfg.WriteTimeout <= conf.RPC.TimeoutBroadcastTxCommit {
cfg.WriteTimeout = conf.RPC.TimeoutBroadcastTxCommit + 1*time.Second
}
internal/rpc/core/routes.go
+1 -1
View File
@@ -28,7 +28,7 @@ func NewRoutesMap(svc RPCService, opts *RouteOptions) RoutesMap {
out := RoutesMap{
// Event subscription. Note that subscribe, unsubscribe, and
// unsubscribe_all are only available via the websocket endpoint.
"events": rpc.NewRPCFunc(svc.Events),
"events": rpc.NewRPCFunc(svc.Events).Timeout(0),
"subscribe": rpc.NewWSRPCFunc(svc.Subscribe),
"unsubscribe": rpc.NewWSRPCFunc(svc.Unsubscribe),
"unsubscribe_all": rpc.NewWSRPCFunc(svc.UnsubscribeAll),
internal/state/execution.go
+80 -41
View File
@@ -1,13 +1,14 @@
package state
import (
"bytes"
"context"
"errors"
"fmt"
"time"
abciclient "github.com/tendermint/tendermint/abci/client"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/crypto/encoding"
"github.com/tendermint/tendermint/crypto/merkle"
"github.com/tendermint/tendermint/internal/eventbus"
@@ -87,9 +88,8 @@ func (blockExec *BlockExecutor) CreateProposalBlock(
ctx context.Context,
height int64,
state State,
commit *types.Commit,
lastExtCommit *types.ExtendedCommit,
proposerAddr []byte,
votes []*types.Vote,
) (*types.Block, error) {
maxBytes := state.ConsensusParams.Block.MaxBytes
@@ -101,15 +101,14 @@ func (blockExec *BlockExecutor) CreateProposalBlock(
maxDataBytes := types.MaxDataBytes(maxBytes, evSize, state.Validators.Size())
txs := blockExec.mempool.ReapMaxBytesMaxGas(maxDataBytes, maxGas)
commit := lastExtCommit.ToCommit()
block := state.MakeBlock(height, txs, commit, evidence, proposerAddr)
localLastCommit := buildLastCommitInfo(block, blockExec.store, state.InitialHeight)
rpp, err := blockExec.appClient.PrepareProposal(
ctx,
&abci.RequestPrepareProposal{
MaxTxBytes: maxDataBytes,
Txs: block.Txs.ToSliceOfBytes(),
LocalLastCommit: extendedCommitInfo(localLastCommit, votes),
LocalLastCommit: buildExtendedCommitInfo(lastExtCommit, blockExec.store, state.InitialHeight, state.ConsensusParams.ABCI),
ByzantineValidators: block.Evidence.ToABCI(),
Height: block.Height,
Time: block.Time,
@@ -247,6 +246,10 @@ func (blockExec *BlockExecutor) ApplyBlock(
}
if len(validatorUpdates) > 0 {
blockExec.logger.Debug("updates to validators", "updates", types.ValidatorListString(validatorUpdates))
blockExec.metrics.ValidatorSetUpdates.Add(1)
}
if finalizeBlockResponse.ConsensusParamUpdates != nil {
blockExec.metrics.ConsensusParamUpdates.Add(1)
}
// Update the state with the block and responses.
@@ -318,7 +321,7 @@ func (blockExec *BlockExecutor) VerifyVoteExtension(ctx context.Context, vote *t
}
if !resp.IsOK() {
return types.ErrVoteInvalidExtension
return errors.New("invalid vote extension")
}
return nil
@@ -377,14 +380,14 @@ func (blockExec *BlockExecutor) Commit(
func buildLastCommitInfo(block *types.Block, store Store, initialHeight int64) abci.CommitInfo {
if block.Height == initialHeight {
// there is no last commmit for the initial height.
// there is no last commit for the initial height.
// return an empty value.
return abci.CommitInfo{}
}
lastValSet, err := store.LoadValidators(block.Height - 1)
if err != nil {
panic(err)
panic(fmt.Errorf("failed to load validator set at height %d: %w", block.Height-1, err))
}
var (
@@ -406,7 +409,7 @@ func buildLastCommitInfo(block *types.Block, store Store, initialHeight int64) a
commitSig := block.LastCommit.Signatures[i]
votes[i] = abci.VoteInfo{
Validator: types.TM2PB.Validator(val),
SignedLastBlock: !commitSig.Absent(),
SignedLastBlock: commitSig.BlockIDFlag != types.BlockIDFlagAbsent,
}
}
@@ -416,44 +419,75 @@ func buildLastCommitInfo(block *types.Block, store Store, initialHeight int64) a
}
}
// extendedCommitInfo expects a CommitInfo struct along with all of the
// original votes relating to that commit, including their vote extensions. The
// order of votes does not matter.
func extendedCommitInfo(c abci.CommitInfo, votes []*types.Vote) abci.ExtendedCommitInfo {
if len(c.Votes) != len(votes) {
panic(fmt.Sprintf("extendedCommitInfo: number of votes from commit differ from the number of votes supplied (%d != %d)", len(c.Votes), len(votes)))
// buildExtendedCommitInfo populates an ABCI extended commit from the
// corresponding Tendermint extended commit ec, using the stored validator set
// from ec. It requires ec to include the original precommit votes along with
// the vote extensions from the last commit.
//
// For heights below the initial height, for which we do not have the required
// data, it returns an empty record.
//
// Assumes that the commit signatures are sorted according to validator index.
func buildExtendedCommitInfo(ec *types.ExtendedCommit, store Store, initialHeight int64, ap types.ABCIParams) abci.ExtendedCommitInfo {
if ec.Height < initialHeight {
// There are no extended commits for heights below the initial height.
return abci.ExtendedCommitInfo{}
}
votesByVal := make(map[string]*types.Vote)
for _, vote := range votes {
if vote != nil {
valAddr := vote.ValidatorAddress.String()
if _, ok := votesByVal[valAddr]; ok {
panic(fmt.Sprintf("extendedCommitInfo: found duplicate vote for validator with address %s", valAddr))
}
votesByVal[valAddr] = vote
valSet, err := store.LoadValidators(ec.Height)
if err != nil {
panic(fmt.Errorf("failed to load validator set at height %d, initial height %d: %w", ec.Height, initialHeight, err))
}
var (
ecSize = ec.Size()
valSetLen = len(valSet.Validators)
)
// Ensure that the size of the validator set in the extended commit matches
// the size of the validator set in the state store.
if ecSize != valSetLen {
panic(fmt.Errorf(
"extended commit size (%d) does not match validator set length (%d) at height %d\n\n%v\n\n%v",
ecSize, valSetLen, ec.Height, ec.ExtendedSignatures, valSet.Validators,
))
}
votes := make([]abci.ExtendedVoteInfo, ecSize)
for i, val := range valSet.Validators {
ecs := ec.ExtendedSignatures[i]
// Absent signatures have empty validator addresses, but otherwise we
// expect the validator addresses to be the same.
if ecs.BlockIDFlag != types.BlockIDFlagAbsent && !bytes.Equal(ecs.ValidatorAddress, val.Address) {
panic(fmt.Errorf("validator address of extended commit signature in position %d (%s) does not match the corresponding validator's at height %d (%s)",
i, ecs.ValidatorAddress, ec.Height, val.Address,
))
}
}
vs := make([]abci.ExtendedVoteInfo, len(c.Votes))
for i := range vs {
var ext []byte
// votes[i] will be nil if c.Votes[i].SignedLastBlock is false
if c.Votes[i].SignedLastBlock {
valAddr := crypto.Address(c.Votes[i].Validator.Address).String()
vote, ok := votesByVal[valAddr]
if !ok || vote == nil {
panic(fmt.Sprintf("extendedCommitInfo: validator with address %s signed last block, but could not find vote for it", valAddr))
// Check if vote extensions were enabled during the commit's height: ec.Height.
// ec is the commit from the previous height, so if extensions were enabled
// during that height, we ensure they are present and deliver the data to
// the proposer. If they were not enabled during this previous height, we
// will not deliver extension data.
if ap.VoteExtensionsEnabled(ec.Height) && ecs.BlockIDFlag == types.BlockIDFlagCommit {
if err := ecs.EnsureExtension(); err != nil {
panic(fmt.Errorf("commit at height %d received with missing vote extensions data", ec.Height))
}
ext = vote.Extension
ext = ecs.Extension
}
vs[i] = abci.ExtendedVoteInfo{
Validator: c.Votes[i].Validator,
SignedLastBlock: c.Votes[i].SignedLastBlock,
votes[i] = abci.ExtendedVoteInfo{
Validator: types.TM2PB.Validator(val),
SignedLastBlock: ecs.BlockIDFlag != types.BlockIDFlagAbsent,
VoteExtension: ext,
}
}
return abci.ExtendedCommitInfo{
Round: c.Round,
Votes: vs,
Round: ec.Round,
Votes: votes,
}
}
@@ -500,7 +534,7 @@ func (state State) Update(
if len(validatorUpdates) > 0 {
err := nValSet.UpdateWithChangeSet(validatorUpdates)
if err != nil {
return state, fmt.Errorf("error changing validator set: %w", err)
return state, fmt.Errorf("changing validator set: %w", err)
}
// Change results from this height but only applies to the next next height.
lastHeightValsChanged = header.Height + 1 + 1
@@ -517,7 +551,12 @@ func (state State) Update(
nextParams = state.ConsensusParams.UpdateConsensusParams(consensusParamUpdates)
err := nextParams.ValidateConsensusParams()
if err != nil {
return state, fmt.Errorf("error updating consensus params: %w", err)
return state, fmt.Errorf("updating consensus params: %w", err)
}
err = state.ConsensusParams.ValidateUpdate(consensusParamUpdates, header.Height)
if err != nil {
return state, fmt.Errorf("updating consensus params: %w", err)
}
state.Version.Consensus.App = nextParams.Version.AppVersion
internal/state/execution_test.go
+144 -30
View File
@@ -79,9 +79,10 @@ func TestApplyBlock(t *testing.T) {
assert.EqualValues(t, 1, state.Version.Consensus.App, "App version wasn't updated")
}
// TestFinalizeBlockDecidedLastCommit ensures we correctly send the DecidedLastCommit to the
// application. The test ensures that the DecidedLastCommit properly reflects
// which validators signed the preceding block.
// TestFinalizeBlockDecidedLastCommit ensures we correctly send the
// DecidedLastCommit to the application. The test ensures that the
// DecidedLastCommit properly reflects which validators signed the preceding
// block.
func TestFinalizeBlockDecidedLastCommit(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -96,7 +97,7 @@ func TestFinalizeBlockDecidedLastCommit(t *testing.T) {
state, stateDB, privVals := makeState(t, 7, 1)
stateStore := sm.NewStore(stateDB)
absentSig := types.NewCommitSigAbsent()
absentSig := types.NewExtendedCommitSigAbsent()
testCases := []struct {
name string
@@ -134,12 +135,12 @@ func TestFinalizeBlockDecidedLastCommit(t *testing.T) {
for idx, isAbsent := range tc.absentCommitSigs {
if isAbsent {
lastCommit.Signatures[idx] = absentSig
lastCommit.ExtendedSignatures[idx] = absentSig
}
}
// block for height 2
block := sf.MakeBlock(state, 2, lastCommit)
block := sf.MakeBlock(state, 2, lastCommit.ToCommit())
bps, err := block.MakePartSet(testPartSize)
require.NoError(t, err)
blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: bps.Header()}
@@ -198,12 +199,15 @@ func TestFinalizeBlockByzantineValidators(t *testing.T) {
ConflictingBlock: &types.LightBlock{
SignedHeader: &types.SignedHeader{
Header: header,
Commit: types.NewCommit(10, 0, makeBlockID(header.Hash(), 100, []byte("partshash")), []types.CommitSig{{
BlockIDFlag: types.BlockIDFlagNil,
ValidatorAddress: crypto.AddressHash([]byte("validator_address")),
Timestamp: defaultEvidenceTime,
Signature: crypto.CRandBytes(types.MaxSignatureSize),
}}),
Commit: &types.Commit{
Height: 10,
BlockID: makeBlockID(header.Hash(), 100, []byte("partshash")),
Signatures: []types.CommitSig{{
BlockIDFlag: types.BlockIDFlagNil,
ValidatorAddress: crypto.AddressHash([]byte("validator_address")),
Timestamp: defaultEvidenceTime,
Signature: crypto.CRandBytes(types.MaxSignatureSize)}},
},
},
ValidatorSet: state.Validators,
},
@@ -324,8 +328,10 @@ func TestProcessProposal(t *testing.T) {
lastCommitSig = append(lastCommitSig, vote.CommitSig())
}
lastCommit := types.NewCommit(height-1, 0, types.BlockID{}, lastCommitSig)
block1 := sf.MakeBlock(state, height, lastCommit)
block1 := sf.MakeBlock(state, height, &types.Commit{
Height: height - 1,
Signatures: lastCommitSig,
})
block1.Txs = txs
expectedRpp := &abci.RequestProcessProposal{
@@ -653,8 +659,8 @@ func TestEmptyPrepareProposal(t *testing.T) {
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit, votes := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
_, err = blockExec.CreateProposalBlock(ctx, height, state, commit, pa, votes)
commit, _ := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
_, err = blockExec.CreateProposalBlock(ctx, height, state, commit, pa)
require.NoError(t, err)
}
@@ -708,8 +714,8 @@ func TestPrepareProposalErrorOnNonExistingRemoved(t *testing.T) {
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit, votes := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, votes)
commit, _ := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa)
require.ErrorContains(t, err, "new transaction incorrectly marked as removed")
require.Nil(t, block)
@@ -764,8 +770,8 @@ func TestPrepareProposalRemoveTxs(t *testing.T) {
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit, votes := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, votes)
commit, _ := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa)
require.NoError(t, err)
require.Len(t, block.Data.Txs.ToSliceOfBytes(), len(trs)-2)
@@ -823,8 +829,8 @@ func TestPrepareProposalAddedTxsIncluded(t *testing.T) {
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit, votes := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, votes)
commit, _ := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa)
require.NoError(t, err)
require.Equal(t, txs[0], block.Data.Txs[0])
@@ -879,8 +885,8 @@ func TestPrepareProposalReorderTxs(t *testing.T) {
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit, votes := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, votes)
commit, _ := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa)
require.NoError(t, err)
for i, tx := range block.Data.Txs {
require.Equal(t, types.Tx(trs[i].Tx), tx)
@@ -939,9 +945,8 @@ func TestPrepareProposalErrorOnTooManyTxs(t *testing.T) {
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit, votes := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, votes)
commit, _ := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa)
require.ErrorContains(t, err, "transaction data size exceeds maximum")
require.Nil(t, block, "")
@@ -991,15 +996,124 @@ func TestPrepareProposalErrorOnPrepareProposalError(t *testing.T) {
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit, votes := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, votes)
commit, _ := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa)
require.Nil(t, block)
require.ErrorContains(t, err, "an injected error")
mp.AssertExpectations(t)
}
// TestCreateProposalBlockPanicOnAbsentVoteExtensions ensures that the CreateProposalBlock
// call correctly panics when the vote extension data is missing from the extended commit
// data that the method receives.
func TestCreateProposalAbsentVoteExtensions(t *testing.T) {
for _, testCase := range []struct {
name string
// The height that is about to be proposed
height int64
// The first height during which vote extensions will be required for consensus to proceed.
extensionEnableHeight int64
expectPanic bool
}{
{
name: "missing extension data on first required height",
height: 2,
extensionEnableHeight: 1,
expectPanic: true,
},
{
name: "missing extension during before required height",
height: 2,
extensionEnableHeight: 2,
expectPanic: false,
},
{
name: "missing extension data and not required",
height: 2,
extensionEnableHeight: 0,
expectPanic: false,
},
{
name: "missing extension data and required in two heights",
height: 2,
extensionEnableHeight: 3,
expectPanic: false,
},
} {
t.Run(testCase.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
logger := log.NewNopLogger()
eventBus := eventbus.NewDefault(logger)
require.NoError(t, eventBus.Start(ctx))
app := abcimocks.NewApplication(t)
if !testCase.expectPanic {
app.On("PrepareProposal", mock.Anything, mock.Anything).Return(&abci.ResponsePrepareProposal{}, nil)
}
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
err := proxyApp.Start(ctx)
require.NoError(t, err)
state, stateDB, privVals := makeState(t, 1, int(testCase.height-1))
stateStore := sm.NewStore(stateDB)
state.ConsensusParams.ABCI.VoteExtensionsEnableHeight = testCase.extensionEnableHeight
mp := &mpmocks.Mempool{}
mp.On("Lock").Return()
mp.On("Unlock").Return()
mp.On("FlushAppConn", mock.Anything).Return(nil)
mp.On("Update",
mock.Anything,
mock.Anything,
mock.Anything,
mock.Anything,
mock.Anything,
mock.Anything).Return(nil)
mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs{})
blockExec := sm.NewBlockExecutor(
stateStore,
logger,
proxyApp,
mp,
sm.EmptyEvidencePool{},
nil,
eventBus,
sm.NopMetrics(),
)
block := sf.MakeBlock(state, testCase.height, new(types.Commit))
bps, err := block.MakePartSet(testPartSize)
require.NoError(t, err)
blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: bps.Header()}
pa, _ := state.Validators.GetByIndex(0)
lastCommit, _ := makeValidCommit(ctx, t, testCase.height-1, blockID, state.Validators, privVals)
stripSignatures(lastCommit)
if testCase.expectPanic {
require.Panics(t, func() {
blockExec.CreateProposalBlock(ctx, testCase.height, state, lastCommit, pa) //nolint:errcheck
})
} else {
_, err = blockExec.CreateProposalBlock(ctx, testCase.height, state, lastCommit, pa)
require.NoError(t, err)
}
})
}
}
func stripSignatures(ec *types.ExtendedCommit) {
for i, commitSig := range ec.ExtendedSignatures {
commitSig.Extension = nil
commitSig.ExtensionSignature = nil
ec.ExtendedSignatures[i] = commitSig
}
}
func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) types.BlockID {
var (
h = make([]byte, crypto.HashSize)
internal/state/helpers_test.go
+9 -5
View File
@@ -39,7 +39,7 @@ func makeAndCommitGoodBlock(
blockExec *sm.BlockExecutor,
privVals map[string]types.PrivValidator,
evidence []types.Evidence,
) (sm.State, types.BlockID, *types.Commit) {
) (sm.State, types.BlockID, *types.ExtendedCommit) {
t.Helper()
// A good block passes
@@ -82,19 +82,23 @@ func makeValidCommit(
blockID types.BlockID,
vals *types.ValidatorSet,
privVals map[string]types.PrivValidator,
) (*types.Commit, []*types.Vote) {
) (*types.ExtendedCommit, []*types.Vote) {
t.Helper()
sigs := make([]types.CommitSig, vals.Size())
sigs := make([]types.ExtendedCommitSig, vals.Size())
votes := make([]*types.Vote, vals.Size())
for i := 0; i < vals.Size(); i++ {
_, val := vals.GetByIndex(int32(i))
vote, err := factory.MakeVote(ctx, privVals[val.Address.String()], chainID, int32(i), height, 0, 2, blockID, time.Now())
require.NoError(t, err)
sigs[i] = vote.CommitSig()
sigs[i] = vote.ExtendedCommitSig()
votes[i] = vote
}
return types.NewCommit(height, 0, blockID, sigs), votes
return &types.ExtendedCommit{
Height: height,
BlockID: blockID,
ExtendedSignatures: sigs,
}, votes
}
func makeState(t *testing.T, nVals, height int) (sm.State, dbm.DB, map[string]types.PrivValidator) {
internal/state/indexer/block/kv/kv.go
+1 -1
View File
@@ -65,7 +65,7 @@ func (idx *BlockerIndexer) Index(bh types.EventDataNewBlockHeader) error {
}
// 2. index FinalizeBlock events
if err := idx.indexEvents(batch, bh.ResultFinalizeBlock.Events, types.EventTypeFinalizeBlock, height); err != nil {
if err := idx.indexEvents(batch, bh.ResultFinalizeBlock.Events, "finalize_block", height); err != nil {
return fmt.Errorf("failed to index FinalizeBlock events: %w", err)
}
internal/state/indexer/metrics.gen.go
+51
View File
@@ -0,0 +1,51 @@
// Code generated by metricsgen. DO NOT EDIT.
package indexer
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
BlockEventsSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_events_seconds",
Help: "Latency for indexing block events.",
}, labels).With(labelsAndValues...),
TxEventsSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "tx_events_seconds",
Help: "Latency for indexing transaction events.",
}, labels).With(labelsAndValues...),
BlocksIndexed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "blocks_indexed",
Help: "Number of complete blocks indexed.",
}, labels).With(labelsAndValues...),
TransactionsIndexed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "transactions_indexed",
Help: "Number of transactions indexed.",
}, labels).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
BlockEventsSeconds: discard.NewHistogram(),
TxEventsSeconds: discard.NewHistogram(),
BlocksIndexed: discard.NewCounter(),
TransactionsIndexed: discard.NewCounter(),
}
}
internal/state/indexer/metrics.go
+2 -50
View File
@@ -2,12 +2,10 @@ package indexer
import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
//go:generate go run ../../../scripts/metricsgen -struct=Metrics
// MetricsSubsystem is a the subsystem label for the indexer package.
const MetricsSubsystem = "indexer"
@@ -25,49 +23,3 @@ type Metrics struct {
// Number of transactions indexed.
TransactionsIndexed metrics.Counter
}
// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
BlockEventsSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_events_seconds",
Help: "Latency for indexing block events.",
}, labels).With(labelsAndValues...),
TxEventsSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "tx_events_seconds",
Help: "Latency for indexing transaction events.",
}, labels).With(labelsAndValues...),
BlocksIndexed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "blocks_indexed",
Help: "Number of complete blocks indexed.",
}, labels).With(labelsAndValues...),
TransactionsIndexed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "transactions_indexed",
Help: "Number of transactions indexed.",
}, labels).With(labelsAndValues...),
}
}
// NopMetrics returns an indexer metrics stub that discards all samples.
func NopMetrics() *Metrics {
return &Metrics{
BlockEventsSeconds: discard.NewHistogram(),
TxEventsSeconds: discard.NewHistogram(),
BlocksIndexed: discard.NewCounter(),
TransactionsIndexed: discard.NewCounter(),
}
}
internal/state/metrics.gen.go
+46
View File
@@ -0,0 +1,46 @@
// Code generated by metricsgen. DO NOT EDIT.
package state
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
BlockProcessingTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_processing_time",
Help: "Time between BeginBlock and EndBlock.",
Buckets: stdprometheus.LinearBuckets(1, 10, 10),
}, labels).With(labelsAndValues...),
ConsensusParamUpdates: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "consensus_param_updates",
Help: "Number of consensus parameter updates returned by the application since process start.",
}, labels).With(labelsAndValues...),
ValidatorSetUpdates: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "validator_set_updates",
Help: "Number of validator set updates returned by the application since process start.",
}, labels).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
BlockProcessingTime: discard.NewHistogram(),
ConsensusParamUpdates: discard.NewCounter(),
ValidatorSetUpdates: discard.NewCounter(),
}
}
internal/state/metrics.go
+11 -28
View File
@@ -2,9 +2,6 @@ package state
import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
const (
@@ -13,34 +10,20 @@ const (
MetricsSubsystem = "state"
)
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics contains metrics exposed by this package.
type Metrics struct {
// Time between BeginBlock and EndBlock.
BlockProcessingTime metrics.Histogram
}
BlockProcessingTime metrics.Histogram `metrics_buckettype:"lin" metrics_bucketsizes:"1,10,10"`
// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
BlockProcessingTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "block_processing_time",
Help: "Time between BeginBlock and EndBlock in ms.",
Buckets: stdprometheus.LinearBuckets(1, 10, 10),
}, labels).With(labelsAndValues...),
}
}
// ConsensusParamUpdates is the total number of times the application has
// udated the consensus params since process start.
//metrics:Number of consensus parameter updates returned by the application since process start.
ConsensusParamUpdates metrics.Counter
// NopMetrics returns no-op Metrics.
func NopMetrics() *Metrics {
return &Metrics{
BlockProcessingTime: discard.NewHistogram(),
}
// ValidatorSetUpdates is the total number of times the application has
// udated the validator set since process start.
//metrics:Number of validator set updates returned by the application since process start.
ValidatorSetUpdates metrics.Counter
}
internal/state/mocks/block_store.go
+21
View File
@@ -107,6 +107,22 @@ func (_m *BlockStore) LoadBlockCommit(height int64) *types.Commit {
return r0
}
// LoadBlockExtendedCommit provides a mock function with given fields: height
func (_m *BlockStore) LoadBlockExtendedCommit(height int64) *types.ExtendedCommit {
ret := _m.Called(height)
var r0 *types.ExtendedCommit
if rf, ok := ret.Get(0).(func(int64) *types.ExtendedCommit); ok {
r0 = rf(height)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*types.ExtendedCommit)
}
}
return r0
}
// LoadBlockMeta provides a mock function with given fields: height
func (_m *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
ret := _m.Called(height)
@@ -197,6 +213,11 @@ func (_m *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
_m.Called(block, blockParts, seenCommit)
}
// SaveBlockWithExtendedCommit provides a mock function with given fields: block, blockParts, seenCommit
func (_m *BlockStore) SaveBlockWithExtendedCommit(block *types.Block, blockParts *types.PartSet, seenCommit *types.ExtendedCommit) {
_m.Called(block, blockParts, seenCommit)
}
// Size provides a mock function with given fields:
func (_m *BlockStore) Size() int64 {
ret := _m.Called()
internal/state/services.go
+2
View File
@@ -27,6 +27,7 @@ type BlockStore interface {
LoadBlock(height int64) *types.Block
SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit)
SaveBlockWithExtendedCommit(block *types.Block, blockParts *types.PartSet, seenCommit *types.ExtendedCommit)
PruneBlocks(height int64) (uint64, error)
@@ -36,6 +37,7 @@ type BlockStore interface {
LoadBlockCommit(height int64) *types.Commit
LoadSeenCommit() *types.Commit
LoadBlockExtendedCommit(height int64) *types.ExtendedCommit
}
//-----------------------------------------------------------------------------
internal/state/store.go
+3 -1
View File
@@ -26,6 +26,9 @@ const (
//------------------------------------------------------------------------
// NB: Before modifying these, cross-check them with those in
// internal/store/store.go
// TODO(thane): Move these and the ones in internal/store/store.go to their own package.
const (
// prefixes are unique across all tm db's
prefixValidators = int64(5)
@@ -134,7 +137,6 @@ func (store dbStore) loadState(key []byte) (state State, err error) {
if err != nil {
return state, err
}
return *sm, nil
}
internal/state/test/factory/block.go
+7 -3
View File
@@ -63,7 +63,7 @@ func makeBlockAndPartSet(
) (*types.Block, *types.PartSet) {
t.Helper()
lastCommit := types.NewCommit(height-1, 0, types.BlockID{}, nil)
lastCommit := &types.Commit{Height: height - 1}
if height > 1 {
vote, err := factory.MakeVote(
ctx,
@@ -73,8 +73,12 @@ func makeBlockAndPartSet(
lastBlockMeta.BlockID,
time.Now())
require.NoError(t, err)
lastCommit = types.NewCommit(vote.Height, vote.Round,
lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
lastCommit = &types.Commit{
Height: vote.Height,
Round: vote.Round,
BlockID: lastBlock.LastBlockID,
Signatures: []types.CommitSig{vote.CommitSig()},
}
}
block := state.MakeBlock(height, []types.Tx{}, lastCommit, nil, state.Validators.GetProposer().Address)
internal/state/validation_test.go
+26 -16
View File
@@ -65,7 +65,8 @@ func TestValidateBlockHeader(t *testing.T) {
eventBus,
sm.NopMetrics(),
)
lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
lastCommit := &types.Commit{}
var lastExtCommit *types.ExtendedCommit
// some bad values
wrongHash := crypto.Checksum([]byte("this hash is wrong"))
@@ -100,7 +101,7 @@ func TestValidateBlockHeader(t *testing.T) {
{"Proposer invalid", func(block *types.Block) { block.ProposerAddress = []byte("wrong size") }},
{"first LastCommit contains signatures", func(block *types.Block) {
block.LastCommit = types.NewCommit(0, 0, types.BlockID{}, []types.CommitSig{types.NewCommitSigAbsent()})
block.LastCommit = &types.Commit{Signatures: []types.CommitSig{types.NewCommitSigAbsent()}}
block.LastCommitHash = block.LastCommit.Hash()
}},
}
@@ -121,8 +122,9 @@ func TestValidateBlockHeader(t *testing.T) {
/*
A good block passes
*/
state, _, lastCommit = makeAndCommitGoodBlock(ctx, t,
state, _, lastExtCommit = makeAndCommitGoodBlock(ctx, t,
state, height, lastCommit, state.Validators.GetProposer().Address, blockExec, privVals, nil)
lastCommit = lastExtCommit.ToCommit()
}
nextHeight := validationTestsStopHeight
@@ -169,8 +171,9 @@ func TestValidateBlockCommit(t *testing.T) {
eventBus,
sm.NopMetrics(),
)
lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
wrongSigsCommit := types.NewCommit(1, 0, types.BlockID{}, nil)
lastCommit := &types.Commit{}
var lastExtCommit *types.ExtendedCommit
wrongSigsCommit := &types.Commit{Height: 1}
badPrivVal := types.NewMockPV()
for height := int64(1); height < validationTestsStopHeight; height++ {
@@ -192,12 +195,12 @@ func TestValidateBlockCommit(t *testing.T) {
time.Now(),
)
require.NoError(t, err)
wrongHeightCommit := types.NewCommit(
wrongHeightVote.Height,
wrongHeightVote.Round,
state.LastBlockID,
[]types.CommitSig{wrongHeightVote.CommitSig()},
)
wrongHeightCommit := &types.Commit{
Height: wrongHeightVote.Height,
Round: wrongHeightVote.Round,
BlockID: state.LastBlockID,
Signatures: []types.CommitSig{wrongHeightVote.CommitSig()},
}
block := statefactory.MakeBlock(state, height, wrongHeightCommit)
err = blockExec.ValidateBlock(ctx, state, block)
_, isErrInvalidCommitHeight := err.(types.ErrInvalidCommitHeight)
@@ -220,7 +223,7 @@ func TestValidateBlockCommit(t *testing.T) {
A good block passes
*/
var blockID types.BlockID
state, blockID, lastCommit = makeAndCommitGoodBlock(
state, blockID, lastExtCommit = makeAndCommitGoodBlock(
ctx,
t,
state,
@@ -231,6 +234,7 @@ func TestValidateBlockCommit(t *testing.T) {
privVals,
nil,
)
lastCommit = lastExtCommit.ToCommit()
/*
wrongSigsCommit is fine except for the extra bad precommit
@@ -270,8 +274,12 @@ func TestValidateBlockCommit(t *testing.T) {
goodVote.Signature, badVote.Signature = g.Signature, b.Signature
wrongSigsCommit = types.NewCommit(goodVote.Height, goodVote.Round,
blockID, []types.CommitSig{goodVote.CommitSig(), badVote.CommitSig()})
wrongSigsCommit = &types.Commit{
Height: goodVote.Height,
Round: goodVote.Round,
BlockID: blockID,
Signatures: []types.CommitSig{goodVote.CommitSig(), badVote.CommitSig()},
}
}
}
@@ -319,7 +327,8 @@ func TestValidateBlockEvidence(t *testing.T) {
eventBus,
sm.NopMetrics(),
)
lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
lastCommit := &types.Commit{}
var lastExtCommit *types.ExtendedCommit
for height := int64(1); height < validationTestsStopHeight; height++ {
proposerAddr := state.Validators.GetProposer().Address
@@ -364,7 +373,7 @@ func TestValidateBlockEvidence(t *testing.T) {
evidence = append(evidence, newEv)
}
state, _, lastCommit = makeAndCommitGoodBlock(
state, _, lastExtCommit = makeAndCommitGoodBlock(
ctx,
t,
state,
@@ -375,6 +384,7 @@ func TestValidateBlockEvidence(t *testing.T) {
privVals,
evidence,
)
lastCommit = lastExtCommit.ToCommit()
}
}
internal/statesync/metrics.gen.go
+72
View File
@@ -0,0 +1,72 @@
// Code generated by metricsgen. DO NOT EDIT.
package statesync
import (
"github.com/go-kit/kit/metrics/discard"
prometheus "github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
TotalSnapshots: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "total_snapshots",
Help: "The total number of snapshots discovered.",
}, labels).With(labelsAndValues...),
ChunkProcessAvgTime: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "chunk_process_avg_time",
Help: "The average processing time per chunk.",
}, labels).With(labelsAndValues...),
SnapshotHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "snapshot_height",
Help: "The height of the current snapshot the has been processed.",
}, labels).With(labelsAndValues...),
SnapshotChunk: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "snapshot_chunk",
Help: "The current number of chunks that have been processed.",
}, labels).With(labelsAndValues...),
SnapshotChunkTotal: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "snapshot_chunk_total",
Help: "The total number of chunks in the current snapshot.",
}, labels).With(labelsAndValues...),
BackFilledBlocks: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "back_filled_blocks",
Help: "The current number of blocks that have been back-filled.",
}, labels).With(labelsAndValues...),
BackFillBlocksTotal: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "back_fill_blocks_total",
Help: "The total number of blocks that need to be back-filled.",
}, labels).With(labelsAndValues...),
}
}
func NopMetrics() *Metrics {
return &Metrics{
TotalSnapshots: discard.NewCounter(),
ChunkProcessAvgTime: discard.NewGauge(),
SnapshotHeight: discard.NewGauge(),
SnapshotChunk: discard.NewCounter(),
SnapshotChunkTotal: discard.NewGauge(),
BackFilledBlocks: discard.NewCounter(),
BackFillBlocksTotal: discard.NewGauge(),
}
}
internal/statesync/metrics.go
+14 -75
View File
@@ -2,9 +2,6 @@ package statesync
import (
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/discard"
"github.com/go-kit/kit/metrics/prometheus"
stdprometheus "github.com/prometheus/client_golang/prometheus"
)
const (
@@ -12,80 +9,22 @@ const (
MetricsSubsystem = "statesync"
)
//go:generate go run ../../scripts/metricsgen -struct=Metrics
// Metrics contains metrics exposed by this package.
type Metrics struct {
TotalSnapshots metrics.Counter
// The total number of snapshots discovered.
TotalSnapshots metrics.Counter
// The average processing time per chunk.
ChunkProcessAvgTime metrics.Gauge
SnapshotHeight metrics.Gauge
SnapshotChunk metrics.Counter
SnapshotChunkTotal metrics.Gauge
BackFilledBlocks metrics.Counter
// The height of the current snapshot the has been processed.
SnapshotHeight metrics.Gauge
// The current number of chunks that have been processed.
SnapshotChunk metrics.Counter
// The total number of chunks in the current snapshot.
SnapshotChunkTotal metrics.Gauge
// The current number of blocks that have been back-filled.
BackFilledBlocks metrics.Counter
// The total number of blocks that need to be back-filled.
BackFillBlocksTotal metrics.Gauge
}
// PrometheusMetrics returns Metrics build using Prometheus client library.
// Optionally, labels can be provided along with their values ("foo",
// "fooValue").
func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
labels := []string{}
for i := 0; i < len(labelsAndValues); i += 2 {
labels = append(labels, labelsAndValues[i])
}
return &Metrics{
TotalSnapshots: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "total_snapshots",
Help: "The total number of snapshots discovered.",
}, labels).With(labelsAndValues...),
ChunkProcessAvgTime: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "chunk_process_avg_time",
Help: "The average processing time per chunk.",
}, labels).With(labelsAndValues...),
SnapshotHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "snapshot_height",
Help: "The height of the current snapshot the has been processed.",
}, labels).With(labelsAndValues...),
SnapshotChunk: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "snapshot_chunk",
Help: "The current number of chunks that have been processed.",
}, labels).With(labelsAndValues...),
SnapshotChunkTotal: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "snapshot_chunks_total",
Help: "The total number of chunks in the current snapshot.",
}, labels).With(labelsAndValues...),
BackFilledBlocks: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "backfilled_blocks",
Help: "The current number of blocks that have been back-filled.",
}, labels).With(labelsAndValues...),
BackFillBlocksTotal: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
Namespace: namespace,
Subsystem: MetricsSubsystem,
Name: "backfilled_blocks_total",
Help: "The total number of blocks that need to be back-filled.",
}, labels).With(labelsAndValues...),
}
}
// NopMetrics returns no-op Metrics.
func NopMetrics() *Metrics {
return &Metrics{
TotalSnapshots: discard.NewCounter(),
ChunkProcessAvgTime: discard.NewGauge(),
SnapshotHeight: discard.NewGauge(),
SnapshotChunk: discard.NewCounter(),
SnapshotChunkTotal: discard.NewGauge(),
BackFilledBlocks: discard.NewCounter(),
BackFillBlocksTotal: discard.NewGauge(),
}
}
internal/statesync/reactor_test.go
+3 -3
View File
@@ -855,13 +855,13 @@ func mockLB(ctx context.Context, t *testing.T, height int64, time time.Time, las
header.NextValidatorsHash = nextVals.Hash()
header.ConsensusHash = types.DefaultConsensusParams().HashConsensusParams()
lastBlockID = factory.MakeBlockIDWithHash(header.Hash())
voteSet := types.NewVoteSet(factory.DefaultTestChainID, height, 0, tmproto.PrecommitType, currentVals)
commit, err := factory.MakeCommit(ctx, lastBlockID, height, 0, voteSet, currentPrivVals, time)
voteSet := types.NewExtendedVoteSet(factory.DefaultTestChainID, height, 0, tmproto.PrecommitType, currentVals)
extCommit, err := factory.MakeExtendedCommit(ctx, lastBlockID, height, 0, voteSet, currentPrivVals, time)
require.NoError(t, err)
return nextVals, nextPrivVals, &types.LightBlock{
SignedHeader: &types.SignedHeader{
Header: header,
Commit: commit,
Commit: extCommit.ToCommit(),
},
ValidatorSet: currentVals,
}
internal/store/store.go
+99 -17
View File
@@ -2,6 +2,7 @@ package store
import (
"bytes"
"errors"
"fmt"
"strconv"
@@ -273,11 +274,34 @@ func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
}
commit, err := types.CommitFromProto(pbc)
if err != nil {
panic(fmt.Errorf("error reading block commit: %w", err))
panic(fmt.Errorf("converting commit to proto: %w", err))
}
return commit
}
// LoadExtendedCommit returns the ExtendedCommit for the given height.
// The extended commit is not guaranteed to contain the same +2/3 precommits data
// as the commit in the block.
func (bs *BlockStore) LoadBlockExtendedCommit(height int64) *types.ExtendedCommit {
pbec := new(tmproto.ExtendedCommit)
bz, err := bs.db.Get(extCommitKey(height))
if err != nil {
panic(fmt.Errorf("fetching extended commit: %w", err))
}
if len(bz) == 0 {
return nil
}
err = proto.Unmarshal(bz, pbec)
if err != nil {
panic(fmt.Errorf("decoding extended commit: %w", err))
}
extCommit, err := types.ExtendedCommitFromProto(pbec)
if err != nil {
panic(fmt.Errorf("converting extended commit: %w", err))
}
return extCommit
}
// LoadSeenCommit returns the last locally seen Commit before being
// cannonicalized. This is useful when we've seen a commit, but there
// has not yet been a new block at `height + 1` that includes this
@@ -298,7 +322,7 @@ func (bs *BlockStore) LoadSeenCommit() *types.Commit {
commit, err := types.CommitFromProto(pbc)
if err != nil {
panic(fmt.Errorf("error from proto commit: %w", err))
panic(fmt.Errorf("converting seen commit: %w", err))
}
return commit
}
@@ -450,17 +474,69 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
if block == nil {
panic("BlockStore can only save a non-nil block")
}
batch := bs.db.NewBatch()
if err := bs.saveBlockToBatch(batch, block, blockParts, seenCommit); err != nil {
panic(err)
}
if err := batch.WriteSync(); err != nil {
panic(err)
}
if err := batch.Close(); err != nil {
panic(err)
}
}
// SaveBlockWithExtendedCommit persists the given block, blockParts, and
// seenExtendedCommit to the underlying db. seenExtendedCommit is stored under
// two keys in the database: as the seenCommit and as the ExtendedCommit data for the
// height. This allows the vote extension data to be persisted for all blocks
// that are saved.
func (bs *BlockStore) SaveBlockWithExtendedCommit(block *types.Block, blockParts *types.PartSet, seenExtendedCommit *types.ExtendedCommit) {
if block == nil {
panic("BlockStore can only save a non-nil block")
}
if err := seenExtendedCommit.EnsureExtensions(); err != nil {
panic(fmt.Errorf("saving block with extensions: %w", err))
}
batch := bs.db.NewBatch()
if err := bs.saveBlockToBatch(batch, block, blockParts, seenExtendedCommit.ToCommit()); err != nil {
panic(err)
}
height := block.Height
pbec := seenExtendedCommit.ToProto()
extCommitBytes := mustEncode(pbec)
if err := batch.Set(extCommitKey(height), extCommitBytes); err != nil {
panic(err)
}
if err := batch.WriteSync(); err != nil {
panic(err)
}
if err := batch.Close(); err != nil {
panic(err)
}
}
func (bs *BlockStore) saveBlockToBatch(batch dbm.Batch, block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) error {
if block == nil {
panic("BlockStore can only save a non-nil block")
}
height := block.Height
hash := block.Hash()
if g, w := height, bs.Height()+1; bs.Base() > 0 && g != w {
panic(fmt.Sprintf("BlockStore can only save contiguous blocks. Wanted %v, got %v", w, g))
return fmt.Errorf("BlockStore can only save contiguous blocks. Wanted %v, got %v", w, g)
}
if !blockParts.IsComplete() {
panic("BlockStore can only save complete block part sets")
return errors.New("BlockStore can only save complete block part sets")
}
if height != seenCommit.Height {
return fmt.Errorf("BlockStore cannot save seen commit of a different height (block: %d, commit: %d)", height, seenCommit.Height)
}
// Save block parts. This must be done before the block meta, since callers
@@ -475,38 +551,32 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
blockMeta := types.NewBlockMeta(block, blockParts)
pbm := blockMeta.ToProto()
if pbm == nil {
panic("nil blockmeta")
return errors.New("nil blockmeta")
}
metaBytes := mustEncode(pbm)
if err := batch.Set(blockMetaKey(height), metaBytes); err != nil {
panic(err)
return err
}
if err := batch.Set(blockHashKey(hash), []byte(fmt.Sprintf("%d", height))); err != nil {
panic(err)
return err
}
pbc := block.LastCommit.ToProto()
blockCommitBytes := mustEncode(pbc)
if err := batch.Set(blockCommitKey(height-1), blockCommitBytes); err != nil {
panic(err)
return err
}
// Save seen commit (seen +2/3 precommits for block)
pbsc := seenCommit.ToProto()
seenCommitBytes := mustEncode(pbsc)
if err := batch.Set(seenCommitKey(), seenCommitBytes); err != nil {
panic(err)
return err
}
if err := batch.WriteSync(); err != nil {
panic(err)
}
if err := batch.Close(); err != nil {
panic(err)
}
return nil
}
func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part, batch dbm.Batch) {
@@ -579,6 +649,9 @@ func (bs *BlockStore) Close() error {
//---------------------------------- KEY ENCODING -----------------------------------------
// key prefixes
// NB: Before modifying these, cross-check them with those in
// internal/state/store.go
// TODO(thane): Move these and the ones in internal/state/store.go to their own package.
const (
// prefixes are unique across all tm db's
prefixBlockMeta = int64(0)
@@ -586,6 +659,7 @@ const (
prefixBlockCommit = int64(2)
prefixSeenCommit = int64(3)
prefixBlockHash = int64(4)
prefixExtCommit = int64(9) // 5..8 are used by state/store
)
func blockMetaKey(height int64) []byte {
@@ -635,6 +709,14 @@ func seenCommitKey() []byte {
return key
}
func extCommitKey(height int64) []byte {
key, err := orderedcode.Append(nil, prefixExtCommit, height)
if err != nil {
panic(err)
}
return key
}
func blockHashKey(hash []byte) []byte {
key, err := orderedcode.Append(nil, prefixBlockHash, string(hash))
if err != nil {
internal/store/store_test.go
+156 -88
View File
@@ -2,7 +2,6 @@ package store
import (
"fmt"
stdlog "log"
"os"
"runtime/debug"
"strings"
@@ -27,22 +26,26 @@ import (
// test.
type cleanupFunc func()
// make a Commit with a single vote containing just the height and a timestamp
func makeTestCommit(height int64, timestamp time.Time) *types.Commit {
commitSigs := []types.CommitSig{{
BlockIDFlag: types.BlockIDFlagCommit,
ValidatorAddress: tmrand.Bytes(crypto.AddressSize),
Timestamp: timestamp,
Signature: []byte("Signature"),
// make an extended commit with a single vote containing just the height and a
// timestamp
func makeTestExtCommit(height int64, timestamp time.Time) *types.ExtendedCommit {
extCommitSigs := []types.ExtendedCommitSig{{
CommitSig: types.CommitSig{
BlockIDFlag: types.BlockIDFlagCommit,
ValidatorAddress: tmrand.Bytes(crypto.AddressSize),
Timestamp: timestamp,
Signature: []byte("Signature"),
},
ExtensionSignature: []byte("ExtensionSignature"),
}}
return types.NewCommit(
height,
0,
types.BlockID{
return &types.ExtendedCommit{
Height: height,
BlockID: types.BlockID{
Hash: crypto.CRandBytes(32),
PartSetHeader: types.PartSetHeader{Hash: crypto.CRandBytes(32), Total: 2},
},
commitSigs)
ExtendedSignatures: extCommitSigs,
}
}
func makeStateAndBlockStore(dir string) (sm.State, *BlockStore, cleanupFunc, error) {
@@ -59,47 +62,11 @@ func makeStateAndBlockStore(dir string) (sm.State, *BlockStore, cleanupFunc, err
return state, NewBlockStore(blockDB), func() { os.RemoveAll(cfg.RootDir) }, nil
}
func freshBlockStore() (*BlockStore, dbm.DB) {
func newInMemoryBlockStore() (*BlockStore, dbm.DB) {
db := dbm.NewMemDB()
return NewBlockStore(db), db
}
var (
state sm.State
block *types.Block
partSet *types.PartSet
part1 *types.Part
part2 *types.Part
seenCommit1 *types.Commit
)
func TestMain(m *testing.M) {
dir, err := os.MkdirTemp("", "store_test")
if err != nil {
stdlog.Fatal(err)
}
var cleanup cleanupFunc
state, _, cleanup, err = makeStateAndBlockStore(dir)
if err != nil {
stdlog.Fatal(err)
}
block = factory.MakeBlock(state, 1, new(types.Commit))
partSet, err = block.MakePartSet(2)
if err != nil {
stdlog.Fatal(err)
}
part1 = partSet.GetPart(0)
part2 = partSet.GetPart(1)
seenCommit1 = makeTestCommit(10, tmtime.Now())
code := m.Run()
cleanup()
os.RemoveAll(dir) // best-effort
os.Exit(code)
}
// TODO: This test should be simplified ...
func TestBlockStoreSaveLoadBlock(t *testing.T) {
state, bs, cleanup, err := makeStateAndBlockStore(t.TempDir())
@@ -120,8 +87,10 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
block := factory.MakeBlock(state, bs.Height()+1, new(types.Commit))
validPartSet, err := block.MakePartSet(2)
require.NoError(t, err)
seenCommit := makeTestCommit(10, tmtime.Now())
bs.SaveBlock(block, partSet, seenCommit)
part2 := validPartSet.GetPart(1)
seenCommit := makeTestExtCommit(block.Header.Height, tmtime.Now())
bs.SaveBlockWithExtendedCommit(block, validPartSet, seenCommit)
require.EqualValues(t, 1, bs.Base(), "expecting the new height to be changed")
require.EqualValues(t, block.Header.Height, bs.Height(), "expecting the new height to be changed")
@@ -139,11 +108,11 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
}
// End of setup, test data
commitAtH10 := makeTestCommit(10, tmtime.Now())
commitAtH10 := makeTestExtCommit(10, tmtime.Now()).ToCommit()
tuples := []struct {
block *types.Block
parts *types.PartSet
seenCommit *types.Commit
seenCommit *types.ExtendedCommit
wantPanic string
wantErr bool
@@ -156,7 +125,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
{
block: newBlock(header1, commitAtH10),
parts: validPartSet,
seenCommit: seenCommit1,
seenCommit: seenCommit,
},
{
@@ -172,22 +141,23 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
ChainID: "block_test",
Time: tmtime.Now(),
ProposerAddress: tmrand.Bytes(crypto.AddressSize)},
makeTestCommit(5, tmtime.Now()),
makeTestExtCommit(5, tmtime.Now()).ToCommit(),
),
parts: validPartSet,
seenCommit: makeTestCommit(5, tmtime.Now()),
seenCommit: makeTestExtCommit(5, tmtime.Now()),
},
{
block: newBlock(header1, commitAtH10),
parts: incompletePartSet,
wantPanic: "only save complete block", // incomplete parts
block: newBlock(header1, commitAtH10),
parts: incompletePartSet,
wantPanic: "only save complete block", // incomplete parts
seenCommit: makeTestExtCommit(10, tmtime.Now()),
},
{
block: newBlock(header1, commitAtH10),
parts: validPartSet,
seenCommit: seenCommit1,
seenCommit: seenCommit,
corruptCommitInDB: true, // Corrupt the DB's commit entry
wantPanic: "error reading block commit",
},
@@ -195,7 +165,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
{
block: newBlock(header1, commitAtH10),
parts: validPartSet,
seenCommit: seenCommit1,
seenCommit: seenCommit,
wantPanic: "unmarshal to tmproto.BlockMeta",
corruptBlockInDB: true, // Corrupt the DB's block entry
},
@@ -203,25 +173,25 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
{
block: newBlock(header1, commitAtH10),
parts: validPartSet,
seenCommit: seenCommit1,
seenCommit: seenCommit,
// Expecting no error and we want a nil back
eraseSeenCommitInDB: true,
},
{
block: newBlock(header1, commitAtH10),
block: block,
parts: validPartSet,
seenCommit: seenCommit1,
seenCommit: seenCommit,
corruptSeenCommitInDB: true,
wantPanic: "error reading block seen commit",
},
{
block: newBlock(header1, commitAtH10),
block: block,
parts: validPartSet,
seenCommit: seenCommit1,
seenCommit: seenCommit,
// Expecting no error and we want a nil back
eraseCommitInDB: true,
@@ -238,10 +208,10 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
for i, tuple := range tuples {
tuple := tuple
bs, db := freshBlockStore()
bs, db := newInMemoryBlockStore()
// SaveBlock
res, err, panicErr := doFn(func() (interface{}, error) {
bs.SaveBlock(tuple.block, tuple.parts, tuple.seenCommit)
bs.SaveBlockWithExtendedCommit(tuple.block, tuple.parts, tuple.seenCommit)
if tuple.block == nil {
return nil, nil
}
@@ -311,6 +281,90 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
}
}
// TestSaveBlockWithExtendedCommitPanicOnAbsentExtension tests that saving a
// block with an extended commit panics when the extension data is absent.
func TestSaveBlockWithExtendedCommitPanicOnAbsentExtension(t *testing.T) {
for _, testCase := range []struct {
name string
malleateCommit func(*types.ExtendedCommit)
shouldPanic bool
}{
{
name: "basic save",
malleateCommit: func(_ *types.ExtendedCommit) {},
shouldPanic: false,
},
{
name: "save commit with no extensions",
malleateCommit: func(c *types.ExtendedCommit) {
c.StripExtensions()
},
shouldPanic: true,
},
} {
t.Run(testCase.name, func(t *testing.T) {
state, bs, cleanup, err := makeStateAndBlockStore(t.TempDir())
require.NoError(t, err)
defer cleanup()
block := factory.MakeBlock(state, bs.Height()+1, new(types.Commit))
seenCommit := makeTestExtCommit(block.Header.Height, tmtime.Now())
ps, err := block.MakePartSet(2)
require.NoError(t, err)
testCase.malleateCommit(seenCommit)
if testCase.shouldPanic {
require.Panics(t, func() {
bs.SaveBlockWithExtendedCommit(block, ps, seenCommit)
})
} else {
bs.SaveBlockWithExtendedCommit(block, ps, seenCommit)
}
})
}
}
// TestLoadBlockExtendedCommit tests loading the extended commit for a previously
// saved block. The load method should return nil when only a commit was saved and
// return the extended commit otherwise.
func TestLoadBlockExtendedCommit(t *testing.T) {
for _, testCase := range []struct {
name string
saveExtended bool
expectResult bool
}{
{
name: "save commit",
saveExtended: false,
expectResult: false,
},
{
name: "save extended commit",
saveExtended: true,
expectResult: true,
},
} {
t.Run(testCase.name, func(t *testing.T) {
state, bs, cleanup, err := makeStateAndBlockStore(t.TempDir())
require.NoError(t, err)
defer cleanup()
block := factory.MakeBlock(state, bs.Height()+1, new(types.Commit))
seenCommit := makeTestExtCommit(block.Header.Height, tmtime.Now())
ps, err := block.MakePartSet(2)
require.NoError(t, err)
if testCase.saveExtended {
bs.SaveBlockWithExtendedCommit(block, ps, seenCommit)
} else {
bs.SaveBlock(block, ps, seenCommit.ToCommit())
}
res := bs.LoadBlockExtendedCommit(block.Height)
if testCase.expectResult {
require.Equal(t, seenCommit, res)
} else {
require.Nil(t, res)
}
})
}
}
func TestLoadBaseMeta(t *testing.T) {
cfg, err := config.ResetTestRoot(t.TempDir(), "blockchain_reactor_test")
require.NoError(t, err)
@@ -324,8 +378,8 @@ func TestLoadBaseMeta(t *testing.T) {
block := factory.MakeBlock(state, h, new(types.Commit))
partSet, err := block.MakePartSet(2)
require.NoError(t, err)
seenCommit := makeTestCommit(h, tmtime.Now())
bs.SaveBlock(block, partSet, seenCommit)
seenCommit := makeTestExtCommit(h, tmtime.Now())
bs.SaveBlockWithExtendedCommit(block, partSet, seenCommit)
}
pruned, err := bs.PruneBlocks(4)
@@ -338,13 +392,19 @@ func TestLoadBaseMeta(t *testing.T) {
}
func TestLoadBlockPart(t *testing.T) {
bs, db := freshBlockStore()
height, index := int64(10), 1
cfg, err := config.ResetTestRoot(t.TempDir(), "blockchain_reactor_test")
require.NoError(t, err)
bs, db := newInMemoryBlockStore()
const height, index = 10, 1
loadPart := func() (interface{}, error) {
part := bs.LoadBlockPart(height, index)
return part, nil
}
state, err := sm.MakeGenesisStateFromFile(cfg.GenesisFile())
require.NoError(t, err)
// Initially no contents.
// 1. Requesting for a non-existent block shouldn't fail
res, _, panicErr := doFn(loadPart)
@@ -352,13 +412,18 @@ func TestLoadBlockPart(t *testing.T) {
require.Nil(t, res, "a non-existent block part should return nil")
// 2. Next save a corrupted block then try to load it
err := db.Set(blockPartKey(height, index), []byte("Tendermint"))
err = db.Set(blockPartKey(height, index), []byte("Tendermint"))
require.NoError(t, err)
res, _, panicErr = doFn(loadPart)
require.NotNil(t, panicErr, "expecting a non-nil panic")
require.Contains(t, panicErr.Error(), "unmarshal to tmproto.Part failed")
// 3. A good block serialized and saved to the DB should be retrievable
block := factory.MakeBlock(state, height, new(types.Commit))
partSet, err := block.MakePartSet(2)
require.NoError(t, err)
part1 := partSet.GetPart(0)
pb1, err := part1.ToProto()
require.NoError(t, err)
err = db.Set(blockPartKey(height, index), mustEncode(pb1))
@@ -391,8 +456,8 @@ func TestPruneBlocks(t *testing.T) {
block := factory.MakeBlock(state, h, new(types.Commit))
partSet, err := block.MakePartSet(2)
require.NoError(t, err)
seenCommit := makeTestCommit(h, tmtime.Now())
bs.SaveBlock(block, partSet, seenCommit)
seenCommit := makeTestExtCommit(h, tmtime.Now())
bs.SaveBlockWithExtendedCommit(block, partSet, seenCommit)
}
assert.EqualValues(t, 1, bs.Base())
@@ -452,7 +517,7 @@ func TestPruneBlocks(t *testing.T) {
}
func TestLoadBlockMeta(t *testing.T) {
bs, db := freshBlockStore()
bs, db := newInMemoryBlockStore()
height := int64(10)
loadMeta := func() (interface{}, error) {
meta := bs.LoadBlockMeta(height)
@@ -499,8 +564,8 @@ func TestBlockFetchAtHeight(t *testing.T) {
partSet, err := block.MakePartSet(2)
require.NoError(t, err)
seenCommit := makeTestCommit(10, tmtime.Now())
bs.SaveBlock(block, partSet, seenCommit)
seenCommit := makeTestExtCommit(block.Header.Height, tmtime.Now())
bs.SaveBlockWithExtendedCommit(block, partSet, seenCommit)
require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed")
blockAtHeight := bs.LoadBlock(bs.Height())
@@ -521,9 +586,12 @@ func TestBlockFetchAtHeight(t *testing.T) {
}
func TestSeenAndCanonicalCommit(t *testing.T) {
bs, _ := freshBlockStore()
state, store, cleanup, err := makeStateAndBlockStore(t.TempDir())
defer cleanup()
require.NoError(t, err)
loadCommit := func() (interface{}, error) {
meta := bs.LoadSeenCommit()
meta := store.LoadSeenCommit()
return meta, nil
}
@@ -536,19 +604,19 @@ func TestSeenAndCanonicalCommit(t *testing.T) {
// produce a few blocks and check that the correct seen and cannoncial commits
// are persisted.
for h := int64(3); h <= 5; h++ {
blockCommit := makeTestCommit(h-1, tmtime.Now())
blockCommit := makeTestExtCommit(h-1, tmtime.Now()).ToCommit()
block := factory.MakeBlock(state, h, blockCommit)
partSet, err := block.MakePartSet(2)
require.NoError(t, err)
seenCommit := makeTestCommit(h, tmtime.Now())
bs.SaveBlock(block, partSet, seenCommit)
c3 := bs.LoadSeenCommit()
seenCommit := makeTestExtCommit(h, tmtime.Now())
store.SaveBlockWithExtendedCommit(block, partSet, seenCommit)
c3 := store.LoadSeenCommit()
require.NotNil(t, c3)
require.Equal(t, h, c3.Height)
require.Equal(t, seenCommit.Hash(), c3.Hash())
c5 := bs.LoadBlockCommit(h)
require.Equal(t, seenCommit.ToCommit().Hash(), c3.Hash())
c5 := store.LoadBlockCommit(h)
require.Nil(t, c5)
c6 := bs.LoadBlockCommit(h - 1)
c6 := store.LoadBlockCommit(h - 1)
require.Equal(t, blockCommit.Hash(), c6.Hash())
}
internal/test/factory/commit.go
+2 -2
View File
@@ -8,7 +8,7 @@ import (
"github.com/tendermint/tendermint/types"
)
func MakeCommit(ctx context.Context, blockID types.BlockID, height int64, round int32, voteSet *types.VoteSet, validators []types.PrivValidator, now time.Time) (*types.Commit, error) {
func MakeExtendedCommit(ctx context.Context, blockID types.BlockID, height int64, round int32, voteSet *types.VoteSet, validators []types.PrivValidator, now time.Time) (*types.ExtendedCommit, error) {
// all sign
for i := 0; i < len(validators); i++ {
pubKey, err := validators[i].GetPubKey(ctx)
@@ -37,5 +37,5 @@ func MakeCommit(ctx context.Context, blockID types.BlockID, height int64, round
}
}
return voteSet.MakeCommit(), nil
return voteSet.MakeExtendedCommit(), nil
}
internal/test/factory/params.go
+1
View File
@@ -18,5 +18,6 @@ func ConsensusParams() *types.ConsensusParams {
VoteDelta: 1 * time.Millisecond,
BypassCommitTimeout: true,
}
c.ABCI.VoteExtensionsEnableHeight = 1
return c
}
light/helpers_test.go
+6 -1
View File
@@ -72,7 +72,12 @@ func (pkz privKeys) signHeader(t testing.TB, header *types.Header, valSet *types
commitSigs[vote.ValidatorIndex] = vote.CommitSig()
}
return types.NewCommit(header.Height, 1, blockID, commitSigs)
return &types.Commit{
Height: header.Height,
Round: 1,
BlockID: blockID,
Signatures: commitSigs,
}
}
func makeVote(t testing.TB, header *types.Header, valset *types.ValidatorSet, key crypto.PrivKey, blockID types.BlockID) *types.Vote {
light/provider/http/http.go
+29 -27
View File
@@ -173,8 +173,7 @@ func (p *http) validatorSet(ctx context.Context, height *int64) (*types.Validato
attempt := uint16(0)
for {
res, err := p.client.Validators(ctx, height, &page, &perPage)
switch e := err.(type) {
case nil: // success!! Now we validate the response
if err == nil {
if len(res.Validators) == 0 {
return nil, provider.ErrBadLightBlock{
Reason: fmt.Errorf("validator set is empty (height: %d, page: %d, per_page: %d)",
@@ -187,35 +186,37 @@ func (p *http) validatorSet(ctx context.Context, height *int64) (*types.Validato
res.Total, height, page, perPage),
}
}
} else {
switch e := err.(type) {
case *url.Error:
if e.Timeout() {
// if we have exceeded retry attempts then return a no response error
if attempt == p.maxRetryAttempts {
return nil, p.noResponse()
case *url.Error:
if e.Timeout() {
// if we have exceeded retry attempts then return a no response error
if attempt == p.maxRetryAttempts {
return nil, p.noResponse()
}
attempt++
// request timed out: we wait and try again with exponential backoff
time.Sleep(backoffTimeout(attempt))
continue
}
attempt++
// request timed out: we wait and try again with exponential backoff
time.Sleep(backoffTimeout(attempt))
continue
return nil, provider.ErrBadLightBlock{Reason: e}
case *rpctypes.RPCError:
// process the rpc error and return the corresponding error to the light client
return nil, p.parseRPCError(e)
default:
// check if the error stems from the context
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return nil, err
}
// If we don't know the error then by default we return an unreliable provider error and
// terminate the connection with the peer.
return nil, provider.ErrUnreliableProvider{Reason: e}
}
return nil, provider.ErrBadLightBlock{Reason: e}
case *rpctypes.RPCError:
// process the rpc error and return the corresponding error to the light client
return nil, p.parseRPCError(e)
default:
// check if the error stems from the context
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return nil, err
}
// If we don't know the error then by default we return an unreliable provider error and
// terminate the connection with the peer.
return nil, provider.ErrUnreliableProvider{Reason: e}
}
// update the total and increment the page index so we can fetch the
// next page of validators if need be
total = res.Total
@@ -223,6 +224,7 @@ func (p *http) validatorSet(ctx context.Context, height *int64) (*types.Validato
page++
break
}
}
valSet, err := types.ValidatorSetFromExistingValidators(vals)
light/verifier.go
+1 -2
View File
@@ -279,8 +279,7 @@ func checkRequiredHeaderFields(h *types.SignedHeader) error {
return errors.New("height in trusted header must be set (non zero")
}
zeroTime := time.Time{}
if h.Time == zeroTime {
if h.Time.IsZero() {
return errors.New("time in trusted header must be set")
}
node/node.go
+1 -1
View File
@@ -266,7 +266,7 @@ func makeNode(
node.evPool = evPool
mpReactor, mp := createMempoolReactor(logger, cfg, proxyApp, stateStore, nodeMetrics.mempool,
peerManager.Subscribe, node.router.OpenChannel, peerManager.GetHeight)
peerManager.Subscribe, node.router.OpenChannel)
node.rpcEnv.Mempool = mp
node.services = append(node.services, mpReactor)
node/node_test.go
+29 -25
View File
@@ -35,6 +35,7 @@ import (
"github.com/tendermint/tendermint/libs/service"
tmtime "github.com/tendermint/tendermint/libs/time"
"github.com/tendermint/tendermint/privval"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
"github.com/tendermint/tendermint/types"
)
@@ -339,13 +340,13 @@ func TestCreateProposalBlock(t *testing.T) {
sm.NopMetrics(),
)
commit := types.NewCommit(height-1, 0, types.BlockID{}, nil)
extCommit := &types.ExtendedCommit{Height: height - 1}
block, err := blockExec.CreateProposalBlock(
ctx,
height,
state, commit,
state,
extCommit,
proposerAddr,
nil,
)
require.NoError(t, err)
@@ -419,13 +420,13 @@ func TestMaxTxsProposalBlockSize(t *testing.T) {
sm.NopMetrics(),
)
commit := types.NewCommit(height-1, 0, types.BlockID{}, nil)
extCommit := &types.ExtendedCommit{Height: height - 1}
block, err := blockExec.CreateProposalBlock(
ctx,
height,
state, commit,
state,
extCommit,
proposerAddr,
nil,
)
require.NoError(t, err)
@@ -525,38 +526,41 @@ func TestMaxProposalBlockSize(t *testing.T) {
}
state.ChainID = maxChainID
cs := types.CommitSig{
BlockIDFlag: types.BlockIDFlagNil,
ValidatorAddress: crypto.AddressHash([]byte("validator_address")),
Timestamp: timestamp,
Signature: crypto.CRandBytes(types.MaxSignatureSize),
}
commit := &types.Commit{
Height: math.MaxInt64,
Round: math.MaxInt32,
BlockID: blockID,
}
votes := make([]*types.Vote, types.MaxVotesCount)
voteSet := types.NewExtendedVoteSet(state.ChainID, math.MaxInt64-1, math.MaxInt32, tmproto.PrecommitType, state.Validators)
// add maximum amount of signatures to a single commit
for i := 0; i < types.MaxVotesCount; i++ {
pubKey, err := privVals[i].GetPubKey(ctx)
require.NoError(t, err)
votes[i] = &types.Vote{
ValidatorAddress: pubKey.Address(),
valIdx, val := state.Validators.GetByAddress(pubKey.Address())
require.NotNil(t, val)
vote := &types.Vote{
Type: tmproto.PrecommitType,
Height: math.MaxInt64 - 1,
Round: math.MaxInt32,
BlockID: blockID,
Timestamp: timestamp,
ValidatorAddress: val.Address,
ValidatorIndex: valIdx,
Extension: []byte("extension"),
}
commit.Signatures = append(commit.Signatures, cs)
vpb := vote.ToProto()
require.NoError(t, privVals[i].SignVote(ctx, state.ChainID, vpb))
vote.Signature = vpb.Signature
vote.ExtensionSignature = vpb.ExtensionSignature
added, err := voteSet.AddVote(vote)
require.NoError(t, err)
require.True(t, added)
}
block, err := blockExec.CreateProposalBlock(
ctx,
math.MaxInt64,
state,
commit,
voteSet.MakeExtendedCommit(),
proposerAddr,
votes,
)
require.NoError(t, err)
partSet, err := block.MakePartSet(types.BlockPartSizeBytes)
node/setup.go
-2
View File
@@ -147,7 +147,6 @@ func createMempoolReactor(
memplMetrics *mempool.Metrics,
peerEvents p2p.PeerEventSubscriber,
chCreator p2p.ChannelCreator,
peerHeight func(types.NodeID) int64,
) (service.Service, mempool.Mempool) {
logger = logger.With("module", "mempool")
@@ -166,7 +165,6 @@ func createMempoolReactor(
mp,
chCreator,
peerEvents,
peerHeight,
)
if cfg.Consensus.WaitForTxs() {
privval/file.go
+4 -4
View File
@@ -375,17 +375,17 @@ func (pv *FilePV) signVote(chainID string, vote *tmproto.Vote) error {
// Vote extensions are non-deterministic, so it is possible that an
// application may have created a different extension. We therefore always
// re-sign the vote extensions of precommits. For prevotes, the extension
// signature will always be empty.
// re-sign the vote extensions of precommits. For prevotes and nil
// precommits, the extension signature will always be empty.
var extSig []byte
if vote.Type == tmproto.PrecommitType {
if vote.Type == tmproto.PrecommitType && !types.ProtoBlockIDIsNil(&vote.BlockID) {
extSignBytes := types.VoteExtensionSignBytes(chainID, vote)
extSig, err = pv.Key.PrivKey.Sign(extSignBytes)
if err != nil {
return err
}
} else if len(vote.Extension) > 0 {
return errors.New("unexpected vote extension - extensions are only allowed in precommits")
return errors.New("unexpected vote extension - extensions are only allowed in non-nil precommits")
}
// We might crash before writing to the wal,

Some files were not shown because too many files have changed in this diff Show More