From d4f26f7d61c3930550d732c50466e1f721ee4aa9 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 20:21:12 -0500 Subject: [PATCH 1/3] consensus: calculate prevote message delay metric (backport #7551) (#7618) * consensus: calculate prevote message delay metric (#7551) ## What does this pull request do? This pull requests adds two metrics intended for use in calculating an experimental value for `MessageDelay`. The metrics are as follows: ``` # HELP tendermint_consensus_complete_prevote_message_delay Difference in seconds between the proposal timestamp and the timestamp of the prevote that achieved 100% of the voting power in the prevote step. # TYPE tendermint_consensus_complete_prevote_message_delay gauge tendermint_consensus_complete_prevote_message_delay{chain_id="test-chain-aZbwF1"} 0.013025505 # HELP tendermint_consensus_quorum_prevote_message_delay Difference in seconds between the proposal timestamp and the timestamp of the prevote that achieved a quorum in the prevote step. # TYPE tendermint_consensus_quorum_prevote_message_delay gauge tendermint_consensus_quorum_prevote_message_delay{chain_id="test-chain-aZbwF1"} 0.013025505 ``` ## Why this change? For more information on what these metrics are calculating, see #7202. The aim is to merge to backport these metrics to v0.34 and run nodes on a few popular chains with these metrics to determine the experimental values for `MessageDelay` on these popular chains and use these to select our default `SynchronyParams.MessageDelay` value. ## Why Gauges for the metrics? Gauges allow us to overwrite the metric on each successive observation. We can then capture these metrics over time to track the highest and lowest observed value. (cherry picked from commit 0c82ceaa5f7964c13247af9b64d72477af9dc973) # Conflicts: # internal/consensus/state.go * fix merge conflicts Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com> Co-authored-by: William Banfield --- internal/consensus/metrics.go | 46 +++++++++++++++++++++++++++++------ internal/consensus/state.go | 23 ++++++++++++++++++ types/vote_set.go | 17 +++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/internal/consensus/metrics.go b/internal/consensus/metrics.go index a75f1505c..dbefb9a5b 100644 --- a/internal/consensus/metrics.go +++ b/internal/consensus/metrics.go @@ -64,6 +64,22 @@ type Metrics struct { // Histogram of time taken per step annotated with reason that the step proceeded. StepTime metrics.Histogram + + // QuroumPrevoteMessageDelay is the interval in seconds between the proposal + // timestamp and the timestamp of the earliest prevote that achieved a quorum + // during the prevote step. + // + // To compute it, sum the voting power over each prevote received, in increasing + // order of timestamp. The timestamp of the first prevote to increase the sum to + // be above 2/3 of the total voting power of the network defines the endpoint + // the endpoint of the interval. Subtract the proposal timestamp from this endpoint + // to obtain the quorum delay. + QuorumPrevoteMessageDelay metrics.Gauge + + // FullPrevoteMessageDelay is the interval in seconds between the proposal + // timestamp and the timestamp of the latest prevote in a round where 100% + // of the voting power on the network issued prevotes. + FullPrevoteMessageDelay metrics.Gauge } // PrometheusMetrics returns Metrics build using Prometheus client library. @@ -196,6 +212,20 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Name: "step_time", Help: "Time spent per step.", }, append(labels, "step", "reason")).With(labelsAndValues...), + QuorumPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "quorum_prevote_message_delay", + Help: "Difference in seconds between the proposal timestamp and the timestamp " + + "of the latest prevote that achieved a quorum in the prevote step.", + }, labels).With(labelsAndValues...), + FullPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: MetricsSubsystem, + Name: "full_prevote_message_delay", + Help: "Difference in seconds between the proposal timestamp and the timestamp " + + "of the latest prevote that achieved 100% of the voting power in the prevote step.", + }, labels).With(labelsAndValues...), } } @@ -219,13 +249,15 @@ func NopMetrics() *Metrics { BlockIntervalSeconds: discard.NewHistogram(), - NumTxs: discard.NewGauge(), - BlockSizeBytes: discard.NewHistogram(), - TotalTxs: discard.NewGauge(), - CommittedHeight: discard.NewGauge(), - BlockSyncing: discard.NewGauge(), - StateSyncing: discard.NewGauge(), - BlockParts: discard.NewCounter(), + NumTxs: discard.NewGauge(), + BlockSizeBytes: discard.NewHistogram(), + TotalTxs: discard.NewGauge(), + CommittedHeight: discard.NewGauge(), + BlockSyncing: discard.NewGauge(), + StateSyncing: discard.NewGauge(), + BlockParts: discard.NewCounter(), + QuorumPrevoteMessageDelay: discard.NewGauge(), + FullPrevoteMessageDelay: discard.NewGauge(), } } diff --git a/internal/consensus/state.go b/internal/consensus/state.go index 837174356..f87455bb9 100644 --- a/internal/consensus/state.go +++ b/internal/consensus/state.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "runtime/debug" + "sort" "time" "github.com/gogo/protobuf/proto" @@ -1623,6 +1624,8 @@ func (cs *State) finalizeCommit(height int64) { return } + cs.calculatePrevoteMessageDelayMetrics() + blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts @@ -2347,6 +2350,26 @@ func (cs *State) checkDoubleSigningRisk(height int64) error { return nil } +func (cs *State) calculatePrevoteMessageDelayMetrics() { + ps := cs.Votes.Prevotes(cs.Round) + pl := ps.List() + sort.Slice(pl, func(i, j int) bool { + return pl[i].Timestamp.Before(pl[j].Timestamp) + }) + var votingPowerSeen int64 + for _, v := range pl { + _, val := cs.Validators.GetByAddress(v.ValidatorAddress) + votingPowerSeen += val.VotingPower + if votingPowerSeen >= cs.Validators.TotalVotingPower()*2/3+1 { + cs.metrics.QuorumPrevoteMessageDelay.Set(v.Timestamp.Sub(cs.Proposal.Timestamp).Seconds()) + break + } + } + if ps.HasAll() { + cs.metrics.FullPrevoteMessageDelay.Set(pl[len(pl)-1].Timestamp.Sub(cs.Proposal.Timestamp).Seconds()) + } +} + //--------------------------------------------------------- func CompareHRS(h1 int64, r1 int32, s1 cstypes.RoundStepType, h2 int64, r2 int32, s2 cstypes.RoundStepType) int { diff --git a/types/vote_set.go b/types/vote_set.go index e014ae7bb..259857e40 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -378,6 +378,20 @@ func (voteSet *VoteSet) GetByIndex(valIndex int32) *Vote { return voteSet.votes[valIndex] } +// List returns a copy of the list of votes stored by the VoteSet. +func (voteSet *VoteSet) List() []Vote { + if voteSet == nil || voteSet.votes == nil { + return nil + } + votes := make([]Vote, 0, len(voteSet.votes)) + for i := range voteSet.votes { + if voteSet.votes[i] != nil { + votes = append(votes, *voteSet.votes[i]) + } + } + return votes +} + func (voteSet *VoteSet) GetByAddress(address []byte) *Vote { if voteSet == nil { return nil @@ -423,6 +437,9 @@ func (voteSet *VoteSet) HasTwoThirdsAny() bool { } func (voteSet *VoteSet) HasAll() bool { + if voteSet == nil { + return false + } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() return voteSet.sum == voteSet.valSet.TotalVotingPower() From b2c4128ac0d6e538dbd1a1d23ed364244c043d33 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 21:36:33 -0800 Subject: [PATCH 2/3] build: Make sure to test packages with external tests (backport #7608) (#7635) * build: Make sure to test packages with external tests (#7608) The test filter was looking for "TestGoFiles", which does not include tests in a separate package (e.g., "package foo_test" for "package foo"). This caused several packages not to be tested in CI, including: github.com/tendermint/tendermint/abci/client github.com/tendermint/tendermint/crypto github.com/tendermint/tendermint/crypto/tmhash github.com/tendermint/tendermint/internal/eventbus github.com/tendermint/tendermint/internal/evidence github.com/tendermint/tendermint/internal/inspect github.com/tendermint/tendermint/internal/jsontypes github.com/tendermint/tendermint/internal/libs/protoio github.com/tendermint/tendermint/internal/libs/sync github.com/tendermint/tendermint/internal/p2p/pex github.com/tendermint/tendermint/internal/pubsub github.com/tendermint/tendermint/internal/pubsub/query github.com/tendermint/tendermint/internal/pubsub/query/syntax github.com/tendermint/tendermint/internal/state/indexer github.com/tendermint/tendermint/internal/state/indexer/block/kv github.com/tendermint/tendermint/libs/json github.com/tendermint/tendermint/libs/log github.com/tendermint/tendermint/libs/os github.com/tendermint/tendermint/light github.com/tendermint/tendermint/light/provider/http github.com/tendermint/tendermint/privval/grpc github.com/tendermint/tendermint/proto/tendermint/blocksync github.com/tendermint/tendermint/proto/tendermint/consensus github.com/tendermint/tendermint/proto/tendermint/statesync github.com/tendermint/tendermint/rpc/client github.com/tendermint/tendermint/rpc/client/mock github.com/tendermint/tendermint/test/e2e/tests github.com/tendermint/tendermint/test/fuzz/mempool github.com/tendermint/tendermint/test/fuzz/p2p/secretconnection github.com/tendermint/tendermint/test/fuzz/rpc/jsonrpc/server Updates #7626 and #7634. (cherry picked from commit aea428d322a557c1618e2a39882c683684c2ebb4) * Fix build tags to satisfy Go 1.16. Co-authored-by: M. J. Fromberger --- Makefile | 6 ++++-- internal/p2p/pex/reactor_test.go | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1176ea844..64a50669c 100644 --- a/Makefile +++ b/Makefile @@ -318,9 +318,11 @@ NUM_SPLIT ?= 4 $(BUILDDIR): mkdir -p $@ -# the format statement filters out all packages that don't have tests. +# The format statement filters out all packages that don't have tests. +# Note we need to check for both in-package tests (.TestGoFiles) and +# out-of-package tests (.XTestGoFiles). $(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR) - go list -f "{{ if .TestGoFiles }}{{ .ImportPath }}{{ end }}" ./... | sort > $@ + go list -f "{{ if (or .TestGoFiles .XTestGoFiles) }}{{ .ImportPath }}{{ end }}" ./... | sort > $@ split-test-packages:$(BUILDDIR)/packages.txt split -d -n l/$(NUM_SPLIT) $< $<. diff --git a/internal/p2p/pex/reactor_test.go b/internal/p2p/pex/reactor_test.go index 63e479e6a..cb1cf117d 100644 --- a/internal/p2p/pex/reactor_test.go +++ b/internal/p2p/pex/reactor_test.go @@ -1,3 +1,7 @@ +// Temporarily disabled pending ttps://github.com/tendermint/tendermint/issues/7626. +//go:build issue7626 +//+build issue7626 + package pex_test import ( From c2f58fa9f9cc4e200f390065cd9fc448713d8bb2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jan 2022 12:03:15 +0100 Subject: [PATCH 3/3] build(deps): Bump github.com/prometheus/client_golang (#7637) --- go.mod | 2 +- go.sum | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 9870e2589..bc29128fc 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/mroth/weightedrand v0.4.1 github.com/oasisprotocol/curve25519-voi v0.0.0-20210609091139-0a56a4bca00b github.com/ory/dockertest v3.3.5+incompatible - github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/client_golang v1.12.0 github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 github.com/rs/cors v1.8.2 github.com/rs/zerolog v1.26.1 diff --git a/go.sum b/go.sum index fd8610f41..03d8e2fc4 100644 --- a/go.sum +++ b/go.sum @@ -837,8 +837,9 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= +github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -850,8 +851,9 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.30.0 h1:JEkYlQnpzrzQFxi6gnukFPdQ+ac82oRhzMcIduJu/Ug= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1340,8 +1342,9 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=