consensus: add additional metrics for abci++ data (#8480)

This pull request adds an additional set of metrics targeted at providing more visibility into `abci++`.

The following set of metrics are added and exposed through the `metrics` endpoint:

```
tendermint_consensus_proposal_receive_count{chain_id="test-chain-IrF74Y",status="accepted"} 34
tendermint_consensus_proposal_create_count{chain_id="test-chain-IrF74Y"} 34
tendermint_consensus_vote_extension_receive_count{chain_id="test-chain-IrF74Y",status="accepted"} 34
tendermint_consensus_round_voting_power_percent{chain_id="test-chain-IrF74Y",vote_type="precommit"} 1
tendermint_consensus_round_voting_power_percent{chain_id="test-chain-IrF74Y",vote_type="prevote"} 1
tendermint_state_consensus_param_updates{chain_id="test-chain-IrF74Y"} 0
tendermint_state_validator_set_updates{chain_id="test-chain-IrF74Y"} 0
tendermint_consensus_late_votes{chain_id="test-chain-IrF74Y",vote_type="precommit"} 16
```

This pull request also updates the `metrics.md` file to include some metrics that were previously missed. My hope is to generate the `metrics.md` file with a future version of the tool being architected in #8479

consensus/metrics.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/go-kit/kit/metrics"
 	cstypes "github.com/tendermint/tendermint/consensus/types"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -92,6 +93,26 @@ type Metrics struct {
 	// of the voting power on the network issued prevotes.
 	//metrics:Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted.
 	FullPrevoteDelay metrics.Gauge `metrics_labels:"proposer_address"`
+
+	// ProposalReceiveCount is the total number of proposals received by this node
+	// since process start.
+	// The metric is annotated by the status of the proposal from the application,
+	// either 'accepted' or 'rejected'.
+	ProposalReceiveCount metrics.Counter `metrics_labels:"status"`
+
+	// ProposalCreationCount is the total number of proposals created by this node
+	// since process start.
+	ProposalCreateCount metrics.Counter
+
+	// RoundVotingPowerPercent is the percentage of the total voting power received
+	// with a round. The value begins at 0 for each round and approaches 1.0 as
+	// additional voting power is observed. The metric is labeled by vote type.
+	RoundVotingPowerPercent metrics.Gauge
+
+	// LateVotes stores the number of votes that were received by this node that
+	// correspond to earlier heights and rounds than this node is currently
+	// in.
+	LateVotes metrics.Counter
 }
 
 // RecordConsMetrics uses for recording the block related metrics during fast-sync.
@@ -102,6 +123,12 @@ func (m *Metrics) RecordConsMetrics(block *types.Block) {
 	m.CommittedHeight.Set(float64(block.Height))
 }
 
+func (m *Metrics) MarkVoteReceived(vt tmproto.SignedMsgType, power, totalPower int64) {
+	p := float64(power) / float64(totalPower)
+	n := strings.ToLower(strings.TrimPrefix(vt.String(), "SIGNED_MSG_TYPE_"))
+	m.RoundVotingPowerPercent.With("vote_type", n).Add(p)
+}
+
 func (m *Metrics) MarkRound(r int32, st time.Time) {
 	m.Rounds.Set(float64(r))
 	roundTime := time.Since(st).Seconds()

consensus/state.go

@@ -1144,6 +1144,7 @@ func (cs *State) defaultDecideProposal(height int64, round int32) {
 		} else if block == nil {
 			panic("Method createProposalBlock should not provide a nil block without errors")
 		}
+		cs.metrics.ProposalCreateCount.Add(1)
 		blockParts, err = block.MakePartSet(types.BlockPartSizeBytes)
 		if err != nil {
 			cs.Logger.Error("unable to create proposal block part set", "error", err)
@@ -1308,6 +1309,7 @@ func (cs *State) defaultDoPrevote(height int64, round int32) {
 			"state machine returned an error (%v) when calling ProcessProposal", err,
 		))
 	}
+	cs.metrics.MarkProposalProcessed(isAppValid)
 
 	// Vote nil if the Application rejected the block
 	if !isAppValid {
@@ -2051,6 +2053,10 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.ID) (added bool, err error
 		"cs_height", cs.Height,
 	)
 
+	if vote.Height < cs.Height || (vote.Height == cs.Height && vote.Round < cs.Round) {
+		cs.metrics.MarkLateVote(vote.Type)
+	}
+
 	// A precommit for the previous height?
 	// These come in while we wait timeoutCommit
 	if vote.Height+1 == cs.Height && vote.Type == tmproto.PrecommitType {
@@ -2095,6 +2101,11 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.ID) (added bool, err error
 		// Either duplicate, or error upon cs.Votes.AddByIndex()
 		return
 	}
+	if vote.Round == cs.Round {
+		vals := cs.state.Validators
+		_, val := vals.GetByIndex(vote.ValidatorIndex)
+		cs.metrics.MarkVoteReceived(vote.Type, val.VotingPower, vals.TotalVotingPower())
+	}
 
 	if err := cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote}); err != nil {
 		return added, err

docs/tendermint-core/metrics.md

@@ -43,6 +43,12 @@ The following metrics are available:
 | consensus_step_duration                | Histogram | step            | Histogram of durations for each step in the consensus protocol                                                                             |
 | consensus_round_duration               | Histogram |                 | Histogram of durations for all the rounds that have occurred since the process started                                                     |
 | consensus_block_gossip_parts_received  | Counter   | matches_current | Number of block parts received by the node                                                                                                 |
+| consensus_quorum_prevote_delay         | Gauge     |                 | Interval in seconds between the proposal timestamp and the timestamp of the earliest prevote that achieved a quorum                        |
+| consensus_full_prevote_delay           | Gauge     |                 | Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted           |
+| consensus_proposal_receive_count       | Counter   | status          | Total number of proposals received by the node since process start                                                                         |
+| consensus_proposal_create_count        | Counter   |                 | Total number of proposals created by the node since process start                                                                          |
+| consensus_round_voting_power_percent   | Gauge     | vote_type       | A value between 0 and 1.0 representing the percentage of the total voting power per vote type received within a round                      |
+| consensus_late_votes                   | Counter   | vote_type       | Number of votes received by the node since process start that correspond to earlier heights and rounds than this node is currently in.     |
 | p2p_peers                              | Gauge     |                 | Number of peers node's connected to                                                                                                        |
 | p2p_peer_receive_bytes_total           | counter   | peer_id, chID   | number of bytes per channel received from a given peer                                                                                     |
 | p2p_peer_send_bytes_total              | counter   | peer_id, chID   | number of bytes per channel sent to a given peer                                                                                           |

state/execution.go

@@ -233,6 +233,10 @@ func (blockExec *BlockExecutor) ApplyBlock(
 	}
 	if len(validatorUpdates) > 0 {
 		blockExec.logger.Debug("updates to validators", "updates", types.ValidatorListString(validatorUpdates))
+		blockExec.metrics.ValidatorSetUpdates.Add(1)
+	}
+	if finalizeBlockResponse.ConsensusParamUpdates != nil {
+		blockExec.metrics.ConsensusParamUpdates.Add(1)
 	}
 
 	// Update the state with the block and responses.