From a58eec6623115dc313efb348fb4c9a25f864fd83 Mon Sep 17 00:00:00 2001
From: William Banfield <wbanfield@gmail.com>
Date: Wed, 23 Nov 2022 11:30:14 -0500
Subject: [PATCH] add message receive and send metrics

---
 consensus/reactor.go |  5 -----
 p2p/metrics.go       | 14 ++++++++++++++
 p2p/peer.go          |  3 +++
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/consensus/reactor.go b/consensus/reactor.go
index c09543382..bf263f65a 100644
--- a/consensus/reactor.go
+++ b/consensus/reactor.go
@@ -755,7 +755,6 @@ OUTER_LOOP:
 		// If peer is lagging by height 1, send LastCommit.
 		if prs.Height != 0 && rs.Height == prs.Height+1 {
 			if ps.PickSendVote(rs.LastCommit) {
-				conR.Metrics.VoteSent.With("type", "precommit").Add(1)
 				logger.Debug("Picked rs.LastCommit to send", "height", prs.Height)
 				continue OUTER_LOOP
 			}
@@ -769,7 +768,6 @@ OUTER_LOOP:
 			// which contains precommit signatures for prs.Height.
 			if commit := conR.conS.blockStore.LoadBlockCommit(prs.Height); commit != nil {
 				if ps.PickSendVote(commit) {
-					conR.Metrics.VoteSent.With("type", "precommit").Add(1)
 					logger.Debug("Picked Catchup commit to send", "height", prs.Height)
 					continue OUTER_LOOP
 				}
@@ -802,7 +800,6 @@ func (conR *Reactor) gossipVotesForHeight(
 	// If there are lastCommits to send...
 	if prs.Step == cstypes.RoundStepNewHeight {
 		if ps.PickSendVote(rs.LastCommit) {
-			conR.Metrics.VoteSent.With("type", "precommit").Add(1)
 			logger.Debug("Picked rs.LastCommit to send")
 			return true
 		}
@@ -822,7 +819,6 @@ func (conR *Reactor) gossipVotesForHeight(
 	// If there are prevotes to send...
 	if prs.Step <= cstypes.RoundStepPrevoteWait && prs.Round != -1 && prs.Round <= rs.Round {
 		if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) {
-			conR.Metrics.VoteSent.With("type", "prevote").Add(1)
 			logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round)
 			return true
 		}
@@ -830,7 +826,6 @@ func (conR *Reactor) gossipVotesForHeight(
 	// If there are precommits to send...
 	if prs.Step <= cstypes.RoundStepPrecommitWait && prs.Round != -1 && prs.Round <= rs.Round {
 		if ps.PickSendVote(rs.Votes.Precommits(prs.Round)) {
-			conR.Metrics.VoteSent.With("type", "precommit").Add(1)
 			logger.Debug("Picked rs.Precommits(prs.Round) to send", "round", prs.Round)
 			return true
 		}
diff --git a/p2p/metrics.go b/p2p/metrics.go
index 7c80658e5..d97b630d7 100644
--- a/p2p/metrics.go
+++ b/p2p/metrics.go
@@ -41,6 +41,8 @@ type Metrics struct {
 	MessageReceiveBytesTotal metrics.Counter
 	// Number of bytes of each message type sent.
 	MessageSendBytesTotal metrics.Counter
+	MessageSend           metrics.Counter
+	MessageReceive        metrics.Counter
 }
 
 // PrometheusMetrics returns Metrics build using Prometheus client library.
@@ -94,6 +96,18 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
 			Name:      "message_send_bytes_total",
 			Help:      "Number of bytes of each message type sent.",
 		}, append(labels, "message_type")).With(labelsAndValues...),
+		MessageSend: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "message_send",
+			Help:      "Number of messages of each type sent.",
+		}, append(labels, "message_type")).With(labelsAndValues...),
+		MessageReceive: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "message_receive",
+			Help:      "Number of messages of each type received.",
+		}, append(labels, "message_type")).With(labelsAndValues...),
 	}
 }
 
diff --git a/p2p/peer.go b/p2p/peer.go
index 36b58f0e4..d88268138 100644
--- a/p2p/peer.go
+++ b/p2p/peer.go
@@ -333,6 +333,7 @@ func (p *peer) SendEnvelope(e Envelope) bool {
 	res := p.Send(e.ChannelID, msgBytes)
 	if res {
 		p.metrics.MessageSendBytesTotal.With("message_type", metricLabelValue).Add(float64(len(msgBytes)))
+		p.metrics.MessageSend.With("message_type", metricLabelValue).Add(1)
 	}
 	return res
 }
@@ -382,6 +383,7 @@ func (p *peer) TrySendEnvelope(e Envelope) bool {
 	res := p.TrySend(e.ChannelID, msgBytes)
 	if res {
 		p.metrics.MessageSendBytesTotal.With("message_type", metricLabelValue).Add(float64(len(msgBytes)))
+		p.metrics.MessageSend.With("message_type", metricLabelValue).Add(1)
 	}
 	return res
 }
@@ -534,6 +536,7 @@ func createMConnection(
 		}
 		p.metrics.PeerReceiveBytesTotal.With(labels...).Add(float64(len(msgBytes)))
 		p.metrics.MessageReceiveBytesTotal.With("message_type", p.mlc.ValueToMetricLabel(msg)).Add(float64(len(msgBytes)))
+		p.metrics.MessageReceive.With("message_type", p.mlc.ValueToMetricLabel(msg)).Add(1)
 		if nr, ok := reactor.(EnvelopeReceiver); ok {
 			nr.ReceiveEnvelope(Envelope{
 				ChannelID: chID,