From b92a19b2ce2368f7ea40b03303d71f7a2b978cad Mon Sep 17 00:00:00 2001
From: William Banfield <4561443+williambanfield@users.noreply.github.com>
Date: Fri, 5 Aug 2022 13:29:00 -0400
Subject: [PATCH] proxy: add initial set of abci metrics (port of #7115)
 (#9169)

* internal/proxy: add initial set of abci metrics (#7115)

This PR adds an initial set of metrics for use ABCI. The initial metrics enable the calculation of timing histograms and call counts for each of the ABCI methods. The metrics are also labeled as either 'sync' or 'async' to determine if the method call was performed using ABCI's `*Async` methods.

An example of these metrics is included here for reference:
```
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0001"} 0
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.0004"} 5
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.002"} 12
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.009"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.02"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.1"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="0.65"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="2"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="6"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="25"} 13
tendermint_abci_connection_method_timing_bucket{chain_id="ci",method="commit",type="sync",le="+Inf"} 13
tendermint_abci_connection_method_timing_sum{chain_id="ci",method="commit",type="sync"} 0.007802058000000001
tendermint_abci_connection_method_timing_count{chain_id="ci",method="commit",type="sync"} 13
```

These metrics can easily be graphed using prometheus's `histogram_quantile(...)` method to pick out a particular quantile to graph or examine. I chose buckets that were somewhat of an estimate of expected range of times for ABCI operations. They start at .0001 seconds and range to 25 seconds. The hope is that this range captures enough possible times to be useful for us and operators.

* fixup

* fixups

* docs: add abci timing metrics to the metrics docs (#7311)

* format table
---
 blockchain/reactor_test.go      |  2 +-
 consensus/replay_file.go        |  2 +-
 consensus/replay_stubs.go       |  2 +-
 consensus/replay_test.go        | 12 ++++-----
 consensus/wal_generator.go      |  2 +-
 docs/tendermint-core/metrics.md |  3 ++-
 node/node.go                    | 19 ++++++-------
 node/node_test.go               |  4 +--
 proxy/app_conn.go               | 44 +++++++++++++++++++++++++++---
 proxy/metrics.go                | 47 +++++++++++++++++++++++++++++++++
 proxy/multi_app_conn.go         | 16 ++++++-----
 proxy/multi_app_conn_test.go    |  4 +--
 state/execution_test.go         | 10 +++----
 state/helpers_test.go           |  2 +-
 14 files changed, 128 insertions(+), 41 deletions(-)
 create mode 100644 proxy/metrics.go

diff --git a/blockchain/reactor_test.go b/blockchain/reactor_test.go
index 7625a1bfa..db4cbead2 100644
--- a/blockchain/reactor_test.go
+++ b/blockchain/reactor_test.go
@@ -62,7 +62,7 @@ func newReactor(
 
 	app := &testApp{}
 	cc := proxy.NewLocalClientCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	if err != nil {
 		panic(fmt.Errorf("error start app: %w", err))
diff --git a/consensus/replay_file.go b/consensus/replay_file.go
index 4bf7466ab..9ec5403cc 100644
--- a/consensus/replay_file.go
+++ b/consensus/replay_file.go
@@ -309,7 +309,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 
 	// Create proxyAppConn connection (consensus, mempool, query)
 	clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
-	proxyApp := proxy.NewAppConns(clientCreator)
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	err = proxyApp.Start()
 	if err != nil {
 		tmos.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err))
diff --git a/consensus/replay_stubs.go b/consensus/replay_stubs.go
index 9b4b3b062..aa74f9420 100644
--- a/consensus/replay_stubs.go
+++ b/consensus/replay_stubs.go
@@ -66,7 +66,7 @@ func newMockProxyApp(appHash []byte, abciResponses *tmstate.ABCIResponses) proxy
 	if err != nil {
 		panic(err)
 	}
-	return proxy.NewAppConnConsensus(cli)
+	return proxy.NewAppConnConsensus(cli, proxy.NopMetrics())
 }
 
 type mockProxyApp struct {
diff --git a/consensus/replay_test.go b/consensus/replay_test.go
index a081e262e..c1d4d9127 100644
--- a/consensus/replay_test.go
+++ b/consensus/replay_test.go
@@ -710,7 +710,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin
 	if nBlocks > 0 {
 		// run nBlocks against a new client to build up the app state.
 		// use a throwaway tendermint state
-		proxyApp := proxy.NewAppConns(clientCreator2)
+		proxyApp := proxy.NewAppConns(clientCreator2, proxy.NopMetrics())
 		stateDB1 := dbm.NewMemDB()
 		stateStore := sm.NewStore(stateDB1)
 		err := stateStore.Save(genesisState)
@@ -730,7 +730,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin
 	// now start the app using the handshake - it should sync
 	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
 	handshaker := NewHandshaker(stateStore, state, store, genDoc)
-	proxyApp := proxy.NewAppConns(clientCreator2)
+	proxyApp := proxy.NewAppConns(clientCreator2, proxy.NopMetrics())
 	if err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}
@@ -839,7 +839,7 @@ func buildTMStateFromChain(
 	clientCreator := proxy.NewLocalClientCreator(
 		kvstore.NewPersistentKVStoreApplication(
 			filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_t", nBlocks, mode))))
-	proxyApp := proxy.NewAppConns(clientCreator)
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	if err := proxyApp.Start(); err != nil {
 		panic(err)
 	}
@@ -905,7 +905,7 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
 	{
 		app := &badApp{numBlocks: 3, allHashesAreWrong: true}
 		clientCreator := proxy.NewLocalClientCreator(app)
-		proxyApp := proxy.NewAppConns(clientCreator)
+		proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 		err := proxyApp.Start()
 		require.NoError(t, err)
 		t.Cleanup(func() {
@@ -929,7 +929,7 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
 	{
 		app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true}
 		clientCreator := proxy.NewLocalClientCreator(app)
-		proxyApp := proxy.NewAppConns(clientCreator)
+		proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 		err := proxyApp.Start()
 		require.NoError(t, err)
 		t.Cleanup(func() {
@@ -1232,7 +1232,7 @@ func TestHandshakeUpdatesValidators(t *testing.T) {
 	// now start the app using the handshake - it should sync
 	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
 	handshaker := NewHandshaker(stateStore, state, store, genDoc)
-	proxyApp := proxy.NewAppConns(clientCreator)
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	if err := proxyApp.Start(); err != nil {
 		t.Fatalf("Error starting proxy app connections: %v", err)
 	}
diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go
index 1c449717b..5a4670456 100644
--- a/consensus/wal_generator.go
+++ b/consensus/wal_generator.go
@@ -59,7 +59,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
 
 	blockStore := store.NewBlockStore(blockStoreDB)
 
-	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app))
+	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app), proxy.NopMetrics())
 	proxyApp.SetLogger(logger.With("module", "proxy"))
 	if err := proxyApp.Start(); err != nil {
 		return fmt.Errorf("failed to start proxy app connections: %w", err)
diff --git a/docs/tendermint-core/metrics.md b/docs/tendermint-core/metrics.md
index d6c2bd82d..562c0cb95 100644
--- a/docs/tendermint-core/metrics.md
+++ b/docs/tendermint-core/metrics.md
@@ -19,7 +19,8 @@ Listen address can be changed in the config file (see
 The following metrics are available:
 
 | **Name**                               | **Type**  | **Tags**      | **Description**                                                        |
-| -------------------------------------- | --------- | ------------- | ---------------------------------------------------------------------- |
+|----------------------------------------|-----------|---------------|------------------------------------------------------------------------|
+| abci_connection_method_timing_seconds  | Histogram | method, type  | Timings for each of the ABCI methods                                   |
 | consensus_height                       | Gauge     |               | Height of the chain                                                    |
 | consensus_validators                   | Gauge     |               | Number of validators                                                   |
 | consensus_validators_power             | Gauge     |               | Total voting power of all validators                                   |
diff --git a/node/node.go b/node/node.go
index 46dc6ed70..51fe490d8 100644
--- a/node/node.go
+++ b/node/node.go
@@ -113,19 +113,20 @@ func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) {
 }
 
 // MetricsProvider returns a consensus, p2p and mempool Metrics.
-type MetricsProvider func(chainID string) (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics)
+type MetricsProvider func(chainID string) (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics, *proxy.Metrics)
 
 // DefaultMetricsProvider returns Metrics build using Prometheus client library
 // if Prometheus is enabled. Otherwise, it returns no-op Metrics.
 func DefaultMetricsProvider(config *cfg.InstrumentationConfig) MetricsProvider {
-	return func(chainID string) (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics) {
+	return func(chainID string) (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics, *proxy.Metrics) {
 		if config.Prometheus {
 			return cs.PrometheusMetrics(config.Namespace, "chain_id", chainID),
 				p2p.PrometheusMetrics(config.Namespace, "chain_id", chainID),
 				mempl.PrometheusMetrics(config.Namespace, "chain_id", chainID),
-				sm.PrometheusMetrics(config.Namespace, "chain_id", chainID)
+				sm.PrometheusMetrics(config.Namespace, "chain_id", chainID),
+				proxy.PrometheusMetrics(config.Namespace, "chain_id", chainID)
 		}
-		return cs.NopMetrics(), p2p.NopMetrics(), mempl.NopMetrics(), sm.NopMetrics()
+		return cs.NopMetrics(), p2p.NopMetrics(), mempl.NopMetrics(), sm.NopMetrics(), proxy.NopMetrics()
 	}
 }
 
@@ -246,8 +247,8 @@ func initDBs(config *cfg.Config, dbProvider DBProvider) (blockStore *store.Block
 	return
 }
 
-func createAndStartProxyAppConns(clientCreator proxy.ClientCreator, logger log.Logger) (proxy.AppConns, error) {
-	proxyApp := proxy.NewAppConns(clientCreator)
+func createAndStartProxyAppConns(clientCreator proxy.ClientCreator, logger log.Logger, metrics *proxy.Metrics) (proxy.AppConns, error) {
+	proxyApp := proxy.NewAppConns(clientCreator, metrics)
 	proxyApp.SetLogger(logger.With("module", "proxy"))
 	if err := proxyApp.Start(); err != nil {
 		return nil, fmt.Errorf("error starting proxy app connections: %v", err)
@@ -720,8 +721,10 @@ func NewNode(config *cfg.Config,
 		return nil, err
 	}
 
+	csMetrics, p2pMetrics, memplMetrics, smMetrics, abciMetrics := metricsProvider(genDoc.ChainID)
+
 	// Create the proxyApp and establish connections to the ABCI app (consensus, mempool, query).
-	proxyApp, err := createAndStartProxyAppConns(clientCreator, logger)
+	proxyApp, err := createAndStartProxyAppConns(clientCreator, logger, abciMetrics)
 	if err != nil {
 		return nil, err
 	}
@@ -786,8 +789,6 @@ func NewNode(config *cfg.Config,
 
 	logNodeStartupInfo(state, pubKey, logger, consensusLogger)
 
-	csMetrics, p2pMetrics, memplMetrics, smMetrics := metricsProvider(genDoc.ChainID)
-
 	// Make MempoolReactor
 	mempool, mempoolReactor := createMempoolAndMempoolReactor(config, proxyApp, state, memplMetrics, logger)
 
diff --git a/node/node_test.go b/node/node_test.go
index b62e79fe4..e60897cb0 100644
--- a/node/node_test.go
+++ b/node/node_test.go
@@ -226,7 +226,7 @@ func TestCreateProposalBlock(t *testing.T) {
 	config := cfg.ResetTestRoot("node_create_proposal")
 	defer os.RemoveAll(config.RootDir)
 	cc := proxy.NewLocalClientCreator(kvstore.NewApplication())
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -331,7 +331,7 @@ func TestMaxProposalBlockSize(t *testing.T) {
 	config := cfg.ResetTestRoot("node_create_proposal")
 	defer os.RemoveAll(config.RootDir)
 	cc := proxy.NewLocalClientCreator(kvstore.NewApplication())
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
diff --git a/proxy/app_conn.go b/proxy/app_conn.go
index 690c08df9..f25c33ce0 100644
--- a/proxy/app_conn.go
+++ b/proxy/app_conn.go
@@ -1,6 +1,9 @@
 package proxy
 
 import (
+	"time"
+
+	"github.com/go-kit/kit/metrics"
 	abcicli "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/types"
 )
@@ -56,11 +59,13 @@ type AppConnSnapshot interface {
 // Implements AppConnConsensus (subset of abcicli.Client)
 
 type appConnConsensus struct {
+	metrics *Metrics
 	appConn abcicli.Client
 }
 
-func NewAppConnConsensus(appConn abcicli.Client) AppConnConsensus {
+func NewAppConnConsensus(appConn abcicli.Client, metrics *Metrics) AppConnConsensus {
 	return &appConnConsensus{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -74,22 +79,27 @@ func (app *appConnConsensus) Error() error {
 }
 
 func (app *appConnConsensus) InitChainSync(req types.RequestInitChain) (*types.ResponseInitChain, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "init_chain", "type", "sync"))()
 	return app.appConn.InitChainSync(req)
 }
 
 func (app *appConnConsensus) BeginBlockSync(req types.RequestBeginBlock) (*types.ResponseBeginBlock, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "begin_block", "type", "sync"))()
 	return app.appConn.BeginBlockSync(req)
 }
 
 func (app *appConnConsensus) DeliverTxAsync(req types.RequestDeliverTx) *abcicli.ReqRes {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "deliver_tx", "type", "async"))()
 	return app.appConn.DeliverTxAsync(req)
 }
 
 func (app *appConnConsensus) EndBlockSync(req types.RequestEndBlock) (*types.ResponseEndBlock, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "end_block", "type", "sync"))()
 	return app.appConn.EndBlockSync(req)
 }
 
 func (app *appConnConsensus) CommitSync() (*types.ResponseCommit, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "commit", "type", "sync"))()
 	return app.appConn.CommitSync()
 }
 
@@ -97,11 +107,13 @@ func (app *appConnConsensus) CommitSync() (*types.ResponseCommit, error) {
 // Implements AppConnMempool (subset of abcicli.Client)
 
 type appConnMempool struct {
+	metrics *Metrics
 	appConn abcicli.Client
 }
 
-func NewAppConnMempool(appConn abcicli.Client) AppConnMempool {
+func NewAppConnMempool(appConn abcicli.Client, metrics *Metrics) AppConnMempool {
 	return &appConnMempool{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -115,18 +127,22 @@ func (app *appConnMempool) Error() error {
 }
 
 func (app *appConnMempool) FlushAsync() *abcicli.ReqRes {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "flush", "type", "async"))()
 	return app.appConn.FlushAsync()
 }
 
 func (app *appConnMempool) FlushSync() error {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "flush", "type", "sync"))()
 	return app.appConn.FlushSync()
 }
 
 func (app *appConnMempool) CheckTxAsync(req types.RequestCheckTx) *abcicli.ReqRes {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "check_tx", "type", "async"))()
 	return app.appConn.CheckTxAsync(req)
 }
 
 func (app *appConnMempool) CheckTxSync(req types.RequestCheckTx) (*types.ResponseCheckTx, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "check_tx", "type", "sync"))()
 	return app.appConn.CheckTxSync(req)
 }
 
@@ -134,11 +150,13 @@ func (app *appConnMempool) CheckTxSync(req types.RequestCheckTx) (*types.Respons
 // Implements AppConnQuery (subset of abcicli.Client)
 
 type appConnQuery struct {
+	metrics *Metrics
 	appConn abcicli.Client
 }
 
-func NewAppConnQuery(appConn abcicli.Client) AppConnQuery {
+func NewAppConnQuery(appConn abcicli.Client, metrics *Metrics) AppConnQuery {
 	return &appConnQuery{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -148,14 +166,17 @@ func (app *appConnQuery) Error() error {
 }
 
 func (app *appConnQuery) EchoSync(msg string) (*types.ResponseEcho, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "echo", "type", "sync"))()
 	return app.appConn.EchoSync(msg)
 }
 
 func (app *appConnQuery) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "info", "type", "sync"))()
 	return app.appConn.InfoSync(req)
 }
 
 func (app *appConnQuery) QuerySync(reqQuery types.RequestQuery) (*types.ResponseQuery, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "query", "type", "sync"))()
 	return app.appConn.QuerySync(reqQuery)
 }
 
@@ -163,11 +184,13 @@ func (app *appConnQuery) QuerySync(reqQuery types.RequestQuery) (*types.Response
 // Implements AppConnSnapshot (subset of abcicli.Client)
 
 type appConnSnapshot struct {
+	metrics *Metrics
 	appConn abcicli.Client
 }
 
-func NewAppConnSnapshot(appConn abcicli.Client) AppConnSnapshot {
+func NewAppConnSnapshot(appConn abcicli.Client, metrics *Metrics) AppConnSnapshot {
 	return &appConnSnapshot{
+		metrics: metrics,
 		appConn: appConn,
 	}
 }
@@ -177,19 +200,32 @@ func (app *appConnSnapshot) Error() error {
 }
 
 func (app *appConnSnapshot) ListSnapshotsSync(req types.RequestListSnapshots) (*types.ResponseListSnapshots, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "list_snapshots", "type", "sync"))()
 	return app.appConn.ListSnapshotsSync(req)
 }
 
 func (app *appConnSnapshot) OfferSnapshotSync(req types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "offer_snapshot", "type", "sync"))()
 	return app.appConn.OfferSnapshotSync(req)
 }
 
 func (app *appConnSnapshot) LoadSnapshotChunkSync(
 	req types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "load_snapshot_chunk", "type", "sync"))()
 	return app.appConn.LoadSnapshotChunkSync(req)
 }
 
 func (app *appConnSnapshot) ApplySnapshotChunkSync(
 	req types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "apply_snapshot_chunk", "type", "sync"))()
 	return app.appConn.ApplySnapshotChunkSync(req)
 }
+
+// addTimeSample returns a function that, when called, adds an observation to m.
+// The observation added to m is the number of seconds ellapsed since addTimeSample
+// was initially called. addTimeSample is meant to be called in a defer to calculate
+// the amount of time a function takes to complete.
+func addTimeSample(m metrics.Histogram) func() {
+	start := time.Now()
+	return func() { m.Observe(time.Since(start).Seconds()) }
+}
diff --git a/proxy/metrics.go b/proxy/metrics.go
new file mode 100644
index 000000000..988c332a6
--- /dev/null
+++ b/proxy/metrics.go
@@ -0,0 +1,47 @@
+package proxy
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/discard"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
+	// package.
+	MetricsSubsystem = "abci_connection"
+)
+
+// Metrics contains the prometheus metrics exposed by the proxy package.
+type Metrics struct {
+	MethodTiming metrics.Histogram
+}
+
+// PrometheusMetrics constructs a Metrics instance that collects metrics samples.
+// The resulting metrics will be prefixed with namespace and labeled with the
+// defaultLabelsAndValues. defaultLabelsAndValues must be a list of string pairs
+// where the first of each pair is the label and the second is the value.
+func PrometheusMetrics(namespace string, defaultLabelsAndValues ...string) *Metrics {
+	defaultLabels := []string{}
+	for i := 0; i < len(defaultLabelsAndValues); i += 2 {
+		defaultLabels = append(defaultLabels, defaultLabelsAndValues[i])
+	}
+	return &Metrics{
+		MethodTiming: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "method_timing_seconds",
+			Help:      "ABCI Method Timing",
+			Buckets:   []float64{.0001, .0004, .002, .009, .02, .1, .65, 2, 6, 25},
+		}, append(defaultLabels, []string{"method", "type"}...)).With(defaultLabelsAndValues...),
+	}
+}
+
+// NopMetrics constructs a Metrics instance that discards all samples and is suitable
+// for testing.
+func NopMetrics() *Metrics {
+	return &Metrics{
+		MethodTiming: discard.NewHistogram(),
+	}
+}
diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go
index a7a6f7014..37a5c166e 100644
--- a/proxy/multi_app_conn.go
+++ b/proxy/multi_app_conn.go
@@ -32,8 +32,8 @@ type AppConns interface {
 }
 
 // NewAppConns calls NewMultiAppConn.
-func NewAppConns(clientCreator ClientCreator) AppConns {
-	return NewMultiAppConn(clientCreator)
+func NewAppConns(clientCreator ClientCreator, metrics *Metrics) AppConns {
+	return NewMultiAppConn(clientCreator, metrics)
 }
 
 // multiAppConn implements AppConns.
@@ -44,6 +44,7 @@ func NewAppConns(clientCreator ClientCreator) AppConns {
 type multiAppConn struct {
 	service.BaseService
 
+	metrics       *Metrics
 	consensusConn AppConnConsensus
 	mempoolConn   AppConnMempool
 	queryConn     AppConnQuery
@@ -58,8 +59,9 @@ type multiAppConn struct {
 }
 
 // NewMultiAppConn makes all necessary abci connections to the application.
-func NewMultiAppConn(clientCreator ClientCreator) AppConns {
+func NewMultiAppConn(clientCreator ClientCreator, metrics *Metrics) AppConns {
 	multiAppConn := &multiAppConn{
+		metrics:       metrics,
 		clientCreator: clientCreator,
 	}
 	multiAppConn.BaseService = *service.NewBaseService(nil, "multiAppConn", multiAppConn)
@@ -88,7 +90,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.queryConnClient = c
-	app.queryConn = NewAppConnQuery(c)
+	app.queryConn = NewAppConnQuery(c, app.metrics)
 
 	c, err = app.abciClientFor(connSnapshot)
 	if err != nil {
@@ -96,7 +98,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.snapshotConnClient = c
-	app.snapshotConn = NewAppConnSnapshot(c)
+	app.snapshotConn = NewAppConnSnapshot(c, app.metrics)
 
 	c, err = app.abciClientFor(connMempool)
 	if err != nil {
@@ -104,7 +106,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.mempoolConnClient = c
-	app.mempoolConn = NewAppConnMempool(c)
+	app.mempoolConn = NewAppConnMempool(c, app.metrics)
 
 	c, err = app.abciClientFor(connConsensus)
 	if err != nil {
@@ -112,7 +114,7 @@ func (app *multiAppConn) OnStart() error {
 		return err
 	}
 	app.consensusConnClient = c
-	app.consensusConn = NewAppConnConsensus(c)
+	app.consensusConn = NewAppConnConsensus(c, app.metrics)
 
 	// Kill Tendermint if the ABCI application crashes.
 	go app.killTMOnClientError()
diff --git a/proxy/multi_app_conn_test.go b/proxy/multi_app_conn_test.go
index 34b0d0830..82ff19281 100644
--- a/proxy/multi_app_conn_test.go
+++ b/proxy/multi_app_conn_test.go
@@ -28,7 +28,7 @@ func TestAppConns_Start_Stop(t *testing.T) {
 
 	clientCreatorMock.On("NewABCIClient").Return(clientMock, nil).Times(4)
 
-	appConns := NewAppConns(clientCreatorMock)
+	appConns := NewAppConns(clientCreatorMock, NopMetrics())
 
 	err := appConns.Start()
 	require.NoError(t, err)
@@ -68,7 +68,7 @@ func TestAppConns_Failure(t *testing.T) {
 
 	clientCreatorMock.On("NewABCIClient").Return(clientMock, nil)
 
-	appConns := NewAppConns(clientCreatorMock)
+	appConns := NewAppConns(clientCreatorMock, NopMetrics())
 
 	err := appConns.Start()
 	require.NoError(t, err)
diff --git a/state/execution_test.go b/state/execution_test.go
index a8aeffe93..8b72dc952 100644
--- a/state/execution_test.go
+++ b/state/execution_test.go
@@ -35,7 +35,7 @@ var (
 func TestApplyBlock(t *testing.T) {
 	app := &testApp{}
 	cc := proxy.NewLocalClientCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -61,7 +61,7 @@ func TestApplyBlock(t *testing.T) {
 func TestBeginBlockValidators(t *testing.T) {
 	app := &testApp{}
 	cc := proxy.NewLocalClientCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // no need to check error again
@@ -124,7 +124,7 @@ func TestBeginBlockValidators(t *testing.T) {
 func TestBeginBlockByzantineValidators(t *testing.T) {
 	app := &testApp{}
 	cc := proxy.NewLocalClientCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -348,7 +348,7 @@ func TestUpdateValidators(t *testing.T) {
 func TestEndBlockValidatorUpdates(t *testing.T) {
 	app := &testApp{}
 	cc := proxy.NewLocalClientCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
@@ -419,7 +419,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) {
 func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) {
 	app := &testApp{}
 	cc := proxy.NewLocalClientCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
+	proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics())
 	err := proxyApp.Start()
 	require.Nil(t, err)
 	defer proxyApp.Stop() //nolint:errcheck // ignore for tests
diff --git a/state/helpers_test.go b/state/helpers_test.go
index 19549f160..5481ac7da 100644
--- a/state/helpers_test.go
+++ b/state/helpers_test.go
@@ -27,7 +27,7 @@ type paramsChangeTestCase struct {
 func newTestApp() proxy.AppConns {
 	app := &testApp{}
 	cc := proxy.NewLocalClientCreator(app)
-	return proxy.NewAppConns(cc)
+	return proxy.NewAppConns(cc, proxy.NopMetrics())
 }
 
 func makeAndCommitGoodBlock(