abci: remove TotalTxs and NumTxs from Header (#3783)

* Removal of TotalTx & NumTx

- Removed totalTx and numTx

closes #2521

Signed-off-by: Marko Baricevic <marbar3778@yahoo.com>

* abci proto changes

* proto number fix

* txfilter_test fix

* comments on PR

* further changes

* bring back metrics

* fix indexer

* fix TestBlockMaxDataBytes and TestBlockMaxDataBytesUnknownEvidence

* indexer service back to header

* statistics.go fix

* fix ci

* listen for blocks, not headers

to be able to record txs throughput

* fix TestNetworkNewBlock

* fix tests

* fix tests in types package

* fixes after Anton's review

* fix tests

* bring back `consensus_total_txs` metric

I mistakenly thought it was removed.

* improve changelog

* remove LastBlockTotalTx from state

* docs: remove getNumTxs from BeginBlock Java example

tools/tm-bench/statistics.go

@@ -67,8 +67,8 @@ func calculateStatistics(
 		numBlocksPerSec[sec]++
 
 		// increase number of txs for that second
-		numTxsPerSec[sec] += blockMeta.Header.NumTxs
-		logger.Debug(fmt.Sprintf("%d txs at block height %d", blockMeta.Header.NumTxs, blockMeta.Header.Height))
+		numTxsPerSec[sec] += blockMeta.NumTxs
+		logger.Debug(fmt.Sprintf("%d txs at block height %d", blockMeta.NumTxs, blockMeta.Header.Height))
 	}
 
 	for i := int64(0); i < int64(duration); i++ {

tools/tm-monitor/monitor/monitor.go

@@ -82,7 +82,7 @@ func (m *Monitor) Monitor(n *Node) error {
 	m.Nodes = append(m.Nodes, n)
 	m.mtx.Unlock()
 
-	blockCh := make(chan tmtypes.Header, 10)
+	blockCh := make(chan *tmtypes.Block, 10)
 	n.SendBlocksTo(blockCh)
 	blockLatencyCh := make(chan float64, 10)
 	n.SendBlockLatenciesTo(blockLatencyCh)
@@ -167,7 +167,7 @@ func (m *Monitor) Stop() {
 // main loop where we listen for events from the node
 func (m *Monitor) listen(
 	nodeName string,
-	blockCh <-chan tmtypes.Header,
+	blockCh <-chan *tmtypes.Block,
 	blockLatencyCh <-chan float64,
 	disconnectCh <-chan bool,
 	quit <-chan struct{}) {

tools/tm-monitor/monitor/network.go

@@ -69,7 +69,7 @@ func NewNetwork() *Network {
 	}
 }
 
-func (n *Network) NewBlock(b tmtypes.Header) {
+func (n *Network) NewBlock(b *tmtypes.Block) {
 	n.mu.Lock()
 	defer n.mu.Unlock()
 
@@ -85,7 +85,7 @@ func (n *Network) NewBlock(b tmtypes.Header) {
 	} else {
 		n.AvgBlockTime = 0.0
 	}
-	n.txThroughputMeter.Mark(b.NumTxs)
+	n.txThroughputMeter.Mark(int64(len(b.Data.Txs)))
 	n.AvgTxThroughput = n.txThroughputMeter.Rate1()
 }
 

tools/tm-monitor/monitor/network_test.go

@@ -13,7 +13,9 @@ import (
 func TestNetworkNewBlock(t *testing.T) {
 	n := monitor.NewNetwork()
 
-	n.NewBlock(tmtypes.Header{Height: 5, NumTxs: 100})
+	n.NewBlock(&tmtypes.Block{
+		Header: tmtypes.Header{Height: 5},
+	})
 	assert.Equal(t, int64(5), n.Height)
 	assert.Equal(t, 0.0, n.AvgBlockTime)
 	assert.Equal(t, 0.0, n.AvgTxThroughput)

tools/tm-monitor/monitor/node.go

@@ -35,7 +35,7 @@ type Node struct {
 	// rpcClient is an client for making RPC calls to TM
 	rpcClient rpc_client.HTTPClient
 
-	blockCh        chan<- tmtypes.Header
+	blockCh        chan<- *tmtypes.Block
 	blockLatencyCh chan<- float64
 	disconnectCh   chan<- bool
 
@@ -83,7 +83,7 @@ func SetCheckIsValidatorInterval(d time.Duration) func(n *Node) {
 	}
 }
 
-func (n *Node) SendBlocksTo(ch chan<- tmtypes.Header) {
+func (n *Node) SendBlocksTo(ch chan<- *tmtypes.Block) {
 	n.blockCh = ch
 }
 
@@ -107,7 +107,7 @@ func (n *Node) Start() error {
 	}
 
 	n.em.RegisterLatencyCallback(latencyCallback(n))
-	err := n.em.Subscribe(tmtypes.EventQueryNewBlockHeader.String(), newBlockCallback(n))
+	err := n.em.Subscribe(tmtypes.EventQueryNewBlock.String(), newBlockCallback(n))
 	if err != nil {
 		return err
 	}
@@ -132,10 +132,10 @@ func (n *Node) Stop() {
 // implements eventmeter.EventCallbackFunc
 func newBlockCallback(n *Node) em.EventCallbackFunc {
 	return func(metric *em.EventMetric, data interface{}) {
-		block := data.(tmtypes.TMEventData).(tmtypes.EventDataNewBlockHeader).Header
+		block := data.(tmtypes.TMEventData).(tmtypes.EventDataNewBlock).Block
 
 		n.Height = block.Height
-		n.logger.Info("new block", "height", block.Height, "numTxs", block.NumTxs)
+		n.logger.Info("new block", "height", block.Height)
 
 		if n.blockCh != nil {
 			n.blockCh <- block

tools/tm-monitor/monitor/node_test.go

@@ -28,16 +28,16 @@ func TestNodeStartStop(t *testing.T) {
 }
 
 func TestNodeNewBlockReceived(t *testing.T) {
-	blockCh := make(chan tmtypes.Header, 100)
+	blockCh := make(chan *tmtypes.Block, 100)
 	n, emMock := startValidatorNode(t)
 	defer n.Stop()
 	n.SendBlocksTo(blockCh)
 
-	blockHeader := tmtypes.Header{Height: 5}
-	emMock.Call("eventCallback", &em.EventMetric{}, tmtypes.EventDataNewBlockHeader{Header: blockHeader})
+	block := &tmtypes.Block{Header: tmtypes.Header{Height: 5}}
+	emMock.Call("eventCallback", &em.EventMetric{}, tmtypes.EventDataNewBlock{Block: block})
 
 	assert.Equal(t, int64(5), n.Height)
-	assert.Equal(t, blockHeader, <-blockCh)
+	assert.Equal(t, block, <-blockCh)
 }
 
 func TestNodeNewBlockLatencyReceived(t *testing.T) {