fix amino overhead computation for Tx (#2792)

* fix amino overhead computation for Tx:

- also count the fieldnum / typ3
- add method to compute overhead per Tx
- slightly clarify comment on MaxAminoOverheadForBlock
- add tests

* fix TestReapMaxBytesMaxGas according to amino overhead

* fix TestMempoolFilters according to amino overhead

* address review comments:

 - add a note about fieldNum = 1
 - add forgotten godoc comment

* fix and use sm.TxPreCheck

* fix test

* remove print statement

types/block.go

@@ -21,6 +21,8 @@ const (
 
 	// MaxAminoOverheadForBlock - maximum amino overhead to encode a block (up to
 	// MaxBlockSizeBytes in size) not including it's parts except Data.
+	// This means it also excludes the overhead for individual transactions.
+	// To compute individual transactions' overhead use types.ComputeAminoOverhead(tx types.Tx, fieldNum int).
 	//
 	// Uvarint length of MaxBlockSizeBytes: 4 bytes
 	// 2 fields (2 embedded):               2 bytes

types/block_test.go

@@ -250,7 +250,7 @@ func TestMaxHeaderBytes(t *testing.T) {
 	timestamp := time.Date(math.MaxInt64, 0, 0, 0, 0, 0, math.MaxInt64, time.UTC)
 
 	h := Header{
-		Version:            version.Consensus{math.MaxInt64, math.MaxInt64},
+		Version:            version.Consensus{Block: math.MaxInt64, App: math.MaxInt64},
 		ChainID:            maxChainID,
 		Height:             math.MaxInt64,
 		Time:               timestamp,

types/tx.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/tendermint/go-amino"
+
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/crypto/merkle"
 	"github.com/tendermint/tendermint/crypto/tmhash"
@@ -118,3 +120,18 @@ type TxResult struct {
 	Tx     Tx                     `json:"tx"`
 	Result abci.ResponseDeliverTx `json:"result"`
 }
+
+// ComputeAminoOverhead calculates the overhead for amino encoding a transaction.
+// The overhead consists of varint encoding the field number and the wire type
+// (= length-delimited = 2), and another varint encoding the length of the
+// transaction.
+// The field number can be the field number of the particular transaction, or
+// the field number of the parenting struct that contains the transactions []Tx
+// as a field (this field number is repeated for each contained Tx).
+// If some []Tx are encoded directly (without a parenting struct), the default
+// fieldNum is also 1 (see BinFieldNum in amino.MarshalBinaryBare).
+func ComputeAminoOverhead(tx Tx, fieldNum int) int64 {
+	fnum := uint64(fieldNum)
+	typ3AndFieldNum := (uint64(fnum) << 3) | uint64(amino.Typ3_ByteLength)
+	return int64(amino.UvarintSize(typ3AndFieldNum)) + int64(amino.UvarintSize(uint64(len(tx))))
+}

types/tx_test.go

@@ -96,6 +96,63 @@ func TestTxProofUnchangable(t *testing.T) {
 	}
 }
 
+func TestComputeTxsOverhead(t *testing.T) {
+	cases := []struct {
+		txs          Txs
+		wantOverhead int
+	}{
+		{Txs{[]byte{6, 6, 6, 6, 6, 6}}, 2},
+		// one 21 Mb transaction:
+		{Txs{make([]byte, 22020096, 22020096)}, 5},
+		// two 21Mb/2 sized transactions:
+		{Txs{make([]byte, 11010048, 11010048), make([]byte, 11010048, 11010048)}, 10},
+		{Txs{[]byte{1, 2, 3}, []byte{1, 2, 3}, []byte{4, 5, 6}}, 6},
+		{Txs{[]byte{100, 5, 64}, []byte{42, 116, 118}, []byte{6, 6, 6}, []byte{6, 6, 6}}, 8},
+	}
+
+	for _, tc := range cases {
+		totalBytes := int64(0)
+		totalOverhead := int64(0)
+		for _, tx := range tc.txs {
+			aminoOverhead := ComputeAminoOverhead(tx, 1)
+			totalOverhead += aminoOverhead
+			totalBytes += aminoOverhead + int64(len(tx))
+		}
+		bz, err := cdc.MarshalBinaryBare(tc.txs)
+		assert.EqualValues(t, tc.wantOverhead, totalOverhead)
+		assert.NoError(t, err)
+		assert.EqualValues(t, len(bz), totalBytes)
+	}
+}
+
+func TestComputeAminoOverhead(t *testing.T) {
+	cases := []struct {
+		tx       Tx
+		fieldNum int
+		want     int
+	}{
+		{[]byte{6, 6, 6}, 1, 2},
+		{[]byte{6, 6, 6}, 16, 3},
+		{[]byte{6, 6, 6}, 32, 3},
+		{[]byte{6, 6, 6}, 64, 3},
+		{[]byte{6, 6, 6}, 512, 3},
+		{[]byte{6, 6, 6}, 1024, 3},
+		{[]byte{6, 6, 6}, 2048, 4},
+		{make([]byte, 64), 1, 2},
+		{make([]byte, 65), 1, 2},
+		{make([]byte, 127), 1, 2},
+		{make([]byte, 128), 1, 3},
+		{make([]byte, 256), 1, 3},
+		{make([]byte, 512), 1, 3},
+		{make([]byte, 1024), 1, 3},
+		{make([]byte, 128), 16, 4},
+	}
+	for _, tc := range cases {
+		got := ComputeAminoOverhead(tc.tx, tc.fieldNum)
+		assert.EqualValues(t, tc.want, got)
+	}
+}
+
 func testTxProofUnchangable(t *testing.T) {
 	// make some proof
 	txs := makeTxs(randInt(2, 100), randInt(16, 128))