evidence: use bytes instead of quantity to limit size (#5449)(#5476)

types/block.go

@@ -273,12 +273,12 @@ func BlockFromProto(bp *tmproto.Block) (*Block, error) {
 // MaxDataBytes returns the maximum size of block's data.
 //
 // XXX: Panics on negative result.
-func MaxDataBytes(maxBytes int64, valsCount, evidenceCount int) int64 {
+func MaxDataBytes(maxBytes, evidenceBytes int64, valsCount int) int64 {
 	maxDataBytes := maxBytes -
 		MaxOverheadForBlock -
 		MaxHeaderBytes -
 		int64(valsCount)*MaxVoteBytes -
-		int64(evidenceCount)*MaxEvidenceBytes
+		evidenceBytes
 
 	if maxDataBytes < 0 {
 		panic(fmt.Sprintf(
@@ -292,18 +292,16 @@ func MaxDataBytes(maxBytes int64, valsCount, evidenceCount int) int64 {
 
 }
 
-// MaxDataBytesUnknownEvidence returns the maximum size of block's data when
+// MaxDataBytesNoEvidence returns the maximum size of block's data when
 // evidence count is unknown. MaxEvidencePerBlock will be used for the size
 // of evidence.
 //
 // XXX: Panics on negative result.
-func MaxDataBytesUnknownEvidence(maxBytes int64, valsCount int, maxNumEvidence uint32) int64 {
-	maxEvidenceBytes := int64(maxNumEvidence) * MaxEvidenceBytes
+func MaxDataBytesNoEvidence(maxBytes int64, valsCount int) int64 {
 	maxDataBytes := maxBytes -
 		MaxOverheadForBlock -
 		MaxHeaderBytes -
-		int64(valsCount)*MaxVoteBytes -
-		maxEvidenceBytes
+		int64(valsCount)*MaxVoteBytes
 
 	if maxDataBytes < 0 {
 		panic(fmt.Sprintf(
@@ -1073,8 +1071,9 @@ func DataFromProto(dp *tmproto.Data) (Data, error) {
 type EvidenceData struct {
 	Evidence EvidenceList `json:"evidence"`
 
-	// Volatile
-	hash tmbytes.HexBytes
+	// Volatile. Used as cache
+	hash     tmbytes.HexBytes
+	byteSize int64
 }
 
 // Hash returns the hash of the data.
@@ -1085,6 +1084,20 @@ func (data *EvidenceData) Hash() tmbytes.HexBytes {
 	return data.hash
 }
 
+// ByteSize returns the total byte size of all the evidence
+func (data *EvidenceData) ByteSize() int64 {
+	if data.byteSize == 0 && len(data.Evidence) != 0 {
+		for _, ev := range data.Evidence {
+			pb, err := EvidenceToProto(ev)
+			if err != nil {
+				panic(err)
+			}
+			data.byteSize += int64(pb.Size())
+		}
+	}
+	return data.byteSize
+}
+
 // StringIndented returns a string representation of the evidence.
 func (data *EvidenceData) StringIndented(indent string) string {
 	if data == nil {
@@ -1142,11 +1155,10 @@ func (data *EvidenceData) FromProto(eviData *tmproto.EvidenceData) error {
 			return err
 		}
 		eviBzs[i] = evi
+		data.byteSize += int64(eviData.Evidence[i].Size())
 	}
 	data.Evidence = eviBzs
 
-	data.hash = eviData.GetHash()
-
 	return nil
 }
 

types/block_test.go

@@ -401,7 +401,7 @@ func TestBlockMaxDataBytes(t *testing.T) {
 	testCases := []struct {
 		maxBytes      int64
 		valsCount     int
-		evidenceCount int
+		evidenceBytes int64
 		panics        bool
 		result        int64
 	}{
@@ -416,43 +416,41 @@ func TestBlockMaxDataBytes(t *testing.T) {
 		tc := tc
 		if tc.panics {
 			assert.Panics(t, func() {
-				MaxDataBytes(tc.maxBytes, tc.valsCount, tc.evidenceCount)
+				MaxDataBytes(tc.maxBytes, tc.evidenceBytes, tc.valsCount)
 			}, "#%v", i)
 		} else {
 			assert.Equal(t,
 				tc.result,
-				MaxDataBytes(tc.maxBytes, tc.valsCount, tc.evidenceCount),
+				MaxDataBytes(tc.maxBytes, tc.evidenceBytes, tc.valsCount),
 				"#%v", i)
 		}
 	}
 }
 
-func TestBlockMaxDataBytesUnknownEvidence(t *testing.T) {
+func TestBlockMaxDataBytesNoEvidence(t *testing.T) {
 	testCases := []struct {
-		maxBytes    int64
-		maxEvidence uint32
-		valsCount   int
-		panics      bool
-		result      int64
+		maxBytes  int64
+		valsCount int
+		panics    bool
+		result    int64
 	}{
-		0: {-10, 0, 1, true, 0},
-		1: {10, 0, 1, true, 0},
-		2: {845, 0, 1, true, 0},
-		3: {846, 0, 1, false, 0},
-		4: {1290, 1, 1, false, 0},
-		5: {1291, 1, 1, false, 1},
+		0: {-10, 1, true, 0},
+		1: {10, 1, true, 0},
+		2: {845, 1, true, 0},
+		3: {846, 1, false, 0},
+		4: {847, 1, false, 1},
 	}
 
 	for i, tc := range testCases {
 		tc := tc
 		if tc.panics {
 			assert.Panics(t, func() {
-				MaxDataBytesUnknownEvidence(tc.maxBytes, tc.valsCount, tc.maxEvidence)
+				MaxDataBytesNoEvidence(tc.maxBytes, tc.valsCount)
 			}, "#%v", i)
 		} else {
 			assert.Equal(t,
 				tc.result,
-				MaxDataBytesUnknownEvidence(tc.maxBytes, tc.valsCount, tc.maxEvidence),
+				MaxDataBytesNoEvidence(tc.maxBytes, tc.valsCount),
 				"#%v", i)
 		}
 	}
@@ -620,7 +618,7 @@ func TestBlockProtoBuf(t *testing.T) {
 			require.NoError(t, err, tc.msg)
 			require.EqualValues(t, tc.b1.Header, block.Header, tc.msg)
 			require.EqualValues(t, tc.b1.Data, block.Data, tc.msg)
-			require.EqualValues(t, tc.b1.Evidence, block.Evidence, tc.msg)
+			require.EqualValues(t, tc.b1.Evidence.Evidence, block.Evidence.Evidence, tc.msg)
 			require.EqualValues(t, *tc.b1.LastCommit, *block.LastCommit, tc.msg)
 		} else {
 			require.Error(t, err, tc.msg)
@@ -653,6 +651,7 @@ func TestDataProtoBuf(t *testing.T) {
 	}
 }
 
+// TestEvidenceDataProtoBuf ensures parity in converting to and from proto.
 func TestEvidenceDataProtoBuf(t *testing.T) {
 	val := NewMockPV()
 	blockID := makeBlockID(tmhash.Sum([]byte("blockhash")), math.MaxInt32, tmhash.Sum([]byte("partshash")))
@@ -662,7 +661,7 @@ func TestEvidenceDataProtoBuf(t *testing.T) {
 	v2 := makeVote(t, val, chainID, math.MaxInt32, math.MaxInt64, 2, 0x01, blockID2, time.Now())
 	ev := NewDuplicateVoteEvidence(v2, v)
 	data := &EvidenceData{Evidence: EvidenceList{ev}}
-	_ = data.Hash()
+	_ = data.ByteSize()
 	testCases := []struct {
 		msg      string
 		data1    *EvidenceData

types/evidence.go

@@ -25,11 +25,6 @@ type Evidence interface {
 	String() string       // string format of the evidence
 }
 
-const (
-	// MaxEvidenceBytes is a maximum size of any evidence (including amino overhead).
-	MaxEvidenceBytes int64 = 444
-)
-
 //--------------------------------------------------------------------------------------
 
 // DuplicateVoteEvidence contains evidence a validator signed two conflicting
@@ -365,20 +360,20 @@ func (err *ErrInvalidEvidence) Error() string {
 	return fmt.Sprintf("Invalid evidence: %v. Evidence: %v", err.Reason, err.Evidence)
 }
 
-// ErrEvidenceOverflow is for when there is too much evidence in a block.
+// ErrEvidenceOverflow is for when there the amount of evidence exceeds the max bytes.
 type ErrEvidenceOverflow struct {
-	MaxNum int
-	GotNum int
+	Max int64
+	Got int64
 }
 
 // NewErrEvidenceOverflow returns a new ErrEvidenceOverflow where got > max.
-func NewErrEvidenceOverflow(max, got int) *ErrEvidenceOverflow {
+func NewErrEvidenceOverflow(max, got int64) *ErrEvidenceOverflow {
 	return &ErrEvidenceOverflow{max, got}
 }
 
 // Error returns a string representation of the error.
 func (err *ErrEvidenceOverflow) Error() string {
-	return fmt.Sprintf("Too much evidence: Max %d, got %d", err.MaxNum, err.GotNum)
+	return fmt.Sprintf("Too much evidence: Max %d, got %d", err.Max, err.Got)
 }
 
 //-------------------------------------------- MOCKING --------------------------------------

types/evidence_test.go

@@ -27,37 +27,6 @@ func TestEvidenceList(t *testing.T) {
 	assert.False(t, evl.Has(&DuplicateVoteEvidence{}))
 }
 
-func TestMaxEvidenceBytes(t *testing.T) {
-	val := NewMockPV()
-	blockID := makeBlockID(tmhash.Sum([]byte("blockhash")), math.MaxInt32, tmhash.Sum([]byte("partshash")))
-	blockID2 := makeBlockID(tmhash.Sum([]byte("blockhash2")), math.MaxInt32, tmhash.Sum([]byte("partshash")))
-	maxTime := time.Date(9999, 0, 0, 0, 0, 0, 0, time.UTC)
-	const chainID = "mychain"
-	ev := &DuplicateVoteEvidence{
-		VoteA: makeVote(t, val, chainID, math.MaxInt32, math.MaxInt64, math.MaxInt32, math.MaxInt64, blockID, maxTime),
-		VoteB: makeVote(t, val, chainID, math.MaxInt32, math.MaxInt64, math.MaxInt32, math.MaxInt64, blockID2, maxTime),
-	}
-
-	//TODO: Add other types of evidence to test and set MaxEvidenceBytes accordingly
-
-	testCases := []struct {
-		testName string
-		evidence Evidence
-	}{
-		{"DuplicateVote", ev},
-	}
-
-	for _, tt := range testCases {
-		pb, err := EvidenceToProto(tt.evidence)
-		require.NoError(t, err, tt.testName)
-		bz, err := pb.Marshal()
-		require.NoError(t, err, tt.testName)
-
-		assert.LessOrEqual(t, int64(len(bz)), MaxEvidenceBytes, tt.testName)
-	}
-
-}
-
 func randomDuplicateVoteEvidence(t *testing.T) *DuplicateVoteEvidence {
 	val := NewMockPV()
 	blockID := makeBlockID([]byte("blockhash"), 1000, []byte("partshash"))

types/params.go

@@ -19,9 +19,6 @@ const (
 
 	// MaxBlockPartsCount is the maximum number of block parts.
 	MaxBlockPartsCount = (MaxBlockSizeBytes / BlockPartSizeBytes) + 1
-
-	// Restrict the upper bound of the amount of evidence (uses uint16 for safe conversion)
-	MaxEvidencePerBlock = 65535
 )
 
 // DefaultConsensusParams returns a default ConsensusParams.
@@ -48,7 +45,7 @@ func DefaultEvidenceParams() tmproto.EvidenceParams {
 	return tmproto.EvidenceParams{
 		MaxAgeNumBlocks: 100000, // 27.8 hrs at 1block/s
 		MaxAgeDuration:  48 * time.Hour,
-		MaxNum:          50,
+		MaxBytes:        1048576, // 1MB
 	}
 }
 
@@ -98,23 +95,23 @@ func ValidateConsensusParams(params tmproto.ConsensusParams) error {
 	}
 
 	if params.Evidence.MaxAgeNumBlocks <= 0 {
-		return fmt.Errorf("evidenceParams.MaxAgeNumBlocks must be greater than 0. Got %d",
+		return fmt.Errorf("evidence.MaxAgeNumBlocks must be greater than 0. Got %d",
 			params.Evidence.MaxAgeNumBlocks)
 	}
 
 	if params.Evidence.MaxAgeDuration <= 0 {
-		return fmt.Errorf("evidenceParams.MaxAgeDuration must be grater than 0 if provided, Got %v",
+		return fmt.Errorf("evidence.MaxAgeDuration must be grater than 0 if provided, Got %v",
 			params.Evidence.MaxAgeDuration)
 	}
 
-	if params.Evidence.MaxNum > MaxEvidencePerBlock {
-		return fmt.Errorf("evidenceParams.MaxNumEvidence is greater than upper bound, %d > %d",
-			params.Evidence.MaxNum, MaxEvidencePerBlock)
+	if params.Evidence.MaxBytes > params.Block.MaxBytes {
+		return fmt.Errorf("evidence.MaxBytesEvidence is greater than upper bound, %d > %d",
+			params.Evidence.MaxBytes, params.Block.MaxBytes)
 	}
 
-	if int64(params.Evidence.MaxNum)*MaxEvidenceBytes > params.Block.MaxBytes {
-		return fmt.Errorf("total possible evidence size is bigger than block.MaxBytes, %d > %d",
-			int64(params.Evidence.MaxNum)*MaxEvidenceBytes, params.Block.MaxBytes)
+	if params.Evidence.MaxBytes < 0 {
+		return fmt.Errorf("evidence.MaxBytes must be non negative. Got: %d",
+			params.Evidence.MaxBytes)
 	}
 
 	if len(params.Validator.PubKeyTypes) == 0 {
@@ -174,7 +171,7 @@ func UpdateConsensusParams(params tmproto.ConsensusParams, params2 *abci.Consens
 	if params2.Evidence != nil {
 		res.Evidence.MaxAgeNumBlocks = params2.Evidence.MaxAgeNumBlocks
 		res.Evidence.MaxAgeDuration = params2.Evidence.MaxAgeDuration
-		res.Evidence.MaxNum = params2.Evidence.MaxNum
+		res.Evidence.MaxBytes = params2.Evidence.MaxBytes
 	}
 	if params2.Validator != nil {
 		// Copy params2.Validator.PubkeyTypes, and set result's value to the copy.

types/params_test.go

@@ -33,7 +33,7 @@ func TestConsensusParamsValidation(t *testing.T) {
 		8: {makeParams(1, 0, -10, 2, 0, valEd25519), false},
 		// test evidence params
 		9:  {makeParams(1, 0, 10, 0, 0, valEd25519), false},
-		10: {makeParams(1, 0, 10, 2, 1, valEd25519), false},
+		10: {makeParams(1, 0, 10, 2, 2, valEd25519), false},
 		11: {makeParams(1000, 0, 10, 2, 1, valEd25519), true},
 		12: {makeParams(1, 0, 10, -1, 0, valEd25519), false},
 		// test no pubkey type provided
@@ -54,7 +54,7 @@ func makeParams(
 	blockBytes, blockGas int64,
 	blockTimeIotaMs int64,
 	evidenceAge int64,
-	maxEvidence uint32,
+	maxEvidenceBytes int64,
 	pubkeyTypes []string,
 ) tmproto.ConsensusParams {
 	return tmproto.ConsensusParams{
@@ -66,7 +66,7 @@ func makeParams(
 		Evidence: tmproto.EvidenceParams{
 			MaxAgeNumBlocks: evidenceAge,
 			MaxAgeDuration:  time.Duration(evidenceAge),
-			MaxNum:          maxEvidence,
+			MaxBytes:        maxEvidenceBytes,
 		},
 		Validator: tmproto.ValidatorParams{
 			PubKeyTypes: pubkeyTypes,
@@ -124,7 +124,7 @@ func TestConsensusParamsUpdate(t *testing.T) {
 				Evidence: &tmproto.EvidenceParams{
 					MaxAgeNumBlocks: 300,
 					MaxAgeDuration:  time.Duration(300),
-					MaxNum:          50,
+					MaxBytes:        50,
 				},
 				Validator: &tmproto.ValidatorParams{
 					PubKeyTypes: valEd25519,

types/part_set.go

@@ -121,7 +121,7 @@ func (psh PartSetHeader) ValidateBasic() error {
 	return nil
 }
 
-// ToProto converts BloPartSetHeaderckID to protobuf
+// ToProto converts PartSetHeader to protobuf
 func (psh *PartSetHeader) ToProto() tmproto.PartSetHeader {
 	if psh == nil {
 		return tmproto.PartSetHeader{}