implement vote extensions

state/execution.go

@@ -1,7 +1,9 @@
 package state
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -94,7 +96,7 @@ func (blockExec *BlockExecutor) SetEventBus(eventBus types.BlockEventPublisher)
 func (blockExec *BlockExecutor) CreateProposalBlock(
 	height int64,
 	state State,
-	commit *types.Commit,
+	lastExtCommit *types.ExtendedCommit,
 	proposerAddr []byte,
 	votes []*types.Vote,
 ) (*types.Block, error) {
@@ -108,14 +110,14 @@ func (blockExec *BlockExecutor) CreateProposalBlock(
 	maxDataBytes := types.MaxDataBytes(maxBytes, evSize, state.Validators.Size())
 
 	txs := blockExec.mempool.ReapMaxBytesMaxGas(maxDataBytes, maxGas)
+	commit := lastExtCommit.ToCommit()
 	block := state.MakeBlock(height, txs, commit, evidence, proposerAddr)
 
-	localLastCommit := buildLastCommitInfo(block, blockExec.store, state.InitialHeight)
 	rpp, err := blockExec.proxyApp.PrepareProposal(context.TODO(),
 		&abci.RequestPrepareProposal{
 			MaxTxBytes:         maxDataBytes,
 			Txs:                block.Txs.ToSliceOfBytes(),
-			LocalLastCommit:    extendedCommitInfo(localLastCommit, votes),
+			LocalLastCommit:    buildExtendedCommitInfo(lastExtCommit, blockExec.store, state.InitialHeight, state.ConsensusParams.ABCI),
 			Misbehavior:        block.Evidence.Evidence.ToABCI(),
 			Height:             block.Height,
 			Time:               block.Time,
@@ -309,6 +311,35 @@ func (blockExec *BlockExecutor) Commit(
 	return res.RetainHeight, err
 }
 
+func (blockExec *BlockExecutor) ExtendVote(ctx context.Context, vote *types.Vote) ([]byte, error) {
+	resp, err := blockExec.proxyApp.ExtendVote(ctx, &abci.RequestExtendVote{
+		BlockHash: vote.BlockID.Hash,
+		Height:    vote.Height,
+	})
+	if err != nil {
+		panic(fmt.Errorf("ExtendVote call failed: %w", err))
+	}
+	return resp.VoteExtension, nil
+}
+
+func (blockExec *BlockExecutor) VerifyVoteExtension(ctx context.Context, vote *types.Vote) error {
+	resp, err := blockExec.proxyApp.VerifyVoteExtension(ctx, &abci.RequestVerifyVoteExtension{
+		BlockHash:        vote.BlockID.Hash,
+		ValidatorAddress: vote.ValidatorAddress,
+		Height:           vote.Height,
+		VoteExtension:    vote.Extension,
+	})
+	if err != nil {
+		panic(fmt.Errorf("VerifyVoteExtension call failed: %w", err))
+	}
+
+	if !resp.IsOK() {
+		return errors.New("invalid vote extension")
+	}
+
+	return nil
+}
+
 //---------------------------------------------------------
 // Helper functions for executing blocks and updating state
 
@@ -387,21 +418,75 @@ func buildLastCommitInfo(block *types.Block, store Store, initialHeight int64) a
 	}
 }
 
-func extendedCommitInfo(c abci.CommitInfo, votes []*types.Vote) abci.ExtendedCommitInfo {
-	vs := make([]abci.ExtendedVoteInfo, len(c.Votes))
-	for i := range vs {
-		vs[i] = abci.ExtendedVoteInfo{
-			Validator:       c.Votes[i].Validator,
-			SignedLastBlock: c.Votes[i].SignedLastBlock,
-			/*
-				TODO: Include vote extensions information when implementing vote extensions.
-				VoteExtension:   []byte{},
-			*/
+// buildExtendedCommitInfo populates an ABCI extended commit from the
+// corresponding Tendermint extended commit ec, using the stored validator set
+// from ec.  It requires ec to include the original precommit votes along with
+// the vote extensions from the last commit.
+//
+// For heights below the initial height, for which we do not have the required
+// data, it returns an empty record.
+//
+// Assumes that the commit signatures are sorted according to validator index.
+func buildExtendedCommitInfo(ec *types.ExtendedCommit, store Store, initialHeight int64, ap types.ABCIParams) abci.ExtendedCommitInfo {
+	if ec.Height < initialHeight {
+		// There are no extended commits for heights below the initial height.
+		return abci.ExtendedCommitInfo{}
+	}
+
+	valSet, err := store.LoadValidators(ec.Height)
+	if err != nil {
+		panic(fmt.Errorf("failed to load validator set at height %d, initial height %d: %w", ec.Height, initialHeight, err))
+	}
+
+	var (
+		ecSize    = ec.Size()
+		valSetLen = len(valSet.Validators)
+	)
+
+	// Ensure that the size of the validator set in the extended commit matches
+	// the size of the validator set in the state store.
+	if ecSize != valSetLen {
+		panic(fmt.Errorf(
+			"extended commit size (%d) does not match validator set length (%d) at height %d\n\n%v\n\n%v",
+			ecSize, valSetLen, ec.Height, ec.ExtendedSignatures, valSet.Validators,
+		))
+	}
+
+	votes := make([]abci.ExtendedVoteInfo, ecSize)
+	for i, val := range valSet.Validators {
+		ecs := ec.ExtendedSignatures[i]
+
+		// Absent signatures have empty validator addresses, but otherwise we
+		// expect the validator addresses to be the same.
+		if ecs.BlockIDFlag != types.BlockIDFlagAbsent && !bytes.Equal(ecs.ValidatorAddress, val.Address) {
+			panic(fmt.Errorf("validator address of extended commit signature in position %d (%s) does not match the corresponding validator's at height %d (%s)",
+				i, ecs.ValidatorAddress, ec.Height, val.Address,
+			))
+		}
+
+		var ext []byte
+		// Check if vote extensions were enabled during the commit's height: ec.Height.
+		// ec is the commit from the previous height, so if extensions were enabled
+		// during that height, we ensure they are present and deliver the data to
+		// the proposer. If they were not enabled during this previous height, we
+		// will not deliver extension data.
+		if ap.VoteExtensionsEnabled(ec.Height) && ecs.BlockIDFlag == types.BlockIDFlagCommit {
+			if err := ecs.EnsureExtension(); err != nil {
+				panic(fmt.Errorf("commit at height %d received with missing vote extensions data", ec.Height))
+			}
+			ext = ecs.Extension
+		}
+
+		votes[i] = abci.ExtendedVoteInfo{
+			Validator:       types.TM2PB.Validator(val),
+			SignedLastBlock: ecs.BlockIDFlag != types.BlockIDFlagAbsent,
+			VoteExtension:   ext,
 		}
 	}
+
 	return abci.ExtendedCommitInfo{
-		Round: c.Round,
-		Votes: vs,
+		Round: ec.Round,
+		Votes: votes,
 	}
 }
 

state/execution_test.go

@@ -622,8 +622,7 @@ func TestEmptyPrepareProposal(t *testing.T) {
 		sm.EmptyEvidencePool{},
 	)
 	pa, _ := state.Validators.GetByIndex(0)
-	commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals)
-	require.NoError(t, err)
+	commit, _ := makeValidExtendedCommit(t, height, types.BlockID{}, state.Validators, privVals)
 	_, err = blockExec.CreateProposalBlock(height, state, commit, pa, nil)
 	require.NoError(t, err)
 }
@@ -663,8 +662,7 @@ func TestPrepareProposalTxsAllIncluded(t *testing.T) {
 		evpool,
 	)
 	pa, _ := state.Validators.GetByIndex(0)
-	commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals)
-	require.NoError(t, err)
+	commit, _ := makeValidExtendedCommit(t, height, types.BlockID{}, state.Validators, privVals)
 	block, err := blockExec.CreateProposalBlock(height, state, commit, pa, nil)
 	require.NoError(t, err)
 
@@ -714,8 +712,7 @@ func TestPrepareProposalReorderTxs(t *testing.T) {
 		evpool,
 	)
 	pa, _ := state.Validators.GetByIndex(0)
-	commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals)
-	require.NoError(t, err)
+	commit, _ := makeValidExtendedCommit(t, height, types.BlockID{}, state.Validators, privVals)
 	block, err := blockExec.CreateProposalBlock(height, state, commit, pa, nil)
 	require.NoError(t, err)
 	for i, tx := range block.Data.Txs {
@@ -767,8 +764,7 @@ func TestPrepareProposalErrorOnTooManyTxs(t *testing.T) {
 		evpool,
 	)
 	pa, _ := state.Validators.GetByIndex(0)
-	commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals)
-	require.NoError(t, err)
+	commit, _ := makeValidExtendedCommit(t, height, types.BlockID{}, state.Validators, privVals)
 
 	block, err := blockExec.CreateProposalBlock(height, state, commit, pa, nil)
 	require.Nil(t, block)
@@ -815,8 +811,7 @@ func TestPrepareProposalErrorOnPrepareProposalError(t *testing.T) {
 		evpool,
 	)
 	pa, _ := state.Validators.GetByIndex(0)
-	commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals)
-	require.NoError(t, err)
+	commit, _ := makeValidExtendedCommit(t, height, types.BlockID{}, state.Validators, privVals)
 
 	block, err := blockExec.CreateProposalBlock(height, state, commit, pa, nil)
 	require.Nil(t, block)

state/helpers_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	dbm "github.com/tendermint/tm-db"
 
 	abci "github.com/tendermint/tendermint/abci/types"
@@ -32,25 +33,25 @@ func newTestApp() proxy.AppConns {
 }
 
 func makeAndCommitGoodBlock(
+	t *testing.T,
 	state sm.State,
 	height int64,
 	lastCommit *types.Commit,
 	proposerAddr []byte,
 	blockExec *sm.BlockExecutor,
 	privVals map[string]types.PrivValidator,
-	evidence []types.Evidence) (sm.State, types.BlockID, *types.Commit, error) {
+	evidence []types.Evidence) (sm.State, types.BlockID, *types.ExtendedCommit) {
+	t.Helper()
+
 	// A good block passes
 	state, blockID, err := makeAndApplyGoodBlock(state, height, lastCommit, proposerAddr, blockExec, evidence)
-	if err != nil {
-		return state, types.BlockID{}, nil, err
-	}
+	require.NoError(t, err)
 
 	// Simulate a lastCommit for this block from all validators for the next height
-	commit, err := makeValidCommit(height, blockID, state.Validators, privVals)
-	if err != nil {
-		return state, types.BlockID{}, nil, err
-	}
-	return state, blockID, commit, nil
+	extCommit, _ := makeValidExtendedCommit(t, height, blockID, state.Validators, privVals)
+	require.NoError(t, err)
+
+	return state, blockID, extCommit
 }
 
 func makeAndApplyGoodBlock(state sm.State, height int64, lastCommit *types.Commit, proposerAddr []byte,
@@ -83,22 +84,29 @@ func makeBlock(state sm.State, height int64, c *types.Commit) *types.Block {
 	)
 }
 
-func makeValidCommit(
+func makeValidExtendedCommit(
+	t *testing.T,
 	height int64,
 	blockID types.BlockID,
 	vals *types.ValidatorSet,
 	privVals map[string]types.PrivValidator,
-) (*types.Commit, error) {
-	sigs := make([]types.CommitSig, 0)
+) (*types.ExtendedCommit, []*types.Vote) {
+	t.Helper()
+	sigs := make([]types.ExtendedCommitSig, vals.Size())
+	votes := make([]*types.Vote, vals.Size())
 	for i := 0; i < vals.Size(); i++ {
 		_, val := vals.GetByIndex(int32(i))
-		vote, err := types.MakeVote(height, blockID, vals, privVals[val.Address.String()], chainID, time.Now())
-		if err != nil {
-			return nil, err
-		}
-		sigs = append(sigs, vote.CommitSig())
+		vote, err := test.MakeVote(privVals[val.Address.String()], chainID, int32(i), height, 0, 2, blockID, time.Now())
+		require.NoError(t, err)
+		sigs[i] = vote.ExtendedCommitSig()
+		votes[i] = vote
 	}
-	return types.NewCommit(height, 0, blockID, sigs), nil
+
+	return &types.ExtendedCommit{
+		Height:             height,
+		BlockID:            blockID,
+		ExtendedSignatures: sigs,
+	}, votes
 }
 
 func makeState(nVals, height int) (sm.State, dbm.DB, map[string]types.PrivValidator) {

state/mocks/block_store.go

@@ -195,6 +195,11 @@ func (_m *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
 	_m.Called(block, blockParts, seenCommit)
 }
 
+// SaveBlockWithExtendedCommit provides a mock function with given fields: block, blockParts, seenExtendedCommit
+func (_m *BlockStore) SaveBlockWithExtendedCommit(block *types.Block, blockParts *types.PartSet, seenExtendedCommit *types.ExtendedCommit) {
+	_m.Called(block, blockParts, seenExtendedCommit)
+}
+
 // Size provides a mock function with given fields:
 func (_m *BlockStore) Size() int64 {
 	ret := _m.Called()

state/services.go

@@ -25,6 +25,7 @@ type BlockStore interface {
 	LoadBlock(height int64) *types.Block
 
 	SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit)
+	SaveBlockWithExtendedCommit(block *types.Block, blockParts *types.PartSet, seenExtendedCommit *types.ExtendedCommit)
 
 	PruneBlocks(height int64) (uint64, error)
 

state/validation_test.go

@@ -52,6 +52,7 @@ func TestValidateBlockHeader(t *testing.T) {
 		sm.EmptyEvidencePool{},
 	)
 	lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
+	var lastExtCommit *types.ExtendedCommit
 
 	// some bad values
 	wrongHash := tmhash.Sum([]byte("this hash is wrong"))
@@ -101,11 +102,17 @@ func TestValidateBlockHeader(t *testing.T) {
 		/*
 			A good block passes
 		*/
-		var err error
-		state, _, lastCommit, err = makeAndCommitGoodBlock(
+		state, _, lastExtCommit = makeAndCommitGoodBlock(t,
 			state, height, lastCommit, state.Validators.GetProposer().Address, blockExec, privVals, nil)
-		require.NoError(t, err, "height %d", height)
+		lastCommit = lastExtCommit.ToCommit()
 	}
+
+	nextHeight := validationTestsStopHeight
+	block := makeBlock(state, nextHeight, lastCommit)
+	state.InitialHeight = nextHeight + 1
+	err := blockExec.ValidateBlock(state, block)
+	require.Error(t, err, "expected an error when state is ahead of block")
+	assert.Contains(t, err.Error(), "lower than initial height")
 }
 
 func TestValidateBlockCommit(t *testing.T) {
@@ -137,6 +144,7 @@ func TestValidateBlockCommit(t *testing.T) {
 		sm.EmptyEvidencePool{},
 	)
 	lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
+	var lastExtCommit *types.ExtendedCommit
 	wrongSigsCommit := types.NewCommit(1, 0, types.BlockID{}, nil)
 	badPrivVal := types.NewMockPV()
 
@@ -188,7 +196,8 @@ func TestValidateBlockCommit(t *testing.T) {
 		*/
 		var err error
 		var blockID types.BlockID
-		state, blockID, lastCommit, err = makeAndCommitGoodBlock(
+		state, blockID, lastExtCommit = makeAndCommitGoodBlock(
+			t,
 			state,
 			height,
 			lastCommit,
@@ -197,7 +206,7 @@ func TestValidateBlockCommit(t *testing.T) {
 			privVals,
 			nil,
 		)
-		require.NoError(t, err, "height %d", height)
+		lastCommit = lastExtCommit.ToCommit()
 
 		/*
 			wrongSigsCommit is fine except for the extra bad precommit
@@ -276,6 +285,7 @@ func TestValidateBlockEvidence(t *testing.T) {
 		evpool,
 	)
 	lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
+	var lastExtCommit *types.ExtendedCommit
 
 	for height := int64(1); height < validationTestsStopHeight; height++ {
 		proposerAddr := state.Validators.GetProposer().Address
@@ -320,8 +330,8 @@ func TestValidateBlockEvidence(t *testing.T) {
 			evidence = append(evidence, newEv)
 		}
 
-		var err error
-		state, _, lastCommit, err = makeAndCommitGoodBlock(
+		state, _, lastExtCommit = makeAndCommitGoodBlock(
+			t,
 			state,
 			height,
 			lastCommit,
@@ -330,6 +340,6 @@ func TestValidateBlockEvidence(t *testing.T) {
 			privVals,
 			evidence,
 		)
-		require.NoError(t, err, "height %d", height)
+		lastCommit = lastExtCommit.ToCommit()
 	}
 }