mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-04-19 15:25:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

store: proto migration (#4974)

Browse Source 
## Description

migrate store to protobuf

Closes: #XXX
This commit is contained in:
Marko
2020-06-08 08:43:29 +02:00
committed by GitHub
parent 6cfbe1eb5c
commit f9f4916bc3
11 changed files with 225 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG_PENDING.md
View File

@@ -35,6 +35,8 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
- [state] \#4679 `TxResult` is a Protobuf type defined in `abci` types directory
- [state] \#4679 `state` reactor migration to Protobuf encoding
- [evidence] \#4959 Add json tags to `DuplicateVoteEvidence`
- [store] \#4778 Transition store module to protobuf encoding
- `BlockStoreStateJSON` is now `BlockStoreState` and is encoded as binary in the database
- Apps

26
consensus/replay_test.go
View File

@@ -12,6 +12,7 @@ import (
"testing"
"time"
"github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -1014,11 +1015,20 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
case EndHeightMessage:
// if its not the first one, we have a full block
if thisBlockParts != nil {
var block = new(types.Block)
_, err = cdc.UnmarshalBinaryLengthPrefixedReader(thisBlockParts.GetReader(), block, 0)
var pbb = new(tmproto.Block)
bz, err := ioutil.ReadAll(thisBlockParts.GetReader())
if err != nil {
panic(err)
}
err = proto.Unmarshal(bz, pbb)
if err != nil {
panic(err)
}
block, err := types.BlockFromProto(pbb)
if err != nil {
panic(err)
}
if block.Height != height+1 {
panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1))
}
@@ -1045,8 +1055,16 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
}
}
// grab the last block too
var block = new(types.Block)
_, err = cdc.UnmarshalBinaryLengthPrefixedReader(thisBlockParts.GetReader(), block, 0)
bz, err := ioutil.ReadAll(thisBlockParts.GetReader())
if err != nil {
panic(err)
}
var pbb = new(tmproto.Block)
err = proto.Unmarshal(bz, pbb)
if err != nil {
panic(err)
}
block, err := types.BlockFromProto(pbb)
if err != nil {
panic(err)
}

32
consensus/state.go
View File

@@ -4,26 +4,28 @@ import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"reflect"
"runtime/debug"
"sync"
"time"
"github.com/tendermint/tendermint/libs/fail"
"github.com/tendermint/tendermint/libs/log"
tmos "github.com/tendermint/tendermint/libs/os"
"github.com/tendermint/tendermint/libs/service"
tmproto "github.com/tendermint/tendermint/proto/types"
tmtime "github.com/tendermint/tendermint/types/time"
"github.com/gogo/protobuf/proto"
cfg "github.com/tendermint/tendermint/config"
cstypes "github.com/tendermint/tendermint/consensus/types"
tmevents "github.com/tendermint/tendermint/libs/events"
"github.com/tendermint/tendermint/libs/fail"
"github.com/tendermint/tendermint/libs/log"
tmmath "github.com/tendermint/tendermint/libs/math"
tmos "github.com/tendermint/tendermint/libs/os"
"github.com/tendermint/tendermint/libs/service"
"github.com/tendermint/tendermint/p2p"
tmproto "github.com/tendermint/tendermint/proto/types"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
tmtime "github.com/tendermint/tendermint/types/time"
)
//-----------------------------------------------------------------------------
@@ -1707,15 +1709,21 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.ID) (add
return added, err
}
if added && cs.ProposalBlockParts.IsComplete() {
// Added and completed!
_, err = cdc.UnmarshalBinaryLengthPrefixedReader(
cs.ProposalBlockParts.GetReader(),
&cs.ProposalBlock,
cs.state.ConsensusParams.Block.MaxBytes,
)
bz, err := ioutil.ReadAll(cs.ProposalBlockParts.GetReader())
if err != nil {
return added, err
}
var pbb = new(tmproto.Block)
err = proto.Unmarshal(bz, pbb)
if err != nil {
return added, err
}
block, err := types.BlockFromProto(pbb)
if err != nil {
return added, err
}
cs.ProposalBlock = block
// NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal
cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash())
cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent())

16
evidence/pool_test.go
View File

@@ -10,6 +10,8 @@ import (
dbm "github.com/tendermint/tm-db"
"github.com/tendermint/tendermint/crypto"
tmrand "github.com/tendermint/tendermint/libs/rand"
tmproto "github.com/tendermint/tendermint/proto/types"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/store"
@@ -26,7 +28,7 @@ func TestMain(m *testing.M) {
func TestEvidencePool(t *testing.T) {
var (
valAddr = []byte("val1")
valAddr = tmrand.Bytes(crypto.AddressSize)
height = int64(52)
stateDB = initializeValidatorState(valAddr, height)
evidenceDB = dbm.NewMemDB()
@@ -75,7 +77,7 @@ func TestEvidencePool(t *testing.T) {
func TestProposingAndCommittingEvidence(t *testing.T) {
var (
valAddr = []byte("validator_address")
valAddr = tmrand.Bytes(crypto.AddressSize)
height = int64(1)
lastBlockTime = time.Now()
stateDB = initializeValidatorState(valAddr, height)
@@ -111,7 +113,7 @@ func TestProposingAndCommittingEvidence(t *testing.T) {
func TestAddEvidence(t *testing.T) {
var (
valAddr = []byte("val1")
valAddr = tmrand.Bytes(crypto.AddressSize)
height = int64(30)
stateDB = initializeValidatorState(valAddr, height)
evidenceDB = dbm.NewMemDB()
@@ -151,7 +153,7 @@ func TestAddEvidence(t *testing.T) {
func TestEvidencePoolUpdate(t *testing.T) {
var (
valAddr = []byte("validator_address")
valAddr = tmrand.Bytes(crypto.AddressSize)
height = int64(21)
stateDB = initializeValidatorState(valAddr, height)
evidenceDB = dbm.NewMemDB()
@@ -186,7 +188,7 @@ func TestEvidencePoolUpdate(t *testing.T) {
func TestEvidencePoolNewPool(t *testing.T) {
var (
valAddr = []byte("validator_address")
valAddr = tmrand.Bytes(crypto.AddressSize)
height = int64(1)
stateDB = initializeValidatorState(valAddr, height)
evidenceDB = dbm.NewMemDB()
@@ -204,7 +206,7 @@ func TestEvidencePoolNewPool(t *testing.T) {
func TestAddingAndPruningPOLC(t *testing.T) {
var (
valAddr = []byte("validator_address")
valAddr = tmrand.Bytes(crypto.AddressSize)
stateDB = initializeValidatorState(valAddr, 1)
evidenceDB = dbm.NewMemDB()
blockStoreDB = dbm.NewMemDB()
@@ -252,7 +254,7 @@ func TestAddingAndPruningPOLC(t *testing.T) {
func TestRecoverPendingEvidence(t *testing.T) {
var (
valAddr = []byte("val1")
valAddr = tmrand.Bytes(crypto.AddressSize)
height = int64(30)
stateDB = initializeValidatorState(valAddr, height)
evidenceDB = dbm.NewMemDB()

6
evidence/reactor_test.go
View File

@@ -13,7 +13,9 @@ import (
dbm "github.com/tendermint/tm-db"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/libs/log"
tmrand "github.com/tendermint/tendermint/libs/rand"
"github.com/tendermint/tendermint/p2p"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
@@ -134,7 +136,7 @@ func TestReactorBroadcastEvidence(t *testing.T) {
// create statedb for everyone
stateDBs := make([]dbm.DB, N)
valAddr := []byte("myval")
valAddr := tmrand.Bytes(crypto.AddressSize)
// we need validators saved for heights at least as high as we have evidence for
height := int64(numEvidence) + 10
for i := 0; i < N; i++ {
@@ -169,7 +171,7 @@ func (ps peerState) GetHeight() int64 {
func TestReactorSelectiveBroadcast(t *testing.T) {
config := cfg.TestConfig()
valAddr := []byte("myval")
valAddr := tmrand.Bytes(crypto.AddressSize)
height1 := int64(numEvidence) + 10
height2 := int64(numEvidence) / 2

13
store/codec.go
View File

@@ -1,13 +0,0 @@
package store
import (
amino "github.com/tendermint/go-amino"
"github.com/tendermint/tendermint/types"
)
var cdc = amino.NewCodec()
func init() {
types.RegisterBlockAmino(cdc)
}

131
store/store.go
View File

@@ -5,8 +5,11 @@ import (
"strconv"
"sync"
"github.com/gogo/protobuf/proto"
dbm "github.com/tendermint/tm-db"
tmstore "github.com/tendermint/tendermint/proto/store"
tmproto "github.com/tendermint/tendermint/proto/types"
"github.com/tendermint/tendermint/types"
)
@@ -38,10 +41,10 @@ type BlockStore struct {
// NewBlockStore returns a new BlockStore with the given DB,
// initialized to the last height that was committed to the DB.
func NewBlockStore(db dbm.DB) *BlockStore {
bsjson := LoadBlockStoreStateJSON(db)
bs := LoadBlockStoreState(db)
return &BlockStore{
base: bsjson.Base,
height: bsjson.Height,
base: bs.Base,
height: bs.Height,
db: db,
}
}
@@ -78,18 +81,24 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block {
return nil
}
var block = new(types.Block)
pbb := new(tmproto.Block)
buf := []byte{}
for i := 0; i < int(blockMeta.BlockID.PartsHeader.Total); i++ {
part := bs.LoadBlockPart(height, i)
buf = append(buf, part.Bytes...)
}
err := cdc.UnmarshalBinaryLengthPrefixed(buf, block)
err := proto.Unmarshal(buf, pbb)
if err != nil {
// NOTE: The existence of meta should imply the existence of the
// block. So, make sure meta is only saved after blocks are saved.
panic(fmt.Sprintf("Error reading block: %v", err))
}
block, err := types.BlockFromProto(pbb)
if err != nil {
panic(fmt.Errorf("error from proto block: %w", err))
}
return block
}
@@ -118,7 +127,8 @@ func (bs *BlockStore) LoadBlockByHash(hash []byte) *types.Block {
// from the block at the given height.
// If no part is found for the given height and index, it returns nil.
func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part {
var part = new(types.Part)
var pbpart = new(tmproto.Part)
bz, err := bs.db.Get(calcBlockPartKey(height, index))
if err != nil {
panic(err)
@@ -126,28 +136,43 @@ func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part {
if len(bz) == 0 {
return nil
}
err = cdc.UnmarshalBinaryBare(bz, part)
err = proto.Unmarshal(bz, pbpart)
if err != nil {
panic(fmt.Errorf("unmarshal to tmproto.Part failed: %w", err))
}
part, err := types.PartFromProto(pbpart)
if err != nil {
panic(fmt.Sprintf("Error reading block part: %v", err))
}
return part
}
// LoadBlockMeta returns the BlockMeta for the given height.
// If no block is found for the given height, it returns nil.
func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
var blockMeta = new(types.BlockMeta)
var pbbm = new(tmproto.BlockMeta)
bz, err := bs.db.Get(calcBlockMetaKey(height))
if err != nil {
panic(err)
}
if len(bz) == 0 {
return nil
}
err = cdc.UnmarshalBinaryBare(bz, blockMeta)
err = proto.Unmarshal(bz, pbbm)
if err != nil {
panic(fmt.Sprintf("Error reading block meta: %v", err))
panic(fmt.Errorf("unmarshal to tmproto.BlockMeta: %w", err))
}
blockMeta, err := types.BlockMetaFromProto(pbbm)
if err != nil {
panic(fmt.Errorf("error from proto blockMeta: %w", err))
}
return blockMeta
}
@@ -156,7 +181,7 @@ func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
// and it comes from the block.LastCommit for `height+1`.
// If no commit is found for the given height, it returns nil.
func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
var commit = new(types.Commit)
var pbc = new(tmproto.Commit)
bz, err := bs.db.Get(calcBlockCommitKey(height))
if err != nil {
panic(err)
@@ -164,7 +189,11 @@ func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
if len(bz) == 0 {
return nil
}
err = cdc.UnmarshalBinaryBare(bz, commit)
err = proto.Unmarshal(bz, pbc)
if err != nil {
panic(fmt.Errorf("error reading block commit: %w", err))
}
commit, err := types.CommitFromProto(pbc)
if err != nil {
panic(fmt.Sprintf("Error reading block commit: %v", err))
}
@@ -175,7 +204,7 @@ func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
// This is useful when we've seen a commit, but there has not yet been
// a new block at `height + 1` that includes this commit in its block.LastCommit.
func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit {
var commit = new(types.Commit)
var pbc = new(tmproto.Commit)
bz, err := bs.db.Get(calcSeenCommitKey(height))
if err != nil {
panic(err)
@@ -183,9 +212,14 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit {
if len(bz) == 0 {
return nil
}
err = cdc.UnmarshalBinaryBare(bz, commit)
err = proto.Unmarshal(bz, pbc)
if err != nil {
panic(fmt.Sprintf("Error reading block seen commit: %v", err))
panic(fmt.Sprintf("error reading block seen commit: %v", err))
}
commit, err := types.CommitFromProto(pbc)
if err != nil {
panic(fmt.Errorf("error from proto commit: %w", err))
}
return commit
}
@@ -281,7 +315,11 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
// Save block meta
blockMeta := types.NewBlockMeta(block, blockParts)
metaBytes := cdc.MustMarshalBinaryBare(blockMeta)
pbm := blockMeta.ToProto()
if pbm == nil {
panic("nil blockmeta")
}
metaBytes := mustEncode(pbm)
bs.db.Set(calcBlockMetaKey(height), metaBytes)
bs.db.Set(calcBlockHashKey(hash), []byte(fmt.Sprintf("%d", height)))
@@ -292,12 +330,14 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
}
// Save block commit (duplicate and separate from the Block)
blockCommitBytes := cdc.MustMarshalBinaryBare(block.LastCommit)
pbc := block.LastCommit.ToProto()
blockCommitBytes := mustEncode(pbc)
bs.db.Set(calcBlockCommitKey(height-1), blockCommitBytes)
// Save seen commit (seen +2/3 precommits for block)
// NOTE: we can delete this at a later height
seenCommitBytes := cdc.MustMarshalBinaryBare(seenCommit)
pbsc := seenCommit.ToProto()
seenCommitBytes := mustEncode(pbsc)
bs.db.Set(calcSeenCommitKey(height), seenCommitBytes)
// Done!
@@ -308,7 +348,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
}
bs.mtx.Unlock()
// Save new BlockStoreStateJSON descriptor
// Save new BlockStoreState descriptor
bs.saveState()
// Flush
@@ -316,23 +356,31 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
}
func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) {
partBytes := cdc.MustMarshalBinaryBare(part)
pbp, err := part.ToProto()
if err != nil {
panic(fmt.Errorf("unable to make part into proto: %w", err))
}
partBytes := mustEncode(pbp)
bs.db.Set(calcBlockPartKey(height, index), partBytes)
}
func (bs *BlockStore) saveState() {
bs.mtx.RLock()
bsJSON := BlockStoreStateJSON{
bss := tmstore.BlockStoreState{
Base: bs.base,
Height: bs.height,
}
bs.mtx.RUnlock()
bsJSON.Save(bs.db)
SaveBlockStoreState(&bss, bs.db)
}
// SaveSeenCommit saves a seen commit, used by e.g. the state sync reactor when bootstrapping node.
func (bs *BlockStore) SaveSeenCommit(height int64, seenCommit *types.Commit) error {
seenCommitBytes := cdc.MustMarshalBinaryBare(seenCommit)
pbc := seenCommit.ToProto()
seenCommitBytes, err := proto.Marshal(pbc)
if err != nil {
return fmt.Errorf("unable to marshal commit: %w", err)
}
return bs.db.Set(calcSeenCommitKey(height), seenCommitBytes)
}
@@ -362,42 +410,47 @@ func calcBlockHashKey(hash []byte) []byte {
var blockStoreKey = []byte("blockStore")
// BlockStoreStateJSON is the block store state JSON structure.
type BlockStoreStateJSON struct {
Base int64 `json:"base"`
Height int64 `json:"height"`
}
// Save persists the blockStore state to the database as JSON.
func (bsj BlockStoreStateJSON) Save(db dbm.DB) {
bytes, err := cdc.MarshalJSON(bsj)
// SaveBlockStoreState persists the blockStore state to the database.
func SaveBlockStoreState(bsj *tmstore.BlockStoreState, db dbm.DB) {
bytes, err := proto.Marshal(bsj)
if err != nil {
panic(fmt.Sprintf("Could not marshal state bytes: %v", err))
}
db.SetSync(blockStoreKey, bytes)
}
// LoadBlockStoreStateJSON returns the BlockStoreStateJSON as loaded from disk.
// If no BlockStoreStateJSON was previously persisted, it returns the zero value.
func LoadBlockStoreStateJSON(db dbm.DB) BlockStoreStateJSON {
// LoadBlockStoreState returns the BlockStoreState as loaded from disk.
// If no BlockStoreState was previously persisted, it returns the zero value.
func LoadBlockStoreState(db dbm.DB) tmstore.BlockStoreState {
bytes, err := db.Get(blockStoreKey)
if err != nil {
panic(err)
}
if len(bytes) == 0 {
return BlockStoreStateJSON{
return tmstore.BlockStoreState{
Base: 0,
Height: 0,
}
}
bsj := BlockStoreStateJSON{}
err = cdc.UnmarshalJSON(bytes, &bsj)
if err != nil {
var bsj tmstore.BlockStoreState
if err := proto.Unmarshal(bytes, &bsj); err != nil {
panic(fmt.Sprintf("Could not unmarshal bytes: %X", bytes))
}
// Backwards compatibility with persisted data from before Base existed.
if bsj.Height > 0 && bsj.Base == 0 {
bsj.Base = 1
}
return bsj
}
//mustEncode proto encodes a proto.message and panics if fails
func mustEncode(pb proto.Message) []byte {
bz, err := proto.Marshal(pb)
if err != nil {
panic(fmt.Errorf("unable to marshal: %w", err))
}
return bz
}

108
store/store_test.go
View File

@@ -9,14 +9,17 @@ import (
"testing"
"time"
"github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
dbm "github.com/tendermint/tm-db"
cfg "github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/libs/log"
tmrand "github.com/tendermint/tendermint/libs/rand"
tmstore "github.com/tendermint/tendermint/proto/store"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
tmtime "github.com/tendermint/tendermint/types/time"
)
@@ -29,11 +32,12 @@ type cleanupFunc func()
func makeTestCommit(height int64, timestamp time.Time) *types.Commit {
commitSigs := []types.CommitSig{{
BlockIDFlag: types.BlockIDFlagCommit,
ValidatorAddress: []byte("ValidatorAddress"),
ValidatorAddress: tmrand.Bytes(crypto.AddressSize),
Timestamp: timestamp,
Signature: []byte("Signature"),
}}
return types.NewCommit(height, 0, types.BlockID{}, commitSigs)
return types.NewCommit(height, 0,
types.BlockID{Hash: []byte(""), PartsHeader: types.PartSetHeader{Hash: []byte(""), Total: 2}}, commitSigs)
}
func makeTxs(height int64) (txs []types.Tx) {
@@ -61,38 +65,34 @@ func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore, cleanupFu
return state, NewBlockStore(blockDB), func() { os.RemoveAll(config.RootDir) }
}
func TestLoadBlockStoreStateJSON(t *testing.T) {
db := dbm.NewMemDB()
bsj := &BlockStoreStateJSON{Base: 100, Height: 1000}
bsj.Save(db)
func TestLoadBlockStoreState(t *testing.T) {
retrBSJ := LoadBlockStoreStateJSON(db)
assert.Equal(t, *bsj, retrBSJ, "expected the retrieved DBs to match")
}
type blockStoreTest struct {
testName string
bss *tmstore.BlockStoreState
want tmstore.BlockStoreState
}
func TestLoadBlockStoreStateJSON_Empty(t *testing.T) {
db := dbm.NewMemDB()
testCases := []blockStoreTest{
{"success", &tmstore.BlockStoreState{Base: 100, Height: 1000},
tmstore.BlockStoreState{Base: 100, Height: 1000}},
{"empty", &tmstore.BlockStoreState{}, tmstore.BlockStoreState{}},
{"no base", &tmstore.BlockStoreState{Height: 1000}, tmstore.BlockStoreState{Base: 1, Height: 1000}},
}
bsj := &BlockStoreStateJSON{}
bsj.Save(db)
retrBSJ := LoadBlockStoreStateJSON(db)
assert.Equal(t, BlockStoreStateJSON{}, retrBSJ, "expected the retrieved DBs to match")
}
func TestLoadBlockStoreStateJSON_NoBase(t *testing.T) {
db := dbm.NewMemDB()
bsj := &BlockStoreStateJSON{Height: 1000}
bsj.Save(db)
retrBSJ := LoadBlockStoreStateJSON(db)
assert.Equal(t, BlockStoreStateJSON{Base: 1, Height: 1000}, retrBSJ, "expected the retrieved DBs to match")
for _, tc := range testCases {
db := dbm.NewMemDB()
SaveBlockStoreState(tc.bss, db)
retrBSJ := LoadBlockStoreState(db)
assert.Equal(t, tc.want, retrBSJ, "expected the retrieved DBs to match: %s", tc.testName)
}
}
func TestNewBlockStore(t *testing.T) {
db := dbm.NewMemDB()
err := db.Set(blockStoreKey, []byte(`{"base": "100", "height": "10000"}`))
bss := tmstore.BlockStoreState{Base: 100, Height: 10000}
bz, _ := proto.Marshal(&bss)
err := db.Set(blockStoreKey, bz)
require.NoError(t, err)
bs := NewBlockStore(db)
require.Equal(t, int64(100), bs.Base(), "failed to properly parse blockstore")
@@ -181,9 +181,10 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
uncontiguousPartSet.AddPart(part2)
header1 := types.Header{
Height: 1,
ChainID: "block_test",
Time: tmtime.Now(),
Height: 1,
ChainID: "block_test",
Time: tmtime.Now(),
ProposerAddress: tmrand.Bytes(crypto.AddressSize),
}
// End of setup, test data
@@ -215,7 +216,11 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
{
block: newBlock( // New block at height 5 in empty block store is fine
types.Header{Height: 5, ChainID: "block_test", Time: tmtime.Now()},
types.Header{
Height: 5,
ChainID: "block_test",
Time: tmtime.Now(),
ProposerAddress: tmrand.Bytes(crypto.AddressSize)},
makeTestCommit(5, tmtime.Now()),
),
parts: validPartSet,
@@ -233,14 +238,14 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
parts: validPartSet,
seenCommit: seenCommit1,
corruptCommitInDB: true, // Corrupt the DB's commit entry
wantPanic: "unmarshal to types.Commit failed",
wantPanic: "error reading block commit",
},
{
block: newBlock(header1, commitAtH10),
parts: validPartSet,
seenCommit: seenCommit1,
wantPanic: "unmarshal to types.BlockMeta failed",
wantPanic: "unmarshal to tmproto.BlockMeta",
corruptBlockInDB: true, // Corrupt the DB's block entry
},
@@ -259,7 +264,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
seenCommit: seenCommit1,
corruptSeenCommitInDB: true,
wantPanic: "unmarshal to types.Commit failed",
wantPanic: "error reading block seen commit",
},
{
@@ -372,10 +377,12 @@ func TestLoadBlockPart(t *testing.T) {
require.NoError(t, err)
res, _, panicErr = doFn(loadPart)
require.NotNil(t, panicErr, "expecting a non-nil panic")
require.Contains(t, panicErr.Error(), "unmarshal to types.Part failed")
require.Contains(t, panicErr.Error(), "unmarshal to tmproto.Part failed")
// 3. A good block serialized and saved to the DB should be retrievable
err = db.Set(calcBlockPartKey(height, index), cdc.MustMarshalBinaryBare(part1))
pb1, err := part1.ToProto()
require.NoError(t, err)
err = db.Set(calcBlockPartKey(height, index), mustEncode(pb1))
require.NoError(t, err)
gotPart, _, panicErr := doFn(loadPart)
require.Nil(t, panicErr, "an existent and proper block should not panic")
@@ -423,10 +430,10 @@ func TestPruneBlocks(t *testing.T) {
assert.EqualValues(t, 1200, bs.Base())
assert.EqualValues(t, 1500, bs.Height())
assert.EqualValues(t, 301, bs.Size())
assert.EqualValues(t, BlockStoreStateJSON{
assert.EqualValues(t, tmstore.BlockStoreState{
Base: 1200,
Height: 1500,
}, LoadBlockStoreStateJSON(db))
}, LoadBlockStoreState(db))
require.NotNil(t, bs.LoadBlock(1200))
require.Nil(t, bs.LoadBlock(1199))
@@ -489,17 +496,22 @@ func TestLoadBlockMeta(t *testing.T) {
require.NoError(t, err)
res, _, panicErr = doFn(loadMeta)
require.NotNil(t, panicErr, "expecting a non-nil panic")
require.Contains(t, panicErr.Error(), "unmarshal to types.BlockMeta")
require.Contains(t, panicErr.Error(), "unmarshal to tmproto.BlockMeta")
// 3. A good blockMeta serialized and saved to the DB should be retrievable
meta := &types.BlockMeta{}
err = db.Set(calcBlockMetaKey(height), cdc.MustMarshalBinaryBare(meta))
meta := &types.BlockMeta{Header: types.Header{Height: 1, ProposerAddress: tmrand.Bytes(crypto.AddressSize)}}
pbm := meta.ToProto()
err = db.Set(calcBlockMetaKey(height), mustEncode(pbm))
require.NoError(t, err)
gotMeta, _, panicErr := doFn(loadMeta)
require.Nil(t, panicErr, "an existent and proper block should not panic")
require.Nil(t, res, "a properly saved blockMeta should return a proper blocMeta ")
require.Equal(t, cdc.MustMarshalBinaryBare(meta), cdc.MustMarshalBinaryBare(gotMeta),
"expecting successful retrieval of previously saved blockMeta")
pbmeta := meta.ToProto()
if gmeta, ok := gotMeta.(*types.BlockMeta); ok {
pbgotMeta := gmeta.ToProto()
require.Equal(t, mustEncode(pbmeta), mustEncode(pbgotMeta),
"expecting successful retrieval of previously saved blockMeta")
}
}
func TestBlockFetchAtHeight(t *testing.T) {
@@ -514,8 +526,12 @@ func TestBlockFetchAtHeight(t *testing.T) {
require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed")
blockAtHeight := bs.LoadBlock(bs.Height())
bz1 := cdc.MustMarshalBinaryBare(block)
bz2 := cdc.MustMarshalBinaryBare(blockAtHeight)
b1, err := block.ToProto()
require.NoError(t, err)
b2, err := blockAtHeight.ToProto()
require.NoError(t, err)
bz1 := mustEncode(b1)
bz2 := mustEncode(b2)
require.Equal(t, bz1, bz2)
require.Equal(t, block.Hash(), blockAtHeight.Hash(),
"expecting a successful load of the last saved block")

12
types/block.go
View File

@@ -8,6 +8,8 @@ import (
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/crypto/merkle"
"github.com/tendermint/tendermint/crypto/tmhash"
@@ -140,9 +142,11 @@ func (b *Block) MakePartSet(partSize uint32) *PartSet {
b.mtx.Lock()
defer b.mtx.Unlock()
// We prefix the byte length, so that unmarshaling
// can easily happen via a reader.
bz, err := cdc.MarshalBinaryLengthPrefixed(b)
pbb, err := b.ToProto()
if err != nil {
panic(err)
}
bz, err := proto.Marshal(pbb)
if err != nil {
panic(err)
}
@@ -1121,7 +1125,7 @@ func (data *Data) ToProto() tmproto.Data {
return *tp
}
// DataFromProto takes a protobud representation of Data &
// DataFromProto takes a protobuf representation of Data &
// returns the native type.
func DataFromProto(dp *tmproto.Data) (Data, error) {
if dp == nil {

1
types/block_test.go
View File

@@ -660,6 +660,7 @@ func TestBlockProtoBuf(t *testing.T) {
} else {
require.Error(t, err, tc.msg)
}
block, err := BlockFromProto(pb)
if tc.expPass2 {
require.NoError(t, err, tc.msg)

10
types/evidence.go
View File

@@ -262,6 +262,7 @@ func EvidenceFromProto(evidence *tmproto.Evidence) (Evidence, error) {
return tp, tp.ValidateBasic()
case *tmproto.Evidence_LunaticValidatorEvidence:
h, err := HeaderFromProto(evi.LunaticValidatorEvidence.GetHeader())
if err != nil {
return nil, err
@@ -526,8 +527,6 @@ type ConflictingHeadersEvidence struct {
H2 *SignedHeader `json:"h_2"`
}
var _ Evidence = &ConflictingHeadersEvidence{}
var _ CompositeEvidence = &ConflictingHeadersEvidence{}
var _ Evidence = ConflictingHeadersEvidence{}
var _ CompositeEvidence = ConflictingHeadersEvidence{}
@@ -778,7 +777,6 @@ type PhantomValidatorEvidence struct {
LastHeightValidatorWasInSet int64 `json:"last_height_validator_was_in_set"`
}
var _ Evidence = &PhantomValidatorEvidence{}
var _ Evidence = PhantomValidatorEvidence{}
func (e PhantomValidatorEvidence) Height() int64 {
@@ -858,7 +856,6 @@ type LunaticValidatorEvidence struct {
InvalidHeaderField string `json:"invalid_header_field"`
}
var _ Evidence = &LunaticValidatorEvidence{}
var _ Evidence = LunaticValidatorEvidence{}
func (e LunaticValidatorEvidence) Height() int64 {
@@ -959,6 +956,10 @@ func (e LunaticValidatorEvidence) VerifyHeader(committedHeader *Header) error {
return fmt.Errorf("%s matches committed hash", field)
}
if committedHeader == nil {
return errors.New("committed header is nil")
}
switch e.InvalidHeaderField {
case ValidatorsHashField:
if bytes.Equal(committedHeader.ValidatorsHash, e.Header.ValidatorsHash) {
@@ -994,7 +995,6 @@ type PotentialAmnesiaEvidence struct {
VoteB *Vote `json:"vote_b"`
}
var _ Evidence = &PotentialAmnesiaEvidence{}
var _ Evidence = PotentialAmnesiaEvidence{}
func (e PotentialAmnesiaEvidence) Height() int64 {