mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-09-19 06:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

store: order-preserving varint key encoding (#5771)

Browse Source
This commit is contained in:
Callum Waters
2021-01-05 16:53:26 +01:00
committed by GitHub
parent 0555772d3a
commit 9b9222f461
9 changed files with 195 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
store/store.go
+58 -27
View File
@@ -5,6 +5,7 @@ import (
"strconv"
"github.com/gogo/protobuf/proto"
"github.com/google/orderedcode"
dbm "github.com/tendermint/tm-db"
tmsync "github.com/tendermint/tendermint/libs/sync"
@@ -126,7 +127,7 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block {
// If no block is found for that hash, it returns nil.
// Panics if it fails to parse height associated with the given hash.
func (bs *BlockStore) LoadBlockByHash(hash []byte) *types.Block {
bz, err := bs.db.Get(calcBlockHashKey(hash))
bz, err := bs.db.Get(blockHashKey(hash))
if err != nil {
panic(err)
}
@@ -149,7 +150,7 @@ func (bs *BlockStore) LoadBlockByHash(hash []byte) *types.Block {
func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part {
var pbpart = new(tmproto.Part)
bz, err := bs.db.Get(calcBlockPartKey(height, index))
bz, err := bs.db.Get(blockPartKey(height, index))
if err != nil {
panic(err)
}
@@ -173,7 +174,7 @@ func (bs *BlockStore) LoadBlockPart(height int64, index int) *types.Part {
// If no block is found for the given height, it returns nil.
func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
var pbbm = new(tmproto.BlockMeta)
bz, err := bs.db.Get(calcBlockMetaKey(height))
bz, err := bs.db.Get(blockMetaKey(height))
if err != nil {
panic(err)
@@ -202,7 +203,7 @@ func (bs *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
// If no commit is found for the given height, it returns nil.
func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
var pbc = new(tmproto.Commit)
bz, err := bs.db.Get(calcBlockCommitKey(height))
bz, err := bs.db.Get(blockCommitKey(height))
if err != nil {
panic(err)
}
@@ -225,7 +226,7 @@ func (bs *BlockStore) LoadBlockCommit(height int64) *types.Commit {
// a new block at `height + 1` that includes this commit in its block.LastCommit.
func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit {
var pbc = new(tmproto.Commit)
bz, err := bs.db.Get(calcSeenCommitKey(height))
bz, err := bs.db.Get(seenCommitKey(height))
if err != nil {
panic(err)
}
@@ -285,20 +286,20 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) {
if meta == nil { // assume already deleted
continue
}
if err := batch.Delete(calcBlockMetaKey(h)); err != nil {
if err := batch.Delete(blockMetaKey(h)); err != nil {
return 0, err
}
if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil {
if err := batch.Delete(blockHashKey(meta.BlockID.Hash)); err != nil {
return 0, err
}
if err := batch.Delete(calcBlockCommitKey(h)); err != nil {
if err := batch.Delete(blockCommitKey(h)); err != nil {
return 0, err
}
if err := batch.Delete(calcSeenCommitKey(h)); err != nil {
if err := batch.Delete(seenCommitKey(h)); err != nil {
return 0, err
}
for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ {
if err := batch.Delete(calcBlockPartKey(h, p)); err != nil {
if err := batch.Delete(blockPartKey(h, p)); err != nil {
return 0, err
}
}
@@ -359,17 +360,17 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
panic("nil blockmeta")
}
metaBytes := mustEncode(pbm)
if err := bs.db.Set(calcBlockMetaKey(height), metaBytes); err != nil {
if err := bs.db.Set(blockMetaKey(height), metaBytes); err != nil {
panic(err)
}
if err := bs.db.Set(calcBlockHashKey(hash), []byte(fmt.Sprintf("%d", height))); err != nil {
if err := bs.db.Set(blockHashKey(hash), []byte(fmt.Sprintf("%d", height))); err != nil {
panic(err)
}
// Save block commit (duplicate and separate from the Block)
pbc := block.LastCommit.ToProto()
blockCommitBytes := mustEncode(pbc)
if err := bs.db.Set(calcBlockCommitKey(height-1), blockCommitBytes); err != nil {
if err := bs.db.Set(blockCommitKey(height-1), blockCommitBytes); err != nil {
panic(err)
}
@@ -377,7 +378,7 @@ func (bs *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, s
// NOTE: we can delete this at a later height
pbsc := seenCommit.ToProto()
seenCommitBytes := mustEncode(pbsc)
if err := bs.db.Set(calcSeenCommitKey(height), seenCommitBytes); err != nil {
if err := bs.db.Set(seenCommitKey(height), seenCommitBytes); err != nil {
panic(err)
}
@@ -399,7 +400,7 @@ func (bs *BlockStore) saveBlockPart(height int64, index int, part *types.Part) {
panic(fmt.Errorf("unable to make part into proto: %w", err))
}
partBytes := mustEncode(pbp)
if err := bs.db.Set(calcBlockPartKey(height, index), partBytes); err != nil {
if err := bs.db.Set(blockPartKey(height, index), partBytes); err != nil {
panic(err)
}
}
@@ -421,29 +422,59 @@ func (bs *BlockStore) SaveSeenCommit(height int64, seenCommit *types.Commit) err
if err != nil {
return fmt.Errorf("unable to marshal commit: %w", err)
}
return bs.db.Set(calcSeenCommitKey(height), seenCommitBytes)
return bs.db.Set(seenCommitKey(height), seenCommitBytes)
}
//-----------------------------------------------------------------------------
//---------------------------------- KEY ENCODING -----------------------------------------
func calcBlockMetaKey(height int64) []byte {
return []byte(fmt.Sprintf("H:%v", height))
// key prefixes
const (
// prefixes are unique across all tm db's
prefixBlockMeta = int64(0)
prefixBlockPart = int64(1)
prefixBlockCommit = int64(2)
prefixSeenCommit = int64(3)
prefixBlockHash = int64(4)
)
func blockMetaKey(height int64) []byte {
key, err := orderedcode.Append(nil, prefixBlockMeta, height)
if err != nil {
panic(err)
}
return key
}
func calcBlockPartKey(height int64, partIndex int) []byte {
return []byte(fmt.Sprintf("P:%v:%v", height, partIndex))
func blockPartKey(height int64, partIndex int) []byte {
key, err := orderedcode.Append(nil, prefixBlockPart, height, int64(partIndex))
if err != nil {
panic(err)
}
return key
}
func calcBlockCommitKey(height int64) []byte {
return []byte(fmt.Sprintf("C:%v", height))
func blockCommitKey(height int64) []byte {
key, err := orderedcode.Append(nil, prefixBlockCommit, height)
if err != nil {
panic(err)
}
return key
}
func calcSeenCommitKey(height int64) []byte {
return []byte(fmt.Sprintf("SC:%v", height))
func seenCommitKey(height int64) []byte {
key, err := orderedcode.Append(nil, prefixSeenCommit, height)
if err != nil {
panic(err)
}
return key
}
func calcBlockHashKey(hash []byte) []byte {
return []byte(fmt.Sprintf("BH:%x", hash))
func blockHashKey(hash []byte) []byte {
key, err := orderedcode.Append(nil, prefixBlockHash, string(hash))
if err != nil {
panic(err)
}
return key
}
//-----------------------------------------------------------------------------
store/store_test.go
+9 -11
View File
@@ -156,7 +156,6 @@ func TestMain(m *testing.M) {
}
// TODO: This test should be simplified ...
func TestBlockStoreSaveLoadBlock(t *testing.T) {
state, bs, cleanup := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer)))
defer cleanup()
@@ -193,7 +192,6 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
}
// End of setup, test data
commitAtH10 := makeTestCommit(10, tmtime.Now())
tuples := []struct {
block *types.Block
@@ -302,29 +300,29 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) {
}
if tuple.corruptBlockInDB {
err := db.Set(calcBlockMetaKey(tuple.block.Height), []byte("block-bogus"))
err := db.Set(blockMetaKey(tuple.block.Height), []byte("block-bogus"))
require.NoError(t, err)
}
bBlock := bs.LoadBlock(tuple.block.Height)
bBlockMeta := bs.LoadBlockMeta(tuple.block.Height)
if tuple.eraseSeenCommitInDB {
err := db.Delete(calcSeenCommitKey(tuple.block.Height))
err := db.Delete(seenCommitKey(tuple.block.Height))
require.NoError(t, err)
}
if tuple.corruptSeenCommitInDB {
err := db.Set(calcSeenCommitKey(tuple.block.Height), []byte("bogus-seen-commit"))
err := db.Set(seenCommitKey(tuple.block.Height), []byte("bogus-seen-commit"))
require.NoError(t, err)
}
bSeenCommit := bs.LoadSeenCommit(tuple.block.Height)
commitHeight := tuple.block.Height - 1
if tuple.eraseCommitInDB {
err := db.Delete(calcBlockCommitKey(commitHeight))
err := db.Delete(blockCommitKey(commitHeight))
require.NoError(t, err)
}
if tuple.corruptCommitInDB {
err := db.Set(calcBlockCommitKey(commitHeight), []byte("foo-bogus"))
err := db.Set(blockCommitKey(commitHeight), []byte("foo-bogus"))
require.NoError(t, err)
}
bCommit := bs.LoadBlockCommit(commitHeight)
@@ -404,7 +402,7 @@ func TestLoadBlockPart(t *testing.T) {
require.Nil(t, res, "a non-existent block part should return nil")
// 2. Next save a corrupted block then try to load it
err := db.Set(calcBlockPartKey(height, index), []byte("Tendermint"))
err := db.Set(blockPartKey(height, index), []byte("Tendermint"))
require.NoError(t, err)
res, _, panicErr = doFn(loadPart)
require.NotNil(t, panicErr, "expecting a non-nil panic")
@@ -413,7 +411,7 @@ func TestLoadBlockPart(t *testing.T) {
// 3. A good block serialized and saved to the DB should be retrievable
pb1, err := part1.ToProto()
require.NoError(t, err)
err = db.Set(calcBlockPartKey(height, index), mustEncode(pb1))
err = db.Set(blockPartKey(height, index), mustEncode(pb1))
require.NoError(t, err)
gotPart, _, panicErr := doFn(loadPart)
require.Nil(t, panicErr, "an existent and proper block should not panic")
@@ -524,7 +522,7 @@ func TestLoadBlockMeta(t *testing.T) {
require.Nil(t, res, "a non-existent blockMeta should return nil")
// 2. Next save a corrupted blockMeta then try to load it
err := db.Set(calcBlockMetaKey(height), []byte("Tendermint-Meta"))
err := db.Set(blockMetaKey(height), []byte("Tendermint-Meta"))
require.NoError(t, err)
res, _, panicErr = doFn(loadMeta)
require.NotNil(t, panicErr, "expecting a non-nil panic")
@@ -535,7 +533,7 @@ func TestLoadBlockMeta(t *testing.T) {
Version: tmversion.Consensus{
Block: version.BlockProtocol, App: 0}, Height: 1, ProposerAddress: tmrand.Bytes(crypto.AddressSize)}}
pbm := meta.ToProto()
err = db.Set(calcBlockMetaKey(height), mustEncode(pbm))
err = db.Set(blockMetaKey(height), mustEncode(pbm))
require.NoError(t, err)
gotMeta, _, panicErr := doFn(loadMeta)
require.Nil(t, panicErr, "an existent and proper block should not panic")