mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2025-12-23 06:15:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use evidence period when pruning (#9505)

Browse Source 
* Added logic so when pruning, the evidence period is taken into consideration and only deletes unecessary data

(cherry picked from commit abbeb919df)

# Conflicts:
#	CHANGELOG_PENDING.md
#	go.sum
#	state/execution.go
This commit is contained in:
samricotta
2022-10-04 17:57:09 +02:00
committed by Mergify
parent 6a3e360ae8
commit 840dccdc3b
12 changed files with 197 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGELOG_PENDING.md
View File

@@ -45,6 +45,18 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
- [proto] \#9356 Migrate from `gogo/protobuf` to `cosmos/gogoproto` (@julienrbrt)
- [rpc] \#9276 Added `header` and `header_by_hash` queries to the RPC client (@samricotta)
- [abci] \#5706 Added `AbciVersion` to `RequestInfo` allowing applications to check ABCI version when connecting to Tendermint. (@marbar3778)
<<<<<<< HEAD
=======
- [node] \#6059 Validate and complete genesis doc before saving to state store (@silasdavis)
- [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778)
- [crypto/ed25519] \#6526 Use [curve25519-voi](https://github.com/oasisprotocol/curve25519-voi) for `ed25519` signing and verification. (@Yawning)
- [crypto/sr25519] \#6526 Use [curve25519-voi](https://github.com/oasisprotocol/curve25519-voi) for `sr25519` signing and verification. (@Yawning)
- [crypto] \#6120 Implement batch verification interface for ed25519 and sr25519. (@marbar3778 & @Yawning)
- [types] \#6120 use batch verification for verifying commits signatures. (@marbar3778 & @cmwaters & @Yawning)
- If the key type supports the batch verification API it will try to batch verify. If the verification fails we will single verify each signature.
- [state] \#9505 Added logic so when pruning, the evidence period is taken into consideration and only deletes unecessary data (@samricotta)
>>>>>>> abbeb919d (Use evidence period when pruning (#9505))
### BUG FIXES

7
consensus/replay_test.go
View File

@@ -735,7 +735,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin
// Prune block store if requested
expectError := false
if mode == 3 {
pruned, err := store.PruneBlocks(2)
pruned, _, err := store.PruneBlocks(2, state)
require.NoError(t, err)
require.EqualValues(t, 1, pruned)
expectError = int64(nBlocks) < 2
@@ -1184,7 +1184,8 @@ func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit {
return bs.commits[height-1]
}
func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) {
func (bs *mockBlockStore) PruneBlocks(height int64, state sm.State) (uint64, int64, error) {
evidencePoint := height
pruned := uint64(0)
for i := int64(0); i < height-1; i++ {
bs.chain[i] = nil
@@ -1192,7 +1193,7 @@ func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) {
pruned++
}
bs.base = height
return pruned, nil
return pruned, evidencePoint, nil
}
//---------------------------------------

17
evidence/verify.go
View File

@@ -21,7 +21,6 @@ func (evpool *Pool) verify(evidence types.Evidence) error {
state = evpool.State()
height = state.LastBlockHeight
evidenceParams = state.ConsensusParams.Evidence
ageNumBlocks = height - evidence.Height()
)
// verify the time of the evidence
@@ -34,10 +33,9 @@ func (evpool *Pool) verify(evidence types.Evidence) error {
return fmt.Errorf("evidence has a different time to the block it is associated with (%v != %v)",
evidence.Time(), evTime)
}
ageDuration := state.LastBlockTime.Sub(evTime)
// check that the evidence hasn't expired
if ageDuration > evidenceParams.MaxAgeDuration && ageNumBlocks > evidenceParams.MaxAgeNumBlocks {
// checking if evidence is expired calculated using the block evidence time and height
if IsEvidenceExpired(height, state.LastBlockTime, evidence.Height(), evTime, evidenceParams) {
return fmt.Errorf(
"evidence from height %d (created at: %v) is too old; min height is %d and evidence can not be older than %v",
evidence.Height(),
@@ -284,3 +282,14 @@ func getSignedHeader(blockStore BlockStore, height int64) (*types.SignedHeader,
Commit: commit,
}, nil
}
// check that the evidence hasn't expired
func IsEvidenceExpired(heightNow int64, timeNow time.Time, heightEv int64, timeEv time.Time, evidenceParams types.EvidenceParams) bool {
ageDuration := timeNow.Sub(timeEv)
ageNumBlocks := heightNow - heightEv
if ageDuration > evidenceParams.MaxAgeDuration && ageNumBlocks > evidenceParams.MaxAgeNumBlocks {
return true
}
return false
}

16
go.sum
View File

@@ -56,7 +56,10 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0=
github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
<<<<<<< HEAD
github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg=
=======
>>>>>>> abbeb919d (Use evidence period when pruning (#9505))
github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
@@ -226,7 +229,6 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU=
github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
github.com/cosmos/gogoproto v1.4.1 h1:WoyH+0/jbCTzpKNvyav5FL1ZTWsp1im1MxEpJEzKUB8=
github.com/cosmos/gogoproto v1.4.1/go.mod h1:Ac9lzL4vFpBMcptJROQ6dQ4M3pOEK5Z/l0Q9p+LoCr4=
@@ -533,9 +535,13 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s=
<<<<<<< HEAD
github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is=
github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s=
github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc=
=======
github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s=
>>>>>>> abbeb919d (Use evidence period when pruning (#9505))
github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
@@ -731,7 +737,10 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
<<<<<<< HEAD
github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 h1:QRUSJEgZn2Snx0EmT/QLXibWjSUDjKWvXIT19NBVp94=
=======
>>>>>>> abbeb919d (Use evidence period when pruning (#9505))
github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM=
github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
@@ -794,6 +803,11 @@ github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3L
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
<<<<<<< HEAD
=======
github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae h1:FatpGJD2jmJfhZiFDElaC0QhZUDQnxUeAwTGkfAHN3I=
github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s=
>>>>>>> abbeb919d (Use evidence period when pruning (#9505))
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=

34
state/execution.go
View File

@@ -256,6 +256,19 @@ func (blockExec *BlockExecutor) ApplyBlock(
fail.Fail() // XXX
<<<<<<< HEAD
=======
// Prune old heights, if requested by ABCI app.
if retainHeight > 0 {
pruned, err := blockExec.pruneBlocks(retainHeight, state)
if err != nil {
blockExec.logger.Error("failed to prune blocks", "retain_height", retainHeight, "err", err)
} else {
blockExec.logger.Debug("pruned blocks", "pruned", pruned, "retain_height", retainHeight)
}
}
>>>>>>> abbeb919d (Use evidence period when pruning (#9505))
// Events are fired after everything else.
// NOTE: if we crash between Commit and Save, events wont be fired during replay
fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses, validatorUpdates)
@@ -625,3 +638,24 @@ func ExecCommitBlock(
// ResponseCommit has no error or log, just data
return res.Data, nil
}
<<<<<<< HEAD
=======
func (blockExec *BlockExecutor) pruneBlocks(retainHeight int64, state State) (uint64, error) {
base := blockExec.blockStore.Base()
if retainHeight <= base {
return 0, nil
}
amountPruned, prunedHeaderHeight, err := blockExec.blockStore.PruneBlocks(retainHeight, state)
if err != nil {
return 0, fmt.Errorf("failed to prune block store: %w", err)
}
err = blockExec.Store().PruneStates(base, retainHeight, prunedHeaderHeight)
if err != nil {
return 0, fmt.Errorf("failed to prune state store: %w", err)
}
return amountPruned, nil
}
>>>>>>> abbeb919d (Use evidence period when pruning (#9505))

28
state/mocks/block_store.go
View File

@@ -4,6 +4,7 @@ package mocks
import (
mock "github.com/stretchr/testify/mock"
state "github.com/tendermint/tendermint/state"
types "github.com/tendermint/tendermint/types"
)
@@ -169,25 +170,32 @@ func (_m *BlockStore) LoadSeenCommit(height int64) *types.Commit {
return r0
}
// PruneBlocks provides a mock function with given fields: height
func (_m *BlockStore) PruneBlocks(height int64) (uint64, error) {
ret := _m.Called(height)
// PruneBlocks provides a mock function with given fields: height, _a1
func (_m *BlockStore) PruneBlocks(height int64, _a1 state.State) (uint64, int64, error) {
ret := _m.Called(height, _a1)
var r0 uint64
if rf, ok := ret.Get(0).(func(int64) uint64); ok {
r0 = rf(height)
if rf, ok := ret.Get(0).(func(int64, state.State) uint64); ok {
r0 = rf(height, _a1)
} else {
r0 = ret.Get(0).(uint64)
}
var r1 error
if rf, ok := ret.Get(1).(func(int64) error); ok {
r1 = rf(height)
var r1 int64
if rf, ok := ret.Get(1).(func(int64, state.State) int64); ok {
r1 = rf(height, _a1)
} else {
r1 = ret.Error(1)
r1 = ret.Get(1).(int64)
}
return r0, r1
var r2 error
if rf, ok := ret.Get(2).(func(int64, state.State) error); ok {
r2 = rf(height, _a1)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// SaveBlock provides a mock function with given fields: block, blockParts, seenCommit

10
state/mocks/store.go
View File

@@ -197,13 +197,13 @@ func (_m *Store) LoadValidators(_a0 int64) (*types.ValidatorSet, error) {
return r0, r1
}
// PruneStates provides a mock function with given fields: _a0, _a1
func (_m *Store) PruneStates(_a0 int64, _a1 int64) error {
ret := _m.Called(_a0, _a1)
// PruneStates provides a mock function with given fields: _a0, _a1, _a2
func (_m *Store) PruneStates(_a0 int64, _a1 int64, _a2 int64) error {
ret := _m.Called(_a0, _a1, _a2)
var r0 error
if rf, ok := ret.Get(0).(func(int64, int64) error); ok {
r0 = rf(_a0, _a1)
if rf, ok := ret.Get(0).(func(int64, int64, int64) error); ok {
r0 = rf(_a0, _a1, _a2)
} else {
r0 = ret.Error(0)
}

2
state/services.go
View File

@@ -26,7 +26,7 @@ type BlockStore interface {
SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit)
PruneBlocks(height int64) (uint64, error)
PruneBlocks(height int64, state State) (uint64, int64, error)
LoadBlockByHash(hash []byte) *types.Block
LoadBlockMetaByHash(hash []byte) *types.BlockMeta

22
state/store.go
View File

@@ -68,10 +68,10 @@ type Store interface {
Save(State) error
// SaveABCIResponses saves ABCIResponses for a given height
SaveABCIResponses(int64, *tmstate.ABCIResponses) error
// Bootstrap is used for bootstrapping state when not starting from a initial height.
// Bootstrap is used for bootstrapping state when not starting from a initial height
Bootstrap(State) error
// PruneStates takes the height from which to start prning and which height stop at
PruneStates(int64, int64) error
// PruneStates takes the height from which to start pruning and which height stop at
PruneStates(int64, int64, int64) error
// Close closes the connection with the database
Close() error
}
@@ -237,14 +237,15 @@ func (store dbStore) Bootstrap(state State) error {
// encoding not preserving ordering: https://github.com/tendermint/tendermint/issues/4567
// This will cause some old states to be left behind when doing incremental partial prunes,
// specifically older checkpoints and LastHeightChanged targets.
func (store dbStore) PruneStates(from int64, to int64) error {
func (store dbStore) PruneStates(from int64, to int64, evidenceThresholdHeight int64) error {
if from <= 0 || to <= 0 {
return fmt.Errorf("from height %v and to height %v must be greater than 0", from, to)
}
if from >= to {
return fmt.Errorf("from height %v must be lower than to height %v", from, to)
}
valInfo, err := loadValidatorsInfo(store.db, to)
valInfo, err := loadValidatorsInfo(store.db, min(to, evidenceThresholdHeight))
if err != nil {
return fmt.Errorf("validators at height %v not found: %w", to, err)
}
@@ -298,12 +299,14 @@ func (store dbStore) PruneStates(from int64, to int64) error {
return err
}
}
} else {
} else if h < evidenceThresholdHeight {
err = batch.Delete(calcValidatorsKey(h))
if err != nil {
return err
}
}
// else we keep the validator set because we might need
// it later on for evidence verification
if keepParams[h] {
p, err := store.loadConsensusParamsInfo(h)
@@ -661,3 +664,10 @@ func (store dbStore) saveConsensusParamsInfo(nextHeight, changeHeight int64, par
func (store dbStore) Close() error {
return store.db.Close()
}
func min(a int64, b int64) int64 {
if a < b {
return a
}
return b
}

32
state/store_test.go
View File

@@ -88,23 +88,25 @@ func BenchmarkLoadValidators(b *testing.B) {
func TestPruneStates(t *testing.T) {
testcases := map[string]struct {
makeHeights int64
pruneFrom int64
pruneTo int64
expectErr bool
expectVals []int64
expectParams []int64
expectABCI []int64
makeHeights int64
pruneFrom int64
pruneTo int64
evidenceThresholdHeight int64
expectErr bool
expectVals []int64
expectParams []int64
expectABCI []int64
}{
"error on pruning from 0": {100, 0, 5, true, nil, nil, nil},
"error when from > to": {100, 3, 2, true, nil, nil, nil},
"error when from == to": {100, 3, 3, true, nil, nil, nil},
"error when to does not exist": {100, 1, 101, true, nil, nil, nil},
"prune all": {100, 1, 100, false, []int64{93, 100}, []int64{95, 100}, []int64{100}},
"prune some": {10, 2, 8, false, []int64{1, 3, 8, 9, 10},
"error on pruning from 0": {100, 0, 5, 100, true, nil, nil, nil},
"error when from > to": {100, 3, 2, 2, true, nil, nil, nil},
"error when from == to": {100, 3, 3, 3, true, nil, nil, nil},
"error when to does not exist": {100, 1, 101, 101, true, nil, nil, nil},
"prune all": {100, 1, 100, 100, false, []int64{93, 100}, []int64{95, 100}, []int64{100}},
"prune some": {10, 2, 8, 8, false, []int64{1, 3, 8, 9, 10},
[]int64{1, 5, 8, 9, 10}, []int64{1, 8, 9, 10}},
"prune across checkpoint": {100001, 1, 100001, false, []int64{99993, 100000, 100001},
"prune across checkpoint": {100001, 1, 100001, 100001, false, []int64{99993, 100000, 100001},
[]int64{99995, 100001}, []int64{100001}},
"prune when evidence height < height": {20, 1, 18, 17, false, []int64{13, 17, 18, 19, 20}, []int64{15, 18, 19, 20}, []int64{18, 19, 20}},
}
for name, tc := range testcases {
tc := tc
@@ -163,7 +165,7 @@ func TestPruneStates(t *testing.T) {
}
// Test assertions
err := stateStore.PruneStates(tc.pruneFrom, tc.pruneTo)
err := stateStore.PruneStates(tc.pruneFrom, tc.pruneTo, tc.evidenceThresholdHeight)
if tc.expectErr {
require.Error(t, err)
return

48
store/store.go
View File

@@ -7,9 +7,11 @@ import (
"github.com/cosmos/gogoproto/proto"
dbm "github.com/tendermint/tm-db"
"github.com/tendermint/tendermint/evidence"
tmsync "github.com/tendermint/tendermint/libs/sync"
tmstore "github.com/tendermint/tendermint/proto/tendermint/store"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
)
@@ -264,20 +266,20 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit {
return commit
}
// PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned.
func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) {
// PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned and the evidence retain height - the height at which data needed to prove evidence must not be removed.
func (bs *BlockStore) PruneBlocks(height int64, state sm.State) (uint64, int64, error) {
if height <= 0 {
return 0, fmt.Errorf("height must be greater than 0")
return 0, -1, fmt.Errorf("height must be greater than 0")
}
bs.mtx.RLock()
if height > bs.height {
bs.mtx.RUnlock()
return 0, fmt.Errorf("cannot prune beyond the latest height %v", bs.height)
return 0, -1, fmt.Errorf("cannot prune beyond the latest height %v", bs.height)
}
base := bs.base
bs.mtx.RUnlock()
if height < base {
return 0, fmt.Errorf("cannot prune to height %v, it is lower than base height %v",
return 0, -1, fmt.Errorf("cannot prune to height %v, it is lower than base height %v",
height, base)
}
@@ -300,26 +302,42 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) {
return nil
}
evidencePoint := height
for h := base; h < height; h++ {
meta := bs.LoadBlockMeta(h)
if meta == nil { // assume already deleted
continue
}
if err := batch.Delete(calcBlockMetaKey(h)); err != nil {
return 0, err
// This logic is in place to protect data that proves malicious behavior.
// If the height is within the evidence age, we continue to persist the header and commit data.
if evidencePoint == height && !evidence.IsEvidenceExpired(state.LastBlockHeight, state.LastBlockTime, h, meta.Header.Time, state.ConsensusParams.Evidence) {
evidencePoint = h
}
// if height is beyond the evidence point we dont delete the header
if h < evidencePoint {
if err := batch.Delete(calcBlockMetaKey(h)); err != nil {
return 0, -1, err
}
}
if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil {
return 0, err
return 0, -1, err
}
if err := batch.Delete(calcBlockCommitKey(h)); err != nil {
return 0, err
// if height is beyond the evidence point we dont delete the commit data
if h < evidencePoint {
if err := batch.Delete(calcBlockCommitKey(h)); err != nil {
return 0, -1, err
}
}
if err := batch.Delete(calcSeenCommitKey(h)); err != nil {
return 0, err
return 0, -1, err
}
for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ {
if err := batch.Delete(calcBlockPartKey(h, p)); err != nil {
return 0, err
return 0, -1, err
}
}
pruned++
@@ -328,7 +346,7 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) {
if pruned%1000 == 0 && pruned > 0 {
err := flush(batch, h)
if err != nil {
return 0, err
return 0, -1, err
}
batch = bs.db.NewBatch()
defer batch.Close()
@@ -337,9 +355,9 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) {
err := flush(batch, height)
if err != nil {
return 0, err
return 0, -1, err
}
return pruned, nil
return pruned, evidencePoint, nil
}
// SaveBlock persists the given block, blockParts, and seenCommit to the underlying db.

48
store/store_test.go
View File

@@ -382,7 +382,7 @@ func TestLoadBaseMeta(t *testing.T) {
bs.SaveBlock(block, partSet, seenCommit)
}
_, err = bs.PruneBlocks(4)
_, _, err = bs.PruneBlocks(4, state)
require.NoError(t, err)
baseBlock := bs.LoadBaseMeta()
@@ -438,10 +438,10 @@ func TestPruneBlocks(t *testing.T) {
assert.EqualValues(t, 0, bs.Size())
// pruning an empty store should error, even when pruning to 0
_, err = bs.PruneBlocks(1)
_, _, err = bs.PruneBlocks(1, state)
require.Error(t, err)
_, err = bs.PruneBlocks(0)
_, _, err = bs.PruneBlocks(0, state)
require.Error(t, err)
// make more than 1000 blocks, to test batch deletions
@@ -457,27 +457,30 @@ func TestPruneBlocks(t *testing.T) {
assert.EqualValues(t, 1500, bs.Height())
assert.EqualValues(t, 1500, bs.Size())
prunedBlock := bs.LoadBlock(1199)
state.LastBlockTime = time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC)
state.LastBlockHeight = 1500
state.ConsensusParams.Evidence.MaxAgeNumBlocks = 400
state.ConsensusParams.Evidence.MaxAgeDuration = 1 * time.Second
// Check that basic pruning works
pruned, err := bs.PruneBlocks(1200)
pruned, evidenceRetainHeight, err := bs.PruneBlocks(1200, state)
require.NoError(t, err)
assert.EqualValues(t, 1199, pruned)
assert.EqualValues(t, 1200, bs.Base())
assert.EqualValues(t, 1500, bs.Height())
assert.EqualValues(t, 301, bs.Size())
assert.EqualValues(t, tmstore.BlockStoreState{
Base: 1200,
Height: 1500,
}, LoadBlockStoreState(db))
assert.EqualValues(t, 1100, evidenceRetainHeight)
require.NotNil(t, bs.LoadBlock(1200))
require.Nil(t, bs.LoadBlock(1199))
require.Nil(t, bs.LoadBlockByHash(prunedBlock.Hash()))
require.Nil(t, bs.LoadBlockCommit(1199))
require.Nil(t, bs.LoadBlockMeta(1199))
require.Nil(t, bs.LoadBlockMetaByHash(prunedBlock.Hash()))
require.Nil(t, bs.LoadBlockPart(1199, 1))
// The header and commit for heights 1100 onwards
// need to remain to verify evidence
require.NotNil(t, bs.LoadBlockMeta(1100))
require.Nil(t, bs.LoadBlockMeta(1099))
require.NotNil(t, bs.LoadBlockCommit(1100))
require.Nil(t, bs.LoadBlockCommit(1099))
for i := int64(1); i < 1200; i++ {
require.Nil(t, bs.LoadBlock(i))
@@ -487,26 +490,33 @@ func TestPruneBlocks(t *testing.T) {
}
// Pruning below the current base should error
_, err = bs.PruneBlocks(1199)
_, _, err = bs.PruneBlocks(1199, state)
require.Error(t, err)
// Pruning to the current base should work
pruned, err = bs.PruneBlocks(1200)
pruned, _, err = bs.PruneBlocks(1200, state)
require.NoError(t, err)
assert.EqualValues(t, 0, pruned)
// Pruning again should work
pruned, err = bs.PruneBlocks(1300)
pruned, _, err = bs.PruneBlocks(1300, state)
require.NoError(t, err)
assert.EqualValues(t, 100, pruned)
assert.EqualValues(t, 1300, bs.Base())
// we should still have the header and the commit
// as they're needed for evidence
require.NotNil(t, bs.LoadBlockMeta(1100))
require.Nil(t, bs.LoadBlockMeta(1099))
require.NotNil(t, bs.LoadBlockCommit(1100))
require.Nil(t, bs.LoadBlockCommit(1099))
// Pruning beyond the current height should error
_, err = bs.PruneBlocks(1501)
_, _, err = bs.PruneBlocks(1501, state)
require.Error(t, err)
// Pruning to the current height should work
pruned, err = bs.PruneBlocks(1500)
pruned, _, err = bs.PruneBlocks(1500, state)
require.NoError(t, err)
assert.EqualValues(t, 200, pruned)
assert.Nil(t, bs.LoadBlock(1499))