add support for block pruning via ABCI Commit response (#4588)

* Added BlockStore.DeleteBlock()

* Added initial block pruner prototype

* wip

* Added BlockStore.PruneBlocks()

* Added consensus setting for block pruning

* Added BlockStore base

* Error on replay if base does not have blocks

* Handle missing blocks when sending VoteSetMaj23Message

* Error message tweak

* Properly update blockstore state

* Error message fix again

* blockchain: ignore peer missing blocks

* Added FIXME

* Added test for block replay with truncated history

* Handle peer base in blockchain reactor

* Improved replay error handling

* Added tests for Store.PruneBlocks()

* Fix non-RPC handling of truncated block history

* Panic on missing block meta in needProofBlock()

* Updated changelog

* Handle truncated block history in RPC layer

* Added info about earliest block in /status RPC

* Reorder height and base in blockchain reactor messages

* Updated changelog

* Fix tests

* Appease linter

* Minor review fixes

* Non-empty BlockStores should always have base > 0

* Update code to assume base > 0 invariant

* Added blockstore tests for pruning to 0

* Make sure we don't prune below the current base

* Added BlockStore.Size()

* config: added retain_blocks recommendations

* Update v1 blockchain reactor to handle blockstore base

* Added state database pruning

* Propagate errors on missing validator sets

* Comment tweaks

* Improved error message

Co-Authored-By: Anton Kaliaev <anton.kalyaev@gmail.com>

* use ABCI field ResponseCommit.retain_height instead of retain-blocks config option

* remove State.RetainHeight, return value instead

* fix minor issues

* rename pruneHeights() to pruneBlocks()

* noop to fix GitHub borkage

Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>

rpc/core/blocks.go

@@ -17,7 +17,7 @@ func BlockchainInfo(ctx *rpctypes.Context, minHeight, maxHeight int64) (*ctypes.
 	// maximum 20 block metas
 	const limit int64 = 20
 	var err error
-	minHeight, maxHeight, err = filterMinMax(blockStore.Height(), minHeight, maxHeight, limit)
+	minHeight, maxHeight, err = filterMinMax(blockStore.Base(), blockStore.Height(), minHeight, maxHeight, limit)
 	if err != nil {
 		return nil, err
 	}
@@ -34,11 +34,10 @@ func BlockchainInfo(ctx *rpctypes.Context, minHeight, maxHeight int64) (*ctypes.
 		BlockMetas: blockMetas}, nil
 }
 
-// error if either min or max are negative or min < max
-// if 0, use 1 for min, latest block height for max
+// error if either min or max are negative or min > max
+// if 0, use blockstore base for min, latest block height for max
 // enforce limit.
-// error if min > max
-func filterMinMax(height, min, max, limit int64) (int64, int64, error) {
+func filterMinMax(base, height, min, max, limit int64) (int64, int64, error) {
 	// filter negatives
 	if min < 0 || max < 0 {
 		return min, max, fmt.Errorf("heights must be non-negative")
@@ -55,6 +54,9 @@ func filterMinMax(height, min, max, limit int64) (int64, int64, error) {
 	// limit max to the height
 	max = tmmath.MinInt64(height, max)
 
+	// limit min to the base
+	min = tmmath.MaxInt64(base, min)
+
 	// limit min to within `limit` of max
 	// so the total number of blocks returned will be `limit`
 	min = tmmath.MaxInt64(min, max-limit+1)
@@ -69,8 +71,7 @@ func filterMinMax(height, min, max, limit int64) (int64, int64, error) {
 // If no height is provided, it will fetch the latest block.
 // More: https://docs.tendermint.com/master/rpc/#/Info/block
 func Block(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlock, error) {
-	storeHeight := blockStore.Height()
-	height, err := getHeight(storeHeight, heightPtr)
+	height, err := getHeight(blockStore.Base(), blockStore.Height(), heightPtr)
 	if err != nil {
 		return nil, err
 	}
@@ -99,8 +100,7 @@ func BlockByHash(ctx *rpctypes.Context, hash []byte) (*ctypes.ResultBlock, error
 // If no height is provided, it will fetch the commit for the latest block.
 // More: https://docs.tendermint.com/master/rpc/#/Info/commit
 func Commit(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultCommit, error) {
-	storeHeight := blockStore.Height()
-	height, err := getHeight(storeHeight, heightPtr)
+	height, err := getHeight(blockStore.Base(), blockStore.Height(), heightPtr)
 	if err != nil {
 		return nil, err
 	}
@@ -113,7 +113,7 @@ func Commit(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultCommit, erro
 
 	// If the next block has not been committed yet,
 	// use a non-canonical commit
-	if height == storeHeight {
+	if height == blockStore.Height() {
 		commit := blockStore.LoadSeenCommit(height)
 		return ctypes.NewResultCommit(&header, commit, false), nil
 	}
@@ -131,8 +131,7 @@ func Commit(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultCommit, erro
 // getBlock(h).Txs[5]
 // More: https://docs.tendermint.com/master/rpc/#/Info/block_results
 func BlockResults(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlockResults, error) {
-	storeHeight := blockStore.Height()
-	height, err := getHeight(storeHeight, heightPtr)
+	height, err := getHeight(blockStore.Base(), blockStore.Height(), heightPtr)
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +151,7 @@ func BlockResults(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlockR
 	}, nil
 }
 
-func getHeight(currentHeight int64, heightPtr *int64) (int64, error) {
+func getHeight(currentBase int64, currentHeight int64, heightPtr *int64) (int64, error) {
 	if heightPtr != nil {
 		height := *heightPtr
 		if height <= 0 {
@@ -161,6 +160,10 @@ func getHeight(currentHeight int64, heightPtr *int64) (int64, error) {
 		if height > currentHeight {
 			return 0, fmt.Errorf("height must be less than or equal to the current blockchain height")
 		}
+		if height < currentBase {
+			return 0, fmt.Errorf("height %v is not available, blocks pruned at height %v",
+				height, currentBase)
+		}
 		return height, nil
 	}
 	return currentHeight, nil

rpc/core/blocks_test.go

@@ -19,42 +19,46 @@ import (
 func TestBlockchainInfo(t *testing.T) {
 	cases := []struct {
 		min, max     int64
-		height       int64
+		base, height int64
 		limit        int64
 		resultLength int64
 		wantErr      bool
 	}{
 
 		// min > max
-		{0, 0, 0, 10, 0, true},  // min set to 1
-		{0, 1, 0, 10, 0, true},  // max set to height (0)
-		{0, 0, 1, 10, 1, false}, // max set to height (1)
-		{2, 0, 1, 10, 0, true},  // max set to height (1)
-		{2, 1, 5, 10, 0, true},
+		{0, 0, 0, 0, 10, 0, true},  // min set to 1
+		{0, 1, 0, 0, 10, 0, true},  // max set to height (0)
+		{0, 0, 0, 1, 10, 1, false}, // max set to height (1)
+		{2, 0, 0, 1, 10, 0, true},  // max set to height (1)
+		{2, 1, 0, 5, 10, 0, true},
 
 		// negative
-		{1, 10, 14, 10, 10, false}, // control
-		{-1, 10, 14, 10, 0, true},
-		{1, -10, 14, 10, 0, true},
-		{-9223372036854775808, -9223372036854775788, 100, 20, 0, true},
+		{1, 10, 0, 14, 10, 10, false}, // control
+		{-1, 10, 0, 14, 10, 0, true},
+		{1, -10, 0, 14, 10, 0, true},
+		{-9223372036854775808, -9223372036854775788, 0, 100, 20, 0, true},
+
+		// check base
+		{1, 1, 1, 1, 1, 1, false},
+		{2, 5, 3, 5, 5, 3, false},
 
 		// check limit and height
-		{1, 1, 1, 10, 1, false},
-		{1, 1, 5, 10, 1, false},
-		{2, 2, 5, 10, 1, false},
-		{1, 2, 5, 10, 2, false},
-		{1, 5, 1, 10, 1, false},
-		{1, 5, 10, 10, 5, false},
-		{1, 15, 10, 10, 10, false},
-		{1, 15, 15, 10, 10, false},
-		{1, 15, 15, 20, 15, false},
-		{1, 20, 15, 20, 15, false},
-		{1, 20, 20, 20, 20, false},
+		{1, 1, 0, 1, 10, 1, false},
+		{1, 1, 0, 5, 10, 1, false},
+		{2, 2, 0, 5, 10, 1, false},
+		{1, 2, 0, 5, 10, 2, false},
+		{1, 5, 0, 1, 10, 1, false},
+		{1, 5, 0, 10, 10, 5, false},
+		{1, 15, 0, 10, 10, 10, false},
+		{1, 15, 0, 15, 10, 10, false},
+		{1, 15, 0, 15, 20, 15, false},
+		{1, 20, 0, 15, 20, 15, false},
+		{1, 20, 0, 20, 20, 20, false},
 	}
 
 	for i, c := range cases {
 		caseString := fmt.Sprintf("test %d failed", i)
-		min, max, err := filterMinMax(c.height, c.min, c.max, c.limit)
+		min, max, err := filterMinMax(c.base, c.height, c.min, c.max, c.limit)
 		if c.wantErr {
 			require.Error(t, err, caseString)
 		} else {
@@ -112,12 +116,15 @@ type mockBlockStore struct {
 	height int64
 }
 
+func (mockBlockStore) Base() int64                                       { return 1 }
 func (store mockBlockStore) Height() int64                               { return store.height }
+func (store mockBlockStore) Size() int64                                 { return store.height }
 func (mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta       { return nil }
 func (mockBlockStore) LoadBlock(height int64) *types.Block               { return nil }
 func (mockBlockStore) LoadBlockByHash(hash []byte) *types.Block          { return nil }
 func (mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil }
 func (mockBlockStore) LoadBlockCommit(height int64) *types.Commit        { return nil }
 func (mockBlockStore) LoadSeenCommit(height int64) *types.Commit         { return nil }
+func (mockBlockStore) PruneBlocks(height int64) (uint64, error)          { return 0, nil }
 func (mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
 }

rpc/core/consensus.go

@@ -18,7 +18,7 @@ func Validators(ctx *rpctypes.Context, heightPtr *int64, page, perPage int) (*ct
 	// The latest validator that we know is the
 	// NextValidator of the last block.
 	height := consensusState.GetState().LastBlockHeight + 1
-	height, err := getHeight(height, heightPtr)
+	height, err := getHeight(blockStore.Base(), height, heightPtr)
 	if err != nil {
 		return nil, err
 	}
@@ -91,7 +91,7 @@ func ConsensusState(ctx *rpctypes.Context) (*ctypes.ResultConsensusState, error)
 // More: https://docs.tendermint.com/master/rpc/#/Info/consensus_params
 func ConsensusParams(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultConsensusParams, error) {
 	height := consensusState.GetState().LastBlockHeight + 1
-	height, err := getHeight(height, heightPtr)
+	height, err := getHeight(blockStore.Base(), height, heightPtr)
 	if err != nil {
 		return nil, err
 	}

rpc/core/status.go

@@ -16,6 +16,20 @@ import (
 // hash, app hash, block height and time.
 // More: https://docs.tendermint.com/master/rpc/#/Info/status
 func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) {
+	var (
+		earliestBlockMeta     *types.BlockMeta
+		earliestBlockHash     tmbytes.HexBytes
+		earliestAppHash       tmbytes.HexBytes
+		earliestBlockTimeNano int64
+	)
+	earliestBlockHeight := blockStore.Base()
+	earliestBlockMeta = blockStore.LoadBlockMeta(earliestBlockHeight)
+	if earliestBlockMeta != nil {
+		earliestAppHash = earliestBlockMeta.Header.AppHash
+		earliestBlockHash = earliestBlockMeta.BlockID.Hash
+		earliestBlockTimeNano = earliestBlockMeta.Header.Time.UnixNano()
+	}
+
 	var latestHeight int64
 	if consensusReactor.FastSync() {
 		latestHeight = blockStore.Height()
@@ -36,8 +50,6 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) {
 		latestBlockTimeNano = latestBlockMeta.Header.Time.UnixNano()
 	}
 
-	latestBlockTime := time.Unix(0, latestBlockTimeNano)
-
 	var votingPower int64
 	if val := validatorAtHeight(latestHeight); val != nil {
 		votingPower = val.VotingPower
@@ -46,11 +58,15 @@ func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) {
 	result := &ctypes.ResultStatus{
 		NodeInfo: p2pTransport.NodeInfo().(p2p.DefaultNodeInfo),
 		SyncInfo: ctypes.SyncInfo{
-			LatestBlockHash:   latestBlockHash,
-			LatestAppHash:     latestAppHash,
-			LatestBlockHeight: latestHeight,
-			LatestBlockTime:   latestBlockTime,
-			CatchingUp:        consensusReactor.FastSync(),
+			LatestBlockHash:     latestBlockHash,
+			LatestAppHash:       latestAppHash,
+			LatestBlockHeight:   latestHeight,
+			LatestBlockTime:     time.Unix(0, latestBlockTimeNano),
+			EarliestBlockHash:   earliestBlockHash,
+			EarliestAppHash:     earliestAppHash,
+			EarliestBlockHeight: earliestBlockHeight,
+			EarliestBlockTime:   time.Unix(0, earliestBlockTimeNano),
+			CatchingUp:          consensusReactor.FastSync(),
 		},
 		ValidatorInfo: ctypes.ValidatorInfo{
 			Address:     pubKey.Address(),

rpc/core/types/responses.go

@@ -65,7 +65,13 @@ type SyncInfo struct {
 	LatestAppHash     bytes.HexBytes `json:"latest_app_hash"`
 	LatestBlockHeight int64          `json:"latest_block_height"`
 	LatestBlockTime   time.Time      `json:"latest_block_time"`
-	CatchingUp        bool           `json:"catching_up"`
+
+	EarliestBlockHash   bytes.HexBytes `json:"earliest_block_hash"`
+	EarliestAppHash     bytes.HexBytes `json:"earliest_app_hash"`
+	EarliestBlockHeight int64          `json:"earliest_block_height"`
+	EarliestBlockTime   time.Time      `json:"earliest_block_time"`
+
+	CatchingUp bool `json:"catching_up"`
 }
 
 // Info about the node's validator