cli: allow node operator to rollback last state (backport #7033) (#7080)

CHANGELOG_PENDING.md

@@ -21,9 +21,10 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 ### FEATURES
 
 - [\#6982](https://github.com/tendermint/tendermint/pull/6982) tendermint binary has built-in suppport for running the end to end application (with state sync support) (@cmwaters).
+- [cli] [#7033](https://github.com/tendermint/tendermint/pull/7033) Add a `rollback` command to rollback to the previous tendermint state in the event of non-determinstic app hash or reverting an upgrade.
 
 ### IMPROVEMENTS
 
 ### BUG FIXES
 
-- [\#7057](https://github.com/tendermint/tendermint/pull/7057) Import Postgres driver support for the psql indexer (@creachadair).
+- [\#7057](https://github.com/tendermint/tendermint/pull/7057) Import Postgres driver support for the psql indexer (@creachadair).

cmd/tendermint/commands/rollback.go

@@ -0,0 +1,69 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	dbm "github.com/tendermint/tm-db"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+)
+
+var RollbackStateCmd = &cobra.Command{
+	Use:   "rollback",
+	Short: "rollback tendermint state by one height",
+	Long: `
+A state rollback is performed to recover from an incorrect application state transition,
+when Tendermint has persisted an incorrect app hash and is thus unable to make
+progress. Rollback overwrites a state at height n with the state at height n - 1.
+The application should also roll back to height n - 1. No blocks are removed, so upon
+restarting Tendermint the transactions in block n will be re-executed against the
+application.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		height, hash, err := RollbackState(config)
+		if err != nil {
+			return fmt.Errorf("failed to rollback state: %w", err)
+		}
+
+		fmt.Printf("Rolled back state to height %d and hash %v", height, hash)
+		return nil
+	},
+}
+
+// RollbackState takes the state at the current height n and overwrites it with the state
+// at height n - 1. Note state here refers to tendermint state not application state.
+// Returns the latest state height and app hash alongside an error if there was one.
+func RollbackState(config *cfg.Config) (int64, []byte, error) {
+	// use the parsed config to load the block and state store
+	blockStore, stateStore, err := loadStateAndBlockStore(config)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	// rollback the last state
+	return state.Rollback(blockStore, stateStore)
+}
+
+func loadStateAndBlockStore(config *cfg.Config) (*store.BlockStore, state.Store, error) {
+	dbType := dbm.BackendType(config.DBBackend)
+
+	// Get BlockStore
+	blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir())
+	if err != nil {
+		return nil, nil, err
+	}
+	blockStore := store.NewBlockStore(blockStoreDB)
+
+	// Get StateStore
+	stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
+	if err != nil {
+		return nil, nil, err
+	}
+	stateStore := state.NewStore(stateDB)
+
+	return blockStore, stateStore, nil
+}

cmd/tendermint/main.go

@@ -27,6 +27,7 @@ func main() {
 		cmd.ShowNodeIDCmd,
 		cmd.GenNodeKeyCmd,
 		cmd.VersionCmd,
+		cmd.RollbackStateCmd,
 		debug.DebugCmd,
 		cli.NewCompletionCmd(rootCmd, true),
 	)

state/mocks/block_store.go

@@ -0,0 +1,194 @@
+// Code generated by mockery 2.9.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+
+	types "github.com/tendermint/tendermint/types"
+)
+
+// BlockStore is an autogenerated mock type for the BlockStore type
+type BlockStore struct {
+	mock.Mock
+}
+
+// Base provides a mock function with given fields:
+func (_m *BlockStore) Base() int64 {
+	ret := _m.Called()
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func() int64); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	return r0
+}
+
+// Height provides a mock function with given fields:
+func (_m *BlockStore) Height() int64 {
+	ret := _m.Called()
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func() int64); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	return r0
+}
+
+// LoadBaseMeta provides a mock function with given fields:
+func (_m *BlockStore) LoadBaseMeta() *types.BlockMeta {
+	ret := _m.Called()
+
+	var r0 *types.BlockMeta
+	if rf, ok := ret.Get(0).(func() *types.BlockMeta); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.BlockMeta)
+		}
+	}
+
+	return r0
+}
+
+// LoadBlock provides a mock function with given fields: height
+func (_m *BlockStore) LoadBlock(height int64) *types.Block {
+	ret := _m.Called(height)
+
+	var r0 *types.Block
+	if rf, ok := ret.Get(0).(func(int64) *types.Block); ok {
+		r0 = rf(height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.Block)
+		}
+	}
+
+	return r0
+}
+
+// LoadBlockByHash provides a mock function with given fields: hash
+func (_m *BlockStore) LoadBlockByHash(hash []byte) *types.Block {
+	ret := _m.Called(hash)
+
+	var r0 *types.Block
+	if rf, ok := ret.Get(0).(func([]byte) *types.Block); ok {
+		r0 = rf(hash)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.Block)
+		}
+	}
+
+	return r0
+}
+
+// LoadBlockCommit provides a mock function with given fields: height
+func (_m *BlockStore) LoadBlockCommit(height int64) *types.Commit {
+	ret := _m.Called(height)
+
+	var r0 *types.Commit
+	if rf, ok := ret.Get(0).(func(int64) *types.Commit); ok {
+		r0 = rf(height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.Commit)
+		}
+	}
+
+	return r0
+}
+
+// LoadBlockMeta provides a mock function with given fields: height
+func (_m *BlockStore) LoadBlockMeta(height int64) *types.BlockMeta {
+	ret := _m.Called(height)
+
+	var r0 *types.BlockMeta
+	if rf, ok := ret.Get(0).(func(int64) *types.BlockMeta); ok {
+		r0 = rf(height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.BlockMeta)
+		}
+	}
+
+	return r0
+}
+
+// LoadBlockPart provides a mock function with given fields: height, index
+func (_m *BlockStore) LoadBlockPart(height int64, index int) *types.Part {
+	ret := _m.Called(height, index)
+
+	var r0 *types.Part
+	if rf, ok := ret.Get(0).(func(int64, int) *types.Part); ok {
+		r0 = rf(height, index)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.Part)
+		}
+	}
+
+	return r0
+}
+
+// LoadSeenCommit provides a mock function with given fields: height
+func (_m *BlockStore) LoadSeenCommit(height int64) *types.Commit {
+	ret := _m.Called(height)
+
+	var r0 *types.Commit
+	if rf, ok := ret.Get(0).(func(int64) *types.Commit); ok {
+		r0 = rf(height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.Commit)
+		}
+	}
+
+	return r0
+}
+
+// PruneBlocks provides a mock function with given fields: height
+func (_m *BlockStore) PruneBlocks(height int64) (uint64, error) {
+	ret := _m.Called(height)
+
+	var r0 uint64
+	if rf, ok := ret.Get(0).(func(int64) uint64); ok {
+		r0 = rf(height)
+	} else {
+		r0 = ret.Get(0).(uint64)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(int64) error); ok {
+		r1 = rf(height)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// SaveBlock provides a mock function with given fields: block, blockParts, seenCommit
+func (_m *BlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+	_m.Called(block, blockParts, seenCommit)
+}
+
+// Size provides a mock function with given fields:
+func (_m *BlockStore) Size() int64 {
+	ret := _m.Called()
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func() int64); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	return r0
+}

state/mocks/evidence_pool.go

@@ -1,4 +1,4 @@
-// Code generated by mockery v2.1.0. DO NOT EDIT.
+// Code generated by mockery 2.9.0. DO NOT EDIT.
 
 package mocks
 

state/mocks/store.go

@@ -1,4 +1,4 @@
-// Code generated by mockery v2.1.0. DO NOT EDIT.
+// Code generated by mockery 2.9.0. DO NOT EDIT.
 
 package mocks
 

state/rollback.go

@@ -0,0 +1,89 @@
+package state
+
+import (
+	"errors"
+	"fmt"
+
+	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	tmversion "github.com/tendermint/tendermint/proto/tendermint/version"
+	"github.com/tendermint/tendermint/version"
+)
+
+// Rollback overwrites the current Tendermint state (height n) with the most
+// recent previous state (height n - 1).
+// Note that this function does not affect application state.
+func Rollback(bs BlockStore, ss Store) (int64, []byte, error) {
+	invalidState, err := ss.Load()
+	if err != nil {
+		return -1, nil, err
+	}
+	if invalidState.IsEmpty() {
+		return -1, nil, errors.New("no state found")
+	}
+
+	rollbackHeight := invalidState.LastBlockHeight
+	rollbackBlock := bs.LoadBlockMeta(rollbackHeight)
+	if rollbackBlock == nil {
+		return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight)
+	}
+
+	previousValidatorSet, err := ss.LoadValidators(rollbackHeight - 1)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	previousParams, err := ss.LoadConsensusParams(rollbackHeight)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	valChangeHeight := invalidState.LastHeightValidatorsChanged
+	// this can only happen if the validator set changed since the last block
+	if valChangeHeight > rollbackHeight {
+		valChangeHeight = rollbackHeight
+	}
+
+	paramsChangeHeight := invalidState.LastHeightConsensusParamsChanged
+	// this can only happen if params changed from the last block
+	if paramsChangeHeight > rollbackHeight {
+		paramsChangeHeight = rollbackHeight
+	}
+
+	// build the new state from the old state and the prior block
+	rolledBackState := State{
+		Version: tmstate.Version{
+			Consensus: tmversion.Consensus{
+				Block: version.BlockProtocol,
+				App:   previousParams.Version.AppVersion,
+			},
+			Software: version.TMCoreSemVer,
+		},
+		// immutable fields
+		ChainID:       invalidState.ChainID,
+		InitialHeight: invalidState.InitialHeight,
+
+		LastBlockHeight: invalidState.LastBlockHeight - 1,
+		LastBlockID:     rollbackBlock.Header.LastBlockID,
+		LastBlockTime:   rollbackBlock.Header.Time,
+
+		NextValidators:              invalidState.Validators,
+		Validators:                  invalidState.LastValidators,
+		LastValidators:              previousValidatorSet,
+		LastHeightValidatorsChanged: valChangeHeight,
+
+		ConsensusParams:                  previousParams,
+		LastHeightConsensusParamsChanged: paramsChangeHeight,
+
+		LastResultsHash: rollbackBlock.Header.LastResultsHash,
+		AppHash:         rollbackBlock.Header.AppHash,
+	}
+
+	// persist the new state. This overrides the invalid one. NOTE: this will also
+	// persist the validator set and consensus params over the existing structures,
+	// but both should be the same
+	if err := ss.Save(rolledBackState); err != nil {
+		return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err)
+	}
+
+	return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil
+}

state/rollback_test.go

@@ -0,0 +1,165 @@
+package state_test
+
+import (
+	"crypto/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
+
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	tmversion "github.com/tendermint/tendermint/proto/tendermint/version"
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/state/mocks"
+	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tendermint/version"
+)
+
+func TestRollback(t *testing.T) {
+	stateStore := state.NewStore(dbm.NewMemDB())
+	blockStore := &mocks.BlockStore{}
+	var (
+		height     int64  = 100
+		appVersion uint64 = 10
+	)
+
+	valSet, _ := types.RandValidatorSet(5, 10)
+
+	params := types.DefaultConsensusParams()
+	params.Version.AppVersion = appVersion
+	newParams := types.DefaultConsensusParams()
+	newParams.Block.MaxBytes = 10000
+
+	initialState := state.State{
+		Version: tmstate.Version{
+			Consensus: tmversion.Consensus{
+				Block: version.BlockProtocol,
+				App:   10,
+			},
+			Software: version.TMCoreSemVer,
+		},
+		ChainID:                          "test-chain",
+		InitialHeight:                    10,
+		LastBlockID:                      makeBlockIDRandom(),
+		AppHash:                          tmhash.Sum([]byte("app_hash")),
+		LastResultsHash:                  tmhash.Sum([]byte("last_results_hash")),
+		LastBlockHeight:                  height,
+		LastValidators:                   valSet,
+		Validators:                       valSet.CopyIncrementProposerPriority(1),
+		NextValidators:                   valSet.CopyIncrementProposerPriority(2),
+		LastHeightValidatorsChanged:      height + 1,
+		ConsensusParams:                  *params,
+		LastHeightConsensusParamsChanged: height + 1,
+	}
+	require.NoError(t, stateStore.Bootstrap(initialState))
+
+	height++
+	block := &types.BlockMeta{
+		Header: types.Header{
+			Height:          height,
+			AppHash:         initialState.AppHash,
+			LastBlockID:     initialState.LastBlockID,
+			LastResultsHash: initialState.LastResultsHash,
+		},
+	}
+	blockStore.On("LoadBlockMeta", height).Return(block)
+
+	appVersion++
+	newParams.Version.AppVersion = appVersion
+	nextState := initialState.Copy()
+	nextState.LastBlockHeight = height
+	nextState.Version.Consensus.App = appVersion
+	nextState.LastBlockID = makeBlockIDRandom()
+	nextState.AppHash = tmhash.Sum([]byte("next_app_hash"))
+	nextState.LastValidators = initialState.Validators
+	nextState.Validators = initialState.NextValidators
+	nextState.NextValidators = initialState.NextValidators.CopyIncrementProposerPriority(1)
+	nextState.ConsensusParams = *newParams
+	nextState.LastHeightConsensusParamsChanged = height + 1
+	nextState.LastHeightValidatorsChanged = height + 1
+
+	// update the state
+	require.NoError(t, stateStore.Save(nextState))
+
+	// rollback the state
+	rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore)
+	require.NoError(t, err)
+	require.EqualValues(t, int64(100), rollbackHeight)
+	require.EqualValues(t, initialState.AppHash, rollbackHash)
+	blockStore.AssertExpectations(t)
+
+	// assert that we've recovered the prior state
+	loadedState, err := stateStore.Load()
+	require.NoError(t, err)
+	require.EqualValues(t, initialState, loadedState)
+}
+
+func TestRollbackNoState(t *testing.T) {
+	stateStore := state.NewStore(dbm.NewMemDB())
+	blockStore := &mocks.BlockStore{}
+
+	_, _, err := state.Rollback(blockStore, stateStore)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "no state found")
+}
+
+func TestRollbackNoBlocks(t *testing.T) {
+	stateStore := state.NewStore(dbm.NewMemDB())
+	blockStore := &mocks.BlockStore{}
+	var (
+		height     int64  = 100
+		appVersion uint64 = 10
+	)
+
+	valSet, _ := types.RandValidatorSet(5, 10)
+
+	params := types.DefaultConsensusParams()
+	params.Version.AppVersion = appVersion
+	newParams := types.DefaultConsensusParams()
+	newParams.Block.MaxBytes = 10000
+
+	initialState := state.State{
+		Version: tmstate.Version{
+			Consensus: tmversion.Consensus{
+				Block: version.BlockProtocol,
+				App:   10,
+			},
+			Software: version.TMCoreSemVer,
+		},
+		ChainID:                          "test-chain",
+		InitialHeight:                    10,
+		LastBlockID:                      makeBlockIDRandom(),
+		AppHash:                          tmhash.Sum([]byte("app_hash")),
+		LastResultsHash:                  tmhash.Sum([]byte("last_results_hash")),
+		LastBlockHeight:                  height,
+		LastValidators:                   valSet,
+		Validators:                       valSet.CopyIncrementProposerPriority(1),
+		NextValidators:                   valSet.CopyIncrementProposerPriority(2),
+		LastHeightValidatorsChanged:      height + 1,
+		ConsensusParams:                  *params,
+		LastHeightConsensusParamsChanged: height + 1,
+	}
+	require.NoError(t, stateStore.Save(initialState))
+	blockStore.On("LoadBlockMeta", height).Return(nil)
+
+	_, _, err := state.Rollback(blockStore, stateStore)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "block at height 100 not found")
+}
+
+func makeBlockIDRandom() types.BlockID {
+	var (
+		blockHash   = make([]byte, tmhash.Size)
+		partSetHash = make([]byte, tmhash.Size)
+	)
+	rand.Read(blockHash)   //nolint: errcheck // ignore errcheck for read
+	rand.Read(partSetHash) //nolint: errcheck // ignore errcheck for read
+	return types.BlockID{
+		Hash: blockHash,
+		PartSetHeader: types.PartSetHeader{
+			Total: 123,
+			Hash:  partSetHash,
+		},
+	}
+}

state/services.go

@@ -12,6 +12,8 @@ import (
 //------------------------------------------------------
 // blockstore
 
+//go:generate mockery --case underscore --name BlockStore
+
 // BlockStore defines the interface used by the ConsensusState.
 type BlockStore interface {
 	Base() int64