cli: allow node operator to rollback last state (backport #7033) (#7081)

CHANGELOG_PENDING.md

@@ -22,6 +22,7 @@ Special thanks to external contributors on this release:
 
 ### FEATURES
 
+- [cli] [#7033](https://github.com/tendermint/tendermint/pull/7033) Add a `rollback` command to rollback to the previous tendermint state in the event of an incorrect app hash. (@cmwaters)
 ### IMPROVEMENTS
 
 ### BUG FIXES

cmd/tendermint/commands/reindex_event.go

@@ -29,11 +29,12 @@ var ReIndexEventCmd = &cobra.Command{
 	Use:   "reindex-event",
 	Short: "reindex events to the event store backends",
 	Long: `
-	reindex-event is an offline tooling to re-index block and tx events to the eventsinks,
-	you can run this command when the event store backend dropped/disconnected or you want to replace the backend.
-	The default start-height is 0, meaning the tooling will start reindex from the base block height(inclusive); and the
-	default end-height is 0, meaning the tooling will reindex until the latest block height(inclusive). User can omits
-	either or both arguments.
+reindex-event is an offline tooling to re-index block and tx events to the eventsinks,
+you can run this command when the event store backend dropped/disconnected or you want to 
+replace the backend. The default start-height is 0, meaning the tooling will start 
+reindex from the base block height(inclusive); and the default end-height is 0, meaning 
+the tooling will reindex until the latest block height(inclusive). User can omit
+either or both arguments.
 	`,
 	Example: `
 	tendermint reindex-event

cmd/tendermint/commands/rollback.go

@@ -0,0 +1,46 @@
+package commands
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/state"
+)
+
+var RollbackStateCmd = &cobra.Command{
+	Use:   "rollback",
+	Short: "rollback tendermint state by one height",
+	Long: `
+A state rollback is performed to recover from an incorrect application state transition,
+when Tendermint has persisted an incorrect app hash and is thus unable to make
+progress. Rollback overwrites a state at height n with the state at height n - 1.
+The application should also roll back to height n - 1. No blocks are removed, so upon
+restarting Tendermint the transactions in block n will be re-executed against the
+application.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		height, hash, err := RollbackState(config)
+		if err != nil {
+			return fmt.Errorf("failed to rollback state: %w", err)
+		}
+
+		fmt.Printf("Rolled back state to height %d and hash %v", height, hash)
+		return nil
+	},
+}
+
+// RollbackState takes the state at the current height n and overwrites it with the state
+// at height n - 1. Note state here refers to tendermint state not application state.
+// Returns the latest state height and app hash alongside an error if there was one.
+func RollbackState(config *cfg.Config) (int64, []byte, error) {
+	// use the parsed config to load the block and state store
+	blockStore, stateStore, err := loadStateAndBlockStore(config)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	// rollback the last state
+	return state.Rollback(blockStore, stateStore)
+}

cmd/tendermint/main.go

@@ -29,6 +29,7 @@ func main() {
 		cmd.GenNodeKeyCmd,
 		cmd.VersionCmd,
 		cmd.InspectCmd,
+		cmd.RollbackStateCmd,
 		cmd.MakeKeyMigrateCommand(),
 		debug.DebugCmd,
 		cli.NewCompletionCmd(rootCmd, true),

internal/state/rollback.go

@@ -0,0 +1,87 @@
+package state
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/tendermint/tendermint/version"
+)
+
+// Rollback overwrites the current Tendermint state (height n) with the most
+// recent previous state (height n - 1).
+// Note that this function does not affect application state.
+func Rollback(bs BlockStore, ss Store) (int64, []byte, error) {
+	invalidState, err := ss.Load()
+	if err != nil {
+		return -1, nil, err
+	}
+	if invalidState.IsEmpty() {
+		return -1, nil, errors.New("no state found")
+	}
+
+	rollbackHeight := invalidState.LastBlockHeight
+	rollbackBlock := bs.LoadBlockMeta(rollbackHeight)
+	if rollbackBlock == nil {
+		return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight)
+	}
+
+	previousValidatorSet, err := ss.LoadValidators(rollbackHeight - 1)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	previousParams, err := ss.LoadConsensusParams(rollbackHeight)
+	if err != nil {
+		return -1, nil, err
+	}
+
+	valChangeHeight := invalidState.LastHeightValidatorsChanged
+	// this can only happen if the validator set changed since the last block
+	if valChangeHeight > rollbackHeight {
+		valChangeHeight = rollbackHeight
+	}
+
+	paramsChangeHeight := invalidState.LastHeightConsensusParamsChanged
+	// this can only happen if params changed from the last block
+	if paramsChangeHeight > rollbackHeight {
+		paramsChangeHeight = rollbackHeight
+	}
+
+	// build the new state from the old state and the prior block
+	rolledBackState := State{
+		Version: Version{
+			Consensus: version.Consensus{
+				Block: version.BlockProtocol,
+				App:   previousParams.Version.AppVersion,
+			},
+			Software: version.TMVersion,
+		},
+		// immutable fields
+		ChainID:       invalidState.ChainID,
+		InitialHeight: invalidState.InitialHeight,
+
+		LastBlockHeight: invalidState.LastBlockHeight - 1,
+		LastBlockID:     rollbackBlock.Header.LastBlockID,
+		LastBlockTime:   rollbackBlock.Header.Time,
+
+		NextValidators:              invalidState.Validators,
+		Validators:                  invalidState.LastValidators,
+		LastValidators:              previousValidatorSet,
+		LastHeightValidatorsChanged: valChangeHeight,
+
+		ConsensusParams:                  previousParams,
+		LastHeightConsensusParamsChanged: paramsChangeHeight,
+
+		LastResultsHash: rollbackBlock.Header.LastResultsHash,
+		AppHash:         rollbackBlock.Header.AppHash,
+	}
+
+	// persist the new state. This overrides the invalid one. NOTE: this will also
+	// persist the validator set and consensus params over the existing structures,
+	// but both should be the same
+	if err := ss.Save(rolledBackState); err != nil {
+		return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err)
+	}
+
+	return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil
+}

internal/state/rollback_test.go

@@ -0,0 +1,146 @@
+package state_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
+
+	"github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/state/mocks"
+	"github.com/tendermint/tendermint/internal/test/factory"
+	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tendermint/version"
+)
+
+func TestRollback(t *testing.T) {
+	stateStore := state.NewStore(dbm.NewMemDB())
+	blockStore := &mocks.BlockStore{}
+	var (
+		height     int64  = 100
+		appVersion uint64 = 10
+	)
+
+	valSet, _ := factory.RandValidatorSet(5, 10)
+
+	params := types.DefaultConsensusParams()
+	params.Version.AppVersion = appVersion
+	newParams := types.DefaultConsensusParams()
+	newParams.Block.MaxBytes = 10000
+
+	initialState := state.State{
+		Version: state.Version{
+			Consensus: version.Consensus{
+				Block: version.BlockProtocol,
+				App:   10,
+			},
+			Software: version.TMVersion,
+		},
+		ChainID:                          factory.DefaultTestChainID,
+		InitialHeight:                    10,
+		LastBlockID:                      factory.MakeBlockID(),
+		AppHash:                          factory.RandomHash(),
+		LastResultsHash:                  factory.RandomHash(),
+		LastBlockHeight:                  height,
+		LastValidators:                   valSet,
+		Validators:                       valSet.CopyIncrementProposerPriority(1),
+		NextValidators:                   valSet.CopyIncrementProposerPriority(2),
+		LastHeightValidatorsChanged:      height + 1,
+		ConsensusParams:                  *params,
+		LastHeightConsensusParamsChanged: height + 1,
+	}
+	require.NoError(t, stateStore.Bootstrap(initialState))
+
+	height++
+	block := &types.BlockMeta{
+		Header: types.Header{
+			Height:          height,
+			AppHash:         initialState.AppHash,
+			LastBlockID:     initialState.LastBlockID,
+			LastResultsHash: initialState.LastResultsHash,
+		},
+	}
+	blockStore.On("LoadBlockMeta", height).Return(block)
+
+	appVersion++
+	newParams.Version.AppVersion = appVersion
+	nextState := initialState.Copy()
+	nextState.LastBlockHeight = height
+	nextState.Version.Consensus.App = appVersion
+	nextState.LastBlockID = factory.MakeBlockID()
+	nextState.AppHash = factory.RandomHash()
+	nextState.LastValidators = initialState.Validators
+	nextState.Validators = initialState.NextValidators
+	nextState.NextValidators = initialState.NextValidators.CopyIncrementProposerPriority(1)
+	nextState.ConsensusParams = *newParams
+	nextState.LastHeightConsensusParamsChanged = height + 1
+	nextState.LastHeightValidatorsChanged = height + 1
+
+	// update the state
+	require.NoError(t, stateStore.Save(nextState))
+
+	// rollback the state
+	rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore)
+	require.NoError(t, err)
+	require.EqualValues(t, int64(100), rollbackHeight)
+	require.EqualValues(t, initialState.AppHash, rollbackHash)
+	blockStore.AssertExpectations(t)
+
+	// assert that we've recovered the prior state
+	loadedState, err := stateStore.Load()
+	require.NoError(t, err)
+	require.EqualValues(t, initialState, loadedState)
+}
+
+func TestRollbackNoState(t *testing.T) {
+	stateStore := state.NewStore(dbm.NewMemDB())
+	blockStore := &mocks.BlockStore{}
+
+	_, _, err := state.Rollback(blockStore, stateStore)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "no state found")
+}
+
+func TestRollbackNoBlocks(t *testing.T) {
+	stateStore := state.NewStore(dbm.NewMemDB())
+	blockStore := &mocks.BlockStore{}
+	var (
+		height     int64  = 100
+		appVersion uint64 = 10
+	)
+
+	valSet, _ := factory.RandValidatorSet(5, 10)
+
+	params := types.DefaultConsensusParams()
+	params.Version.AppVersion = appVersion
+	newParams := types.DefaultConsensusParams()
+	newParams.Block.MaxBytes = 10000
+
+	initialState := state.State{
+		Version: state.Version{
+			Consensus: version.Consensus{
+				Block: version.BlockProtocol,
+				App:   10,
+			},
+			Software: version.TMVersion,
+		},
+		ChainID:                          factory.DefaultTestChainID,
+		InitialHeight:                    10,
+		LastBlockID:                      factory.MakeBlockID(),
+		AppHash:                          factory.RandomHash(),
+		LastResultsHash:                  factory.RandomHash(),
+		LastBlockHeight:                  height,
+		LastValidators:                   valSet,
+		Validators:                       valSet.CopyIncrementProposerPriority(1),
+		NextValidators:                   valSet.CopyIncrementProposerPriority(2),
+		LastHeightValidatorsChanged:      height + 1,
+		ConsensusParams:                  *params,
+		LastHeightConsensusParamsChanged: height + 1,
+	}
+	require.NoError(t, stateStore.Save(initialState))
+	blockStore.On("LoadBlockMeta", height).Return(nil)
+
+	_, _, err := state.Rollback(blockStore, stateStore)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "block at height 100 not found")
+}

internal/state/state.go

@@ -64,6 +64,8 @@ func VersionFromProto(v tmstate.Version) Version {
 // Instead, use state.Copy() or updateState(...).
 // NOTE: not goroutine-safe.
 type State struct {
+	// FIXME: This can be removed as TMVersion is a constant, and version.Consensus should
+	// eventually be replaced by VersionParams in ConsensusParams
 	Version Version
 
 	// immutable

test/e2e/app/app.go

@@ -11,6 +11,7 @@ import (
 	"github.com/tendermint/tendermint/abci/example/code"
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/proto/tendermint/types"
 	"github.com/tendermint/tendermint/version"
 )
 
@@ -116,6 +117,11 @@ func (app *Application) InitChain(req abci.RequestInitChain) abci.ResponseInitCh
 	}
 	resp := abci.ResponseInitChain{
 		AppHash: app.state.Hash,
+		ConsensusParams: &types.ConsensusParams{
+			Version: &types.VersionParams{
+				AppVersion: 1,
+			},
+		},
 	}
 	if resp.Validators, err = app.validatorUpdates(0); err != nil {
 		panic(err)