separate out replaying of blocks to application and run in OnStart

internal/consensus/replay.go

@@ -1,19 +1,12 @@
 package consensus
 
 import (
-	"bytes"
-	"context"
 	"fmt"
 	"hash/crc32"
 	"io"
 	"reflect"
 	"time"
 
-	abci "github.com/tendermint/tendermint/abci/types"
-	"github.com/tendermint/tendermint/crypto/merkle"
-	"github.com/tendermint/tendermint/libs/log"
-	"github.com/tendermint/tendermint/proxy"
-	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -191,358 +184,3 @@ func makeHeightSearchFunc(height int64) auto.SearchFunc {
 		}
 	}
 }*/
-
-//---------------------------------------------------
-// 2. Recover from failure while applying the block.
-// (by handshaking with the app to figure out where
-// we were last, and using the WAL to recover there.)
-//---------------------------------------------------
-
-type Handshaker struct {
-	stateStore   sm.Store
-	initialState sm.State
-	store        sm.BlockStore
-	eventBus     types.BlockEventPublisher
-	genDoc       *types.GenesisDoc
-	logger       log.Logger
-
-	nBlocks int // number of blocks applied to the state
-}
-
-func NewHandshaker(stateStore sm.Store, state sm.State,
-	store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker {
-
-	return &Handshaker{
-		stateStore:   stateStore,
-		initialState: state,
-		store:        store,
-		eventBus:     types.NopEventBus{},
-		genDoc:       genDoc,
-		logger:       log.NewNopLogger(),
-		nBlocks:      0,
-	}
-}
-
-func (h *Handshaker) SetLogger(l log.Logger) {
-	h.logger = l
-}
-
-// SetEventBus - sets the event bus for publishing block related events.
-// If not called, it defaults to types.NopEventBus.
-func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) {
-	h.eventBus = eventBus
-}
-
-// NBlocks returns the number of blocks applied to the state.
-func (h *Handshaker) NBlocks() int {
-	return h.nBlocks
-}
-
-// TODO: retry the handshake/replay if it fails ?
-func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
-
-	// Handshake is done via ABCI Info on the query conn.
-	res, err := proxyApp.Query().InfoSync(context.Background(), proxy.RequestInfo)
-	if err != nil {
-		return fmt.Errorf("error calling Info: %v", err)
-	}
-
-	blockHeight := res.LastBlockHeight
-	if blockHeight < 0 {
-		return fmt.Errorf("got a negative last block height (%d) from the app", blockHeight)
-	}
-	appHash := res.LastBlockAppHash
-
-	h.logger.Info("ABCI Handshake App Info",
-		"height", blockHeight,
-		"hash", appHash,
-		"software-version", res.Version,
-		"protocol-version", res.AppVersion,
-	)
-
-	// Only set the version if there is no existing state.
-	if h.initialState.LastBlockHeight == 0 {
-		h.initialState.Version.Consensus.App = res.AppVersion
-	}
-
-	// Replay blocks up to the latest in the blockstore.
-	_, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp)
-	if err != nil {
-		return fmt.Errorf("error on replay: %v", err)
-	}
-
-	h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced",
-		"appHeight", blockHeight, "appHash", appHash)
-
-	// TODO: (on restart) replay mempool
-
-	return nil
-}
-
-// ReplayBlocks replays all blocks since appBlockHeight and ensures the result
-// matches the current state.
-// Returns the final AppHash or an error.
-func (h *Handshaker) ReplayBlocks(
-	state sm.State,
-	appHash []byte,
-	appBlockHeight int64,
-	proxyApp proxy.AppConns,
-) ([]byte, error) {
-	storeBlockBase := h.store.Base()
-	storeBlockHeight := h.store.Height()
-	stateBlockHeight := state.LastBlockHeight
-	h.logger.Info(
-		"ABCI Replay Blocks",
-		"appHeight",
-		appBlockHeight,
-		"storeHeight",
-		storeBlockHeight,
-		"stateHeight",
-		stateBlockHeight)
-
-	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain.
-	if appBlockHeight == 0 {
-		validators := make([]*types.Validator, len(h.genDoc.Validators))
-		for i, val := range h.genDoc.Validators {
-			validators[i] = types.NewValidator(val.PubKey, val.Power)
-		}
-		validatorSet := types.NewValidatorSet(validators)
-		nextVals := types.TM2PB.ValidatorUpdates(validatorSet)
-		pbParams := h.genDoc.ConsensusParams.ToProto()
-		req := abci.RequestInitChain{
-			Time:            h.genDoc.GenesisTime,
-			ChainId:         h.genDoc.ChainID,
-			InitialHeight:   h.genDoc.InitialHeight,
-			ConsensusParams: &pbParams,
-			Validators:      nextVals,
-			AppStateBytes:   h.genDoc.AppState,
-		}
-		res, err := proxyApp.Consensus().InitChainSync(context.Background(), req)
-		if err != nil {
-			return nil, err
-		}
-
-		appHash = res.AppHash
-
-		if stateBlockHeight == 0 { // we only update state when we are in initial state
-			// If the app did not return an app hash, we keep the one set from the genesis doc in
-			// the state. We don't set appHash since we don't want the genesis doc app hash
-			// recorded in the genesis block. We should probably just remove GenesisDoc.AppHash.
-			if len(res.AppHash) > 0 {
-				state.AppHash = res.AppHash
-			}
-			// If the app returned validators or consensus params, update the state.
-			if len(res.Validators) > 0 {
-				vals, err := types.PB2TM.ValidatorUpdates(res.Validators)
-				if err != nil {
-					return nil, err
-				}
-				state.Validators = types.NewValidatorSet(vals)
-				state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1)
-			} else if len(h.genDoc.Validators) == 0 {
-				// If validator set is not set in genesis and still empty after InitChain, exit.
-				return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain")
-			}
-
-			if res.ConsensusParams != nil {
-				state.ConsensusParams = state.ConsensusParams.UpdateConsensusParams(res.ConsensusParams)
-				state.Version.Consensus.App = state.ConsensusParams.Version.AppVersion
-			}
-			// We update the last results hash with the empty hash, to conform with RFC-6962.
-			state.LastResultsHash = merkle.HashFromByteSlices(nil)
-			if err := h.stateStore.Save(state); err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	// First handle edge cases and constraints on the storeBlockHeight and storeBlockBase.
-	switch {
-	case storeBlockHeight == 0:
-		assertAppHashEqualsOneFromState(appHash, state)
-		return appHash, nil
-
-	case appBlockHeight == 0 && state.InitialHeight < storeBlockBase:
-		// the app has no state, and the block store is truncated above the initial height
-		return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
-
-	case appBlockHeight > 0 && appBlockHeight < storeBlockBase-1:
-		// the app is too far behind truncated store (can be 1 behind since we replay the next)
-		return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
-
-	case storeBlockHeight < appBlockHeight:
-		// the app should never be ahead of the store (but this is under app's control)
-		return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight}
-
-	case storeBlockHeight < stateBlockHeight:
-		// the state should never be ahead of the store (this is under tendermint's control)
-		panic(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
-
-	case storeBlockHeight > stateBlockHeight+1:
-		// store should be at most one ahead of the state (this is under tendermint's control)
-		panic(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
-	}
-
-	var err error
-	// Now either store is equal to state, or one ahead.
-	// For each, consider all cases of where the app could be, given app <= store
-	if storeBlockHeight == stateBlockHeight {
-		// Tendermint ran Commit and saved the state.
-		// Either the app is asking for replay, or we're all synced up.
-		if appBlockHeight < storeBlockHeight {
-			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
-			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false)
-
-		} else if appBlockHeight == storeBlockHeight {
-			// We're good!
-			assertAppHashEqualsOneFromState(appHash, state)
-			return appHash, nil
-		}
-
-	} else if storeBlockHeight == stateBlockHeight+1 {
-		// We saved the block in the store but haven't updated the state,
-		// so we'll need to replay a block using the WAL.
-		switch {
-		case appBlockHeight < stateBlockHeight:
-			// the app is further behind than it should be, so replay blocks
-			// but leave the last block to go through the WAL
-			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true)
-
-		case appBlockHeight == stateBlockHeight:
-			// We haven't run Commit (both the state and app are one block behind),
-			// so replayBlock with the real app.
-			// NOTE: We could instead use the cs.WAL on cs.Start,
-			// but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT
-			h.logger.Info("Replay last block using real app")
-			state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
-			return state.AppHash, err
-
-		case appBlockHeight == storeBlockHeight:
-			// We ran Commit, but didn't save the state, so replayBlock with mock app.
-			abciResponses, err := h.stateStore.LoadABCIResponses(storeBlockHeight)
-			if err != nil {
-				return nil, err
-			}
-			mockApp := newMockProxyApp(appHash, abciResponses)
-			h.logger.Info("Replay last block using mock app")
-			state, err = h.replayBlock(state, storeBlockHeight, mockApp)
-			return state.AppHash, err
-		}
-
-	}
-
-	panic(fmt.Sprintf("uncovered case! appHeight: %d, storeHeight: %d, stateHeight: %d",
-		appBlockHeight, storeBlockHeight, stateBlockHeight))
-}
-
-func (h *Handshaker) replayBlocks(
-	state sm.State,
-	proxyApp proxy.AppConns,
-	appBlockHeight,
-	storeBlockHeight int64,
-	mutateState bool) ([]byte, error) {
-	// App is further behind than it should be, so we need to replay blocks.
-	// We replay all blocks from appBlockHeight+1.
-	//
-	// Note that we don't have an old version of the state,
-	// so we by-pass state validation/mutation using sm.ExecCommitBlock.
-	// This also means we won't be saving validator sets if they change during this period.
-	// TODO: Load the historical information to fix this and just use state.ApplyBlock
-	//
-	// If mutateState == true, the final block is replayed with h.replayBlock()
-
-	var appHash []byte
-	var err error
-	finalBlock := storeBlockHeight
-	if mutateState {
-		finalBlock--
-	}
-	firstBlock := appBlockHeight + 1
-	if firstBlock == 1 {
-		firstBlock = state.InitialHeight
-	}
-	for i := firstBlock; i <= finalBlock; i++ {
-		h.logger.Info("Applying block", "height", i)
-		block := h.store.LoadBlock(i)
-		// Extra check to ensure the app was not changed in a way it shouldn't have.
-		if len(appHash) > 0 {
-			assertAppHashEqualsOneFromBlock(appHash, block)
-		}
-
-		if i == finalBlock && !mutateState {
-			// We emit events for the index services at the final block due to the sync issue when
-			// the node shutdown during the block committing status.
-			blockExec := sm.NewBlockExecutor(
-				h.stateStore, h.logger, proxyApp.Consensus(), emptyMempool{}, sm.EmptyEvidencePool{}, h.store)
-			blockExec.SetEventBus(h.eventBus)
-			appHash, err = sm.ExecCommitBlock(
-				blockExec, proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight, state)
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			appHash, err = sm.ExecCommitBlock(
-				nil, proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight, state)
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		h.nBlocks++
-	}
-
-	if mutateState {
-		// sync the final block
-		state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
-		if err != nil {
-			return nil, err
-		}
-		appHash = state.AppHash
-	}
-
-	assertAppHashEqualsOneFromState(appHash, state)
-	return appHash, nil
-}
-
-// ApplyBlock on the proxyApp with the last block.
-func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) {
-	block := h.store.LoadBlock(height)
-	meta := h.store.LoadBlockMeta(height)
-
-	// Use stubs for both mempool and evidence pool since no transactions nor
-	// evidence are needed here - block already exists.
-	blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{}, h.store)
-	blockExec.SetEventBus(h.eventBus)
-
-	var err error
-	state, err = blockExec.ApplyBlock(state, meta.BlockID, block)
-	if err != nil {
-		return sm.State{}, err
-	}
-
-	h.nBlocks++
-
-	return state, nil
-}
-
-func assertAppHashEqualsOneFromBlock(appHash []byte, block *types.Block) {
-	if !bytes.Equal(appHash, block.AppHash) {
-		panic(fmt.Sprintf(`block.AppHash does not match AppHash after replay. Got %X, expected %X.
-
-Block: %v
-`,
-			appHash, block.AppHash, block))
-	}
-}
-
-func assertAppHashEqualsOneFromState(appHash []byte, state sm.State) {
-	if !bytes.Equal(appHash, state.AppHash) {
-		panic(fmt.Sprintf(`state.AppHash does not match AppHash after replay. Got
-%X, expected %X.
-
-State: %v
-
-Did you reset Tendermint without resetting your application's data?`,
-			appHash, state.AppHash, state))
-	}
-}

internal/consensus/replay_file.go

@@ -323,13 +323,6 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 		tmos.Exit(fmt.Sprintf("Failed to start event bus: %v", err))
 	}
 
-	handshaker := NewHandshaker(stateStore, state, blockStore, gdoc)
-	handshaker.SetEventBus(eventBus)
-	err = handshaker.Handshake(proxyApp)
-	if err != nil {
-		tmos.Exit(fmt.Sprintf("Error on handshake: %v", err))
-	}
-
 	mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore)
 

internal/consensus/replay_stubs.go

@@ -6,8 +6,6 @@ import (
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/internal/libs/clist"
 	mempl "github.com/tendermint/tendermint/internal/mempool"
-	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
-	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -45,48 +43,3 @@ func (emptyMempool) TxsWaitChan() <-chan struct{} { return nil }
 
 func (emptyMempool) InitWAL() error { return nil }
 func (emptyMempool) CloseWAL()      {}
-
-//-----------------------------------------------------------------------------
-// mockProxyApp uses ABCIResponses to give the right results.
-//
-// Useful because we don't want to call Commit() twice for the same block on
-// the real app.
-
-func newMockProxyApp(appHash []byte, abciResponses *tmstate.ABCIResponses) proxy.AppConnConsensus {
-	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{
-		appHash:       appHash,
-		abciResponses: abciResponses,
-	})
-	cli, _ := clientCreator.NewABCIClient()
-	err := cli.Start()
-	if err != nil {
-		panic(err)
-	}
-	return proxy.NewAppConnConsensus(cli)
-}
-
-type mockProxyApp struct {
-	abci.BaseApplication
-
-	appHash       []byte
-	txCount       int
-	abciResponses *tmstate.ABCIResponses
-}
-
-func (mock *mockProxyApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx {
-	r := mock.abciResponses.DeliverTxs[mock.txCount]
-	mock.txCount++
-	if r == nil {
-		return abci.ResponseDeliverTx{}
-	}
-	return *r
-}
-
-func (mock *mockProxyApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
-	mock.txCount = 0
-	return *mock.abciResponses.EndBlock
-}
-
-func (mock *mockProxyApp) Commit() abci.ResponseCommit {
-	return abci.ResponseCommit{Data: mock.appHash}
-}

internal/consensus/replay_test.go

@@ -1,12 +1,10 @@
 package consensus
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -25,15 +23,11 @@ import (
 	"github.com/tendermint/tendermint/crypto"
 	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
 	mempl "github.com/tendermint/tendermint/internal/mempool"
-	"github.com/tendermint/tendermint/internal/test/factory"
 	"github.com/tendermint/tendermint/libs/log"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
-	"github.com/tendermint/tendermint/privval"
-	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
-	sf "github.com/tendermint/tendermint/state/test/factory"
 	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
 )
@@ -559,109 +553,6 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	return sim
 }
 
-// Sync from scratch
-func TestHandshakeReplayAll(t *testing.T) {
-	sim := setupSimulator(t)
-
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, 0, m, false)
-	}
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, 0, m, true)
-	}
-}
-
-// Sync many, not from scratch
-func TestHandshakeReplaySome(t *testing.T) {
-	sim := setupSimulator(t)
-
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, 2, m, false)
-	}
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, 2, m, true)
-	}
-}
-
-// Sync from lagging by one
-func TestHandshakeReplayOne(t *testing.T) {
-	sim := setupSimulator(t)
-
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, numBlocks-1, m, false)
-	}
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, numBlocks-1, m, true)
-	}
-}
-
-// Sync from caught up
-func TestHandshakeReplayNone(t *testing.T) {
-	sim := setupSimulator(t)
-
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, numBlocks, m, false)
-	}
-	for _, m := range modes {
-		testHandshakeReplay(t, sim, numBlocks, m, true)
-	}
-}
-
-// Test mockProxyApp should not panic when app return ABCIResponses with some empty ResponseDeliverTx
-func TestMockProxyApp(t *testing.T) {
-	sim := setupSimulator(t) // setup config and simulator
-	config := sim.Config
-	assert.NotNil(t, config)
-
-	logger := log.TestingLogger()
-	var validTxs, invalidTxs = 0, 0
-	txIndex := 0
-
-	assert.NotPanics(t, func() {
-		abciResWithEmptyDeliverTx := new(tmstate.ABCIResponses)
-		abciResWithEmptyDeliverTx.DeliverTxs = make([]*abci.ResponseDeliverTx, 0)
-		abciResWithEmptyDeliverTx.DeliverTxs = append(abciResWithEmptyDeliverTx.DeliverTxs, &abci.ResponseDeliverTx{})
-
-		// called when saveABCIResponses:
-		bytes, err := proto.Marshal(abciResWithEmptyDeliverTx)
-		require.NoError(t, err)
-		loadedAbciRes := new(tmstate.ABCIResponses)
-
-		// this also happens sm.LoadABCIResponses
-		err = proto.Unmarshal(bytes, loadedAbciRes)
-		require.NoError(t, err)
-
-		mock := newMockProxyApp([]byte("mock_hash"), loadedAbciRes)
-
-		abciRes := new(tmstate.ABCIResponses)
-		abciRes.DeliverTxs = make([]*abci.ResponseDeliverTx, len(loadedAbciRes.DeliverTxs))
-		// Execute transactions and get hash.
-		proxyCb := func(req *abci.Request, res *abci.Response) {
-			if r, ok := res.Value.(*abci.Response_DeliverTx); ok {
-				// TODO: make use of res.Log
-				// TODO: make use of this info
-				// Blocks may include invalid txs.
-				txRes := r.DeliverTx
-				if txRes.Code == abci.CodeTypeOK {
-					validTxs++
-				} else {
-					logger.Debug("Invalid tx", "code", txRes.Code, "log", txRes.Log)
-					invalidTxs++
-				}
-				abciRes.DeliverTxs[txIndex] = txRes
-				txIndex++
-			}
-		}
-		mock.SetResponseCallback(proxyCb)
-
-		someTx := []byte("tx")
-		_, err = mock.DeliverTxAsync(context.Background(), abci.RequestDeliverTx{Tx: someTx})
-		assert.NoError(t, err)
-	})
-	assert.True(t, validTxs == 1)
-	assert.True(t, invalidTxs == 0)
-}
-
 func tempWALWithData(data []byte) string {
 	walFile, err := ioutil.TempFile("", "wal")
 	if err != nil {
@@ -677,138 +568,6 @@ func tempWALWithData(data []byte) string {
 	return walFile.Name()
 }
 
-// Make some blocks. Start a fresh app and apply nBlocks blocks.
-// Then restart the app and sync it up with the remaining blocks
-func testHandshakeReplay(t *testing.T, sim *simulatorTestSuite, nBlocks int, mode uint, testValidatorsChange bool) {
-	var chain []*types.Block
-	var commits []*types.Commit
-	var store *mockBlockStore
-	var stateDB dbm.DB
-	var genesisState sm.State
-
-	config := sim.Config
-
-	if testValidatorsChange {
-		testConfig := ResetConfig(fmt.Sprintf("%s_%v_m", t.Name(), mode))
-		defer func() { _ = os.RemoveAll(testConfig.RootDir) }()
-		stateDB = dbm.NewMemDB()
-
-		genesisState = sim.GenesisState
-		config = sim.Config
-		chain = append([]*types.Block{}, sim.Chain...) // copy chain
-		commits = sim.Commits
-		store = newMockBlockStore(config, genesisState.ConsensusParams)
-	} else { // test single node
-		testConfig := ResetConfig(fmt.Sprintf("%s_%v_s", t.Name(), mode))
-		defer func() { _ = os.RemoveAll(testConfig.RootDir) }()
-		walBody, err := WALWithNBlocks(t, numBlocks)
-		require.NoError(t, err)
-		walFile := tempWALWithData(walBody)
-		config.Consensus.SetWalFile(walFile)
-
-		privVal, err := privval.LoadFilePV(config.PrivValidator.KeyFile(), config.PrivValidator.StateFile())
-		require.NoError(t, err)
-
-		wal, err := NewWAL(walFile)
-		require.NoError(t, err)
-		wal.SetLogger(log.TestingLogger())
-		err = wal.Start()
-		require.NoError(t, err)
-		t.Cleanup(func() {
-			if err := wal.Stop(); err != nil {
-				t.Error(err)
-			}
-		})
-		chain, commits, err = makeBlockchainFromWAL(wal)
-		require.NoError(t, err)
-		pubKey, err := privVal.GetPubKey(context.Background())
-		require.NoError(t, err)
-		stateDB, genesisState, store = stateAndStore(config, pubKey, kvstore.ProtocolVersion)
-
-	}
-	stateStore := sm.NewStore(stateDB)
-	store.chain = chain
-	store.commits = commits
-
-	state := genesisState.Copy()
-	// run the chain through state.ApplyBlock to build up the tendermint state
-	state = buildTMStateFromChain(config, sim.Mempool, sim.Evpool, stateStore, state, chain, nBlocks, mode, store)
-	latestAppHash := state.AppHash
-
-	// make a new client creator
-	kvstoreApp := kvstore.NewPersistentKVStoreApplication(
-		filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_a_r%d", nBlocks, mode, rand.Int())))
-	t.Cleanup(func() { require.NoError(t, kvstoreApp.Close()) })
-
-	clientCreator2 := proxy.NewLocalClientCreator(kvstoreApp)
-	if nBlocks > 0 {
-		// run nBlocks against a new client to build up the app state.
-		// use a throwaway tendermint state
-		proxyApp := proxy.NewAppConns(clientCreator2)
-		stateDB1 := dbm.NewMemDB()
-		stateStore := sm.NewStore(stateDB1)
-		err := stateStore.Save(genesisState)
-		require.NoError(t, err)
-		buildAppStateFromChain(proxyApp, stateStore, sim.Mempool, sim.Evpool, genesisState, chain, nBlocks, mode, store)
-	}
-
-	// Prune block store if requested
-	expectError := false
-	if mode == 3 {
-		pruned, err := store.PruneBlocks(2)
-		require.NoError(t, err)
-		require.EqualValues(t, 1, pruned)
-		expectError = int64(nBlocks) < 2
-	}
-
-	// now start the app using the handshake - it should sync
-	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
-	handshaker := NewHandshaker(stateStore, state, store, genDoc)
-	proxyApp := proxy.NewAppConns(clientCreator2)
-	if err := proxyApp.Start(); err != nil {
-		t.Fatalf("Error starting proxy app connections: %v", err)
-	}
-
-	t.Cleanup(func() {
-		if err := proxyApp.Stop(); err != nil {
-			t.Error(err)
-		}
-	})
-
-	err := handshaker.Handshake(proxyApp)
-	if expectError {
-		require.Error(t, err)
-		return
-	} else if err != nil {
-		t.Fatalf("Error on abci handshake: %v", err)
-	}
-
-	// get the latest app hash from the app
-	res, err := proxyApp.Query().InfoSync(context.Background(), abci.RequestInfo{Version: ""})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// the app hash should be synced up
-	if !bytes.Equal(latestAppHash, res.LastBlockAppHash) {
-		t.Fatalf(
-			"Expected app hashes to match after handshake/replay. got %X, expected %X",
-			res.LastBlockAppHash,
-			latestAppHash)
-	}
-
-	expectedBlocksToSync := numBlocks - nBlocks
-	if nBlocks == numBlocks && mode > 0 {
-		expectedBlocksToSync++
-	} else if nBlocks > 0 && mode == 1 {
-		expectedBlocksToSync++
-	}
-
-	if handshaker.NBlocks() != expectedBlocksToSync {
-		t.Fatalf("Expected handshake to sync %d blocks, got %d", expectedBlocksToSync, handshaker.NBlocks())
-	}
-}
-
 func applyBlock(stateStore sm.Store,
 	mempool mempl.Mempool,
 	evpool sm.EvidencePool,
@@ -932,75 +691,6 @@ func buildTMStateFromChain(
 	return state
 }
 
-func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) {
-	// 1. Initialize tendermint and commit 3 blocks with the following app hashes:
-	//		- 0x01
-	//		- 0x02
-	//		- 0x03
-	config := ResetConfig("handshake_test_")
-	t.Cleanup(func() { os.RemoveAll(config.RootDir) })
-	privVal, err := privval.LoadFilePV(config.PrivValidator.KeyFile(), config.PrivValidator.StateFile())
-	require.NoError(t, err)
-	const appVersion = 0x0
-	pubKey, err := privVal.GetPubKey(context.Background())
-	require.NoError(t, err)
-	stateDB, state, store := stateAndStore(config, pubKey, appVersion)
-	stateStore := sm.NewStore(stateDB)
-	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
-	state.LastValidators = state.Validators.Copy()
-	// mode = 0 for committing all the blocks
-	blocks := sf.MakeBlocks(3, &state, privVal)
-	store.chain = blocks
-
-	// 2. Tendermint must panic if app returns wrong hash for the first block
-	//		- RANDOM HASH
-	//		- 0x02
-	//		- 0x03
-	{
-		app := &badApp{numBlocks: 3, allHashesAreWrong: true}
-		clientCreator := proxy.NewLocalClientCreator(app)
-		proxyApp := proxy.NewAppConns(clientCreator)
-		err := proxyApp.Start()
-		require.NoError(t, err)
-		t.Cleanup(func() {
-			if err := proxyApp.Stop(); err != nil {
-				t.Error(err)
-			}
-		})
-
-		assert.Panics(t, func() {
-			h := NewHandshaker(stateStore, state, store, genDoc)
-			if err = h.Handshake(proxyApp); err != nil {
-				t.Log(err)
-			}
-		})
-	}
-
-	// 3. Tendermint must panic if app returns wrong hash for the last block
-	//		- 0x01
-	//		- 0x02
-	//		- RANDOM HASH
-	{
-		app := &badApp{numBlocks: 3, onlyLastHashIsWrong: true}
-		clientCreator := proxy.NewLocalClientCreator(app)
-		proxyApp := proxy.NewAppConns(clientCreator)
-		err := proxyApp.Start()
-		require.NoError(t, err)
-		t.Cleanup(func() {
-			if err := proxyApp.Stop(); err != nil {
-				t.Error(err)
-			}
-		})
-
-		assert.Panics(t, func() {
-			h := NewHandshaker(stateStore, state, store, genDoc)
-			if err = h.Handshake(proxyApp); err != nil {
-				t.Log(err)
-			}
-		})
-	}
-}
-
 type badApp struct {
 	abci.BaseApplication
 	numBlocks           byte
@@ -1218,52 +908,6 @@ func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) {
 	return pruned, nil
 }
 
-//---------------------------------------
-// Test handshake/init chain
-
-func TestHandshakeUpdatesValidators(t *testing.T) {
-	val, _ := factory.RandValidator(true, 10)
-	vals := types.NewValidatorSet([]*types.Validator{val})
-	app := &initChainApp{vals: types.TM2PB.ValidatorUpdates(vals)}
-	clientCreator := proxy.NewLocalClientCreator(app)
-
-	config := ResetConfig("handshake_test_")
-	t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) })
-
-	privVal, err := privval.LoadFilePV(config.PrivValidator.KeyFile(), config.PrivValidator.StateFile())
-	require.NoError(t, err)
-	pubKey, err := privVal.GetPubKey(context.Background())
-	require.NoError(t, err)
-	stateDB, state, store := stateAndStore(config, pubKey, 0x0)
-	stateStore := sm.NewStore(stateDB)
-
-	oldValAddr := state.Validators.Validators[0].Address
-
-	// now start the app using the handshake - it should sync
-	genDoc, _ := sm.MakeGenesisDocFromFile(config.GenesisFile())
-	handshaker := NewHandshaker(stateStore, state, store, genDoc)
-	proxyApp := proxy.NewAppConns(clientCreator)
-	if err := proxyApp.Start(); err != nil {
-		t.Fatalf("Error starting proxy app connections: %v", err)
-	}
-	t.Cleanup(func() {
-		if err := proxyApp.Stop(); err != nil {
-			t.Error(err)
-		}
-	})
-	if err := handshaker.Handshake(proxyApp); err != nil {
-		t.Fatalf("Error on abci handshake: %v", err)
-	}
-	// reload the state, check the validator set was updated
-	state, err = stateStore.Load()
-	require.NoError(t, err)
-
-	newValAddr := state.Validators.Validators[0].Address
-	expectValAddr := val.Address
-	assert.NotEqual(t, oldValAddr, newValAddr)
-	assert.Equal(t, newValAddr, expectValAddr)
-}
-
 // returns the vals on InitChain
 type initChainApp struct {
 	abci.BaseApplication