Backport of sam/abci-responses (#9090) (#9159)

*backport of sam/abci-responses

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>
This commit is contained in:
samricotta
2022-08-18 10:53:58 +02:00
committed by Sam Ricotta
co-authored by William Banfield
parent 1069ffc6aa
commit 4be78e3125
92 changed files with 1044 additions and 495 deletions
+6 -1
View File
@@ -1,6 +1,9 @@
package state
import "fmt"
import (
"errors"
"fmt"
)
type (
ErrInvalidBlock error
@@ -99,3 +102,5 @@ func (e ErrNoConsensusParamsForHeight) Error() string {
func (e ErrNoABCIResponsesForHeight) Error() string {
return fmt.Sprintf("could not find results for height #%d", e.Height)
}
var ErrABCIResponsesNotPersisted = errors.New("node is not persisting abci responses")
+15 -5
View File
@@ -41,7 +41,9 @@ func TestApplyBlock(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // ignore for tests
state, stateDB, _ := makeState(1, 1)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
mmock.Mempool{}, sm.EmptyEvidencePool{})
@@ -67,7 +69,9 @@ func TestBeginBlockValidators(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // no need to check error again
state, stateDB, _ := makeState(2, 2)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
prevHash := state.LastBlockID.Hash
prevParts := types.PartSetHeader{}
@@ -130,7 +134,9 @@ func TestBeginBlockByzantineValidators(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // ignore for tests
state, stateDB, privVals := makeState(1, 1)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
defaultEvidenceTime := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)
privVal := privVals[state.Validators.Validators[0].Address.String()]
@@ -354,7 +360,9 @@ func TestEndBlockValidatorUpdates(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // ignore for tests
state, stateDB, _ := makeState(1, 1)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
blockExec := sm.NewBlockExecutor(
stateStore,
@@ -425,7 +433,9 @@ func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // ignore for tests
state, stateDB, _ := makeState(1, 1)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
blockExec := sm.NewBlockExecutor(
stateStore,
log.TestingLogger(),
+1 -1
View File
@@ -43,6 +43,6 @@ func ValidateValidatorUpdates(abciUpdates []abci.ValidatorUpdate, params tmproto
// SaveValidatorsInfo is an alias for the private saveValidatorsInfo method in
// store.go, exported exclusively and explicitly for testing.
func SaveValidatorsInfo(db dbm.DB, height, lastHeightChanged int64, valSet *types.ValidatorSet) error {
stateStore := dbStore{db}
stateStore := dbStore{db, StoreOptions{DiscardABCIResponses: false}}
return stateStore.saveValidatorsInfo(height, lastHeightChanged, valSet)
}
+3 -1
View File
@@ -115,7 +115,9 @@ func makeState(nVals, height int) (sm.State, dbm.DB, map[string]types.PrivValida
})
stateDB := dbm.NewMemDB()
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
if err := stateStore.Save(s); err != nil {
panic(err)
}
+23
View File
@@ -153,6 +153,29 @@ func (_m *Store) LoadFromDBOrGenesisFile(_a0 string) (state.State, error) {
return r0, r1
}
// LoadLastABCIResponse provides a mock function with given fields: _a0
func (_m *Store) LoadLastABCIResponse(_a0 int64) (*tendermintstate.ABCIResponses, error) {
ret := _m.Called(_a0)
var r0 *tendermintstate.ABCIResponses
if rf, ok := ret.Get(0).(func(int64) *tendermintstate.ABCIResponses); ok {
r0 = rf(_a0)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*tendermintstate.ABCIResponses)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(int64) error); ok {
r1 = rf(_a0)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// LoadValidators provides a mock function with given fields: _a0
func (_m *Store) LoadValidators(_a0 int64) (*tenderminttypes.ValidatorSet, error) {
ret := _m.Called(_a0)
+5 -2
View File
@@ -82,7 +82,10 @@ func TestRollback(t *testing.T) {
}
func TestRollbackNoState(t *testing.T) {
stateStore := state.NewStore(dbm.NewMemDB())
stateStore := state.NewStore(dbm.NewMemDB(),
state.StoreOptions{
DiscardABCIResponses: false,
})
blockStore := &mocks.BlockStore{}
_, _, err := state.Rollback(blockStore, stateStore)
@@ -115,7 +118,7 @@ func TestRollbackDifferentStateHeight(t *testing.T) {
}
func setupStateStore(t *testing.T, height int64) state.Store {
stateStore := state.NewStore(dbm.NewMemDB())
stateStore := state.NewStore(dbm.NewMemDB(), state.StoreOptions{DiscardABCIResponses: false})
valSet, _ := types.RandValidatorSet(5, 10)
params := types.DefaultConsensusParams()
+1 -1
View File
@@ -17,7 +17,7 @@ import (
"github.com/tendermint/tendermint/version"
)
// database keys
// database key
var (
stateKey = []byte("stateKey")
)
+27 -9
View File
@@ -29,7 +29,9 @@ func setupTestCase(t *testing.T) (func(t *testing.T), dbm.DB, sm.State) {
config := cfg.ResetTestRoot("state_")
dbType := dbm.BackendType(config.DBBackend)
stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
require.NoError(t, err)
state, err := stateStore.LoadFromDBOrGenesisFile(config.GenesisFile())
assert.NoError(t, err, "expected no error on LoadStateFromDBOrGenesisFile")
@@ -76,7 +78,9 @@ func TestMakeGenesisStateNilValidators(t *testing.T) {
func TestStateSaveLoad(t *testing.T) {
tearDown, stateDB, state := setupTestCase(t)
defer tearDown(t)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
assert := assert.New(t)
state.LastBlockHeight++
@@ -95,7 +99,9 @@ func TestStateSaveLoad(t *testing.T) {
func TestABCIResponsesSaveLoad1(t *testing.T) {
tearDown, stateDB, state := setupTestCase(t)
defer tearDown(t)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
assert := assert.New(t)
state.LastBlockHeight++
@@ -128,7 +134,9 @@ func TestABCIResponsesSaveLoad2(t *testing.T) {
defer tearDown(t)
assert := assert.New(t)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
cases := [...]struct {
// Height is implied to equal index+2,
@@ -216,7 +224,9 @@ func TestValidatorSimpleSaveLoad(t *testing.T) {
defer tearDown(t)
assert := assert.New(t)
statestore := sm.NewStore(stateDB)
statestore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
// Can't load anything for height 0.
_, err := statestore.LoadValidators(0)
@@ -249,7 +259,9 @@ func TestValidatorSimpleSaveLoad(t *testing.T) {
func TestOneValidatorChangesSaveLoad(t *testing.T) {
tearDown, stateDB, state := setupTestCase(t)
defer tearDown(t)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
// Change vals at these heights.
changeHeights := []int64{1, 2, 4, 5, 10, 15, 16, 17, 20}
@@ -901,7 +913,9 @@ func TestStoreLoadValidatorsIncrementsProposerPriority(t *testing.T) {
const valSetSize = 2
tearDown, stateDB, state := setupTestCase(t)
t.Cleanup(func() { tearDown(t) })
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
state.Validators = genValSet(valSetSize)
state.NextValidators = state.Validators.CopyIncrementProposerPriority(1)
err := stateStore.Save(state)
@@ -926,7 +940,9 @@ func TestManyValidatorChangesSaveLoad(t *testing.T) {
const valSetSize = 7
tearDown, stateDB, state := setupTestCase(t)
defer tearDown(t)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
require.Equal(t, int64(0), state.LastBlockHeight)
state.Validators = genValSet(valSetSize)
state.NextValidators = state.Validators.CopyIncrementProposerPriority(1)
@@ -990,7 +1006,9 @@ func TestConsensusParamsChangesSaveLoad(t *testing.T) {
tearDown, stateDB, state := setupTestCase(t)
defer tearDown(t)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
// Change vals at these heights.
changeHeights := []int64{1, 2, 4, 5, 10, 15, 16, 17, 20}
+80 -18
View File
@@ -37,6 +37,10 @@ func calcABCIResponsesKey(height int64) []byte {
return []byte(fmt.Sprintf("abciResponsesKey:%v", height))
}
func calcLastABCIResponsesKey(height int64) []byte {
return []byte(fmt.Sprintf("lastABCIResponsesKey:%v", height))
}
//----------------------
//go:generate ../scripts/mockery_generate.sh Store
@@ -58,6 +62,8 @@ type Store interface {
LoadValidators(int64) (*types.ValidatorSet, error)
// LoadABCIResponses loads the abciResponse for a given height
LoadABCIResponses(int64) (*tmstate.ABCIResponses, error)
// LoadLastABCIResponse loads the last abciResponse for a given height
LoadLastABCIResponse(int64) (*tmstate.ABCIResponses, error)
// LoadConsensusParams loads the consensus params for a given height
LoadConsensusParams(int64) (tmproto.ConsensusParams, error)
// Save overwrites the previous state with the updated one
@@ -75,13 +81,24 @@ type Store interface {
// dbStore wraps a db (github.com/tendermint/tm-db)
type dbStore struct {
db dbm.DB
StoreOptions
}
type StoreOptions struct {
// DiscardABCIResponses determines whether or not the store
// retains all ABCIResponses. If DiscardABCiResponses is enabled,
// the store will maintain only the response object from the latest
// height.
DiscardABCIResponses bool
}
var _ Store = (*dbStore)(nil)
// NewStore creates the dbStore of the state pkg.
func NewStore(db dbm.DB) Store {
return dbStore{db}
func NewStore(db dbm.DB, options StoreOptions) Store {
return dbStore{db, options}
}
// LoadStateFromDBOrGenesisFile loads the most recent state from the database,
@@ -358,12 +375,13 @@ func ABCIResponsesResultsHash(ar *tmstate.ABCIResponses) []byte {
}
// LoadABCIResponses loads the ABCIResponses for the given height from the
// database. If not found, ErrNoABCIResponsesForHeight is returned.
//
// This is useful for recovering from crashes where we called app.Commit and
// before we called s.Save(). It can also be used to produce Merkle proofs of
// the result of txs.
// database. If the node has DiscardABCIResponses set to true, ErrABCIResponsesNotPersisted
// is persisted. If not found, ErrNoABCIResponsesForHeight is returned.
func (store dbStore) LoadABCIResponses(height int64) (*tmstate.ABCIResponses, error) {
if store.DiscardABCIResponses {
return nil, ErrABCIResponsesNotPersisted
}
buf, err := store.db.Get(calcABCIResponsesKey(height))
if err != nil {
return nil, err
@@ -385,12 +403,43 @@ func (store dbStore) LoadABCIResponses(height int64) (*tmstate.ABCIResponses, er
return abciResponses, nil
}
// LoadLastABCIResponses loads the ABCIResponses from the most recent height.
// The height parameter is used to ensure that the response corresponds to the latest height.
// If not, an error is returned.
//
// This method is used for recovering in the case that we called the Commit ABCI
// method on the application but crashed before persisting the results.
func (store dbStore) LoadLastABCIResponse(height int64) (*tmstate.ABCIResponses, error) {
bz, err := store.db.Get(calcLastABCIResponsesKey(height))
if err != nil {
return nil, err
}
if len(bz) == 0 {
return nil, errors.New("no last ABCI response has been persisted")
}
abciResponse := new(tmstate.ABCIResponsesInfo)
err = abciResponse.Unmarshal(bz)
if err != nil {
tmos.Exit(fmt.Sprintf(`LoadLastABCIResponses: Data has been corrupted or its spec has
changed: %v\n`, err))
}
// Here we validate the result by comparing its height to the expected height.
if height != abciResponse.GetHeight() {
return nil, errors.New("expected height %d but last stored abci responses was at height %d")
}
return abciResponse.AbciResponses, nil
}
// SaveABCIResponses persists the ABCIResponses to the database.
// This is useful in case we crash after app.Commit and before s.Save().
// Responses are indexed by height so they can also be loaded later to produce
// Merkle proofs.
//
// Exposed for testing.
// CONTRACT: height must be monotonically increasing every time this is called.
func (store dbStore) SaveABCIResponses(height int64, abciResponses *tmstate.ABCIResponses) error {
var dtxs []*abci.ResponseDeliverTx
// strip nil values,
@@ -401,17 +450,30 @@ func (store dbStore) SaveABCIResponses(height int64, abciResponses *tmstate.ABCI
}
abciResponses.DeliverTxs = dtxs
bz, err := abciResponses.Marshal()
// If the flag is false then we save the ABCIResponse. This can be used for the /BlockResults
// query or to reindex an event using the command line.
if !store.DiscardABCIResponses {
bz, err := abciResponses.Marshal()
if err != nil {
return err
}
if err := store.db.Set(calcABCIResponsesKey(height), bz); err != nil {
return err
}
}
// We always save the last ABCI response for crash recovery.
// This overwrites the previous saved ABCI Response.
response := &tmstate.ABCIResponsesInfo{
AbciResponses: abciResponses,
Height: height,
}
bz, err := response.Marshal()
if err != nil {
return err
}
err = store.db.SetSync(calcABCIResponsesKey(height), bz)
if err != nil {
return err
}
return nil
return store.db.SetSync(calcLastABCIResponsesKey(height), bz)
}
//-----------------------------------------------------------------------------
@@ -471,7 +533,7 @@ func loadValidatorsInfo(db dbm.DB, height int64) (*tmstate.ValidatorsInfo, error
}
if len(buf) == 0 {
return nil, errors.New("value retrieved from db is empty")
return nil, errors.New("no last ABCI response has been persisted")
}
v := new(tmstate.ValidatorsInfo)
@@ -479,7 +541,7 @@ func loadValidatorsInfo(db dbm.DB, height int64) (*tmstate.ValidatorsInfo, error
if err != nil {
// DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED
tmos.Exit(fmt.Sprintf(`LoadValidators: Data has been corrupted or its spec has changed:
%v\n`, err))
%v\n`, err))
}
// TODO: ensure that buf is completely read.
@@ -557,7 +619,7 @@ func (store dbStore) loadConsensusParamsInfo(height int64) (*tmstate.ConsensusPa
return nil, err
}
if len(buf) == 0 {
return nil, errors.New("value retrieved from db is empty")
return nil, errors.New("no last ABCI response has been persisted")
}
paramsInfo := new(tmstate.ConsensusParamsInfo)
+78 -3
View File
@@ -23,7 +23,9 @@ import (
func TestStoreLoadValidators(t *testing.T) {
stateDB := dbm.NewMemDB()
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
val, _ := types.RandValidator(true, 10)
vals := types.NewValidatorSet([]*types.Validator{val})
@@ -54,7 +56,9 @@ func BenchmarkLoadValidators(b *testing.B) {
dbType := dbm.BackendType(config.DBBackend)
stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
require.NoError(b, err)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
state, err := stateStore.LoadFromDBOrGenesisFile(config.GenesisFile())
if err != nil {
b.Fatal(err)
@@ -107,7 +111,9 @@ func TestPruneStates(t *testing.T) {
tc := tc
t.Run(name, func(t *testing.T) {
db := dbm.NewMemDB()
stateStore := sm.NewStore(db)
stateStore := sm.NewStore(db, sm.StoreOptions{
DiscardABCIResponses: false,
})
pk := ed25519.GenPrivKey().PubKey()
// Generate a bunch of state data. Validators change for heights ending with 3, and
@@ -229,3 +235,72 @@ func sliceToMap(s []int64) map[int64]bool {
}
return m
}
func TestLastABCIResponses(t *testing.T) {
// create an empty state store.
t.Run("Not persisting responses", func(t *testing.T) {
stateDB := dbm.NewMemDB()
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
responses, err := stateStore.LoadABCIResponses(1)
require.Error(t, err)
require.Nil(t, responses)
// stub the abciresponses.
response1 := &tmstate.ABCIResponses{
BeginBlock: &abci.ResponseBeginBlock{},
DeliverTxs: []*abci.ResponseDeliverTx{
{Code: 32, Data: []byte("Hello"), Log: "Huh?"},
},
EndBlock: &abci.ResponseEndBlock{},
}
// create new db and state store and set discard abciresponses to false.
stateDB = dbm.NewMemDB()
stateStore = sm.NewStore(stateDB, sm.StoreOptions{DiscardABCIResponses: false})
height := int64(10)
// save the last abci response.
err = stateStore.SaveABCIResponses(height, response1)
require.NoError(t, err)
// search for the last abciresponse and check if it has saved.
lastResponse, err := stateStore.LoadLastABCIResponse(height)
require.NoError(t, err)
// check to see if the saved response height is the same as the loaded height.
assert.Equal(t, lastResponse, response1)
// use an incorret height to make sure the state store errors.
_, err = stateStore.LoadLastABCIResponse(height + 1)
assert.Error(t, err)
// check if the abci response didnt save in the abciresponses.
responses, err = stateStore.LoadABCIResponses(height)
require.NoError(t, err, responses)
require.Equal(t, response1, responses)
})
t.Run("persisting responses", func(t *testing.T) {
stateDB := dbm.NewMemDB()
height := int64(10)
// stub the second abciresponse.
response2 := &tmstate.ABCIResponses{
BeginBlock: &abci.ResponseBeginBlock{},
DeliverTxs: []*abci.ResponseDeliverTx{
{Code: 44, Data: []byte("Hello again"), Log: "????"},
},
EndBlock: &abci.ResponseEndBlock{},
}
// create a new statestore with the responses on.
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: true,
})
// save an additional response.
err := stateStore.SaveABCIResponses(height+1, response2)
require.NoError(t, err)
// check to see if the response saved by calling the last response.
lastResponse2, err := stateStore.LoadLastABCIResponse(height + 1)
require.NoError(t, err)
// check to see if the saved response height is the same as the loaded height.
assert.Equal(t, response2, lastResponse2)
// should error as we are no longer saving the response.
_, err = stateStore.LoadABCIResponses(height + 1)
assert.Equal(t, sm.ErrABCIResponsesNotPersisted, err)
})
}
+3 -1
View File
@@ -33,7 +33,9 @@ func TestTxFilter(t *testing.T) {
for i, tc := range testCases {
stateDB, err := dbm.NewDB("state", "memdb", os.TempDir())
require.NoError(t, err)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc)
require.NoError(t, err)
+9 -3
View File
@@ -28,7 +28,9 @@ func TestValidateBlockHeader(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // ignore for tests
state, stateDB, privVals := makeState(3, 1)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
blockExec := sm.NewBlockExecutor(
stateStore,
log.TestingLogger(),
@@ -99,7 +101,9 @@ func TestValidateBlockCommit(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // ignore for tests
state, stateDB, privVals := makeState(1, 1)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
blockExec := sm.NewBlockExecutor(
stateStore,
log.TestingLogger(),
@@ -213,7 +217,9 @@ func TestValidateBlockEvidence(t *testing.T) {
defer proxyApp.Stop() //nolint:errcheck // ignore for tests
state, stateDB, privVals := makeState(4, 1)
stateStore := sm.NewStore(stateDB)
stateStore := sm.NewStore(stateDB, sm.StoreOptions{
DiscardABCIResponses: false,
})
defaultEvidenceTime := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)
evpool := &mocks.EvidencePool{}