From 1a86c869e86852bc00c4a9a5d026b84f84028095 Mon Sep 17 00:00:00 2001 From: jaekwon Date: Wed, 17 Apr 2019 11:27:37 -0700 Subject: [PATCH] WIP --- lite/base_verifier.go | 6 +- lite/base_verifier_test.go | 2 +- lite/dynamic_verifier.go | 275 ------------ lite/errors/errors.go | 21 + lite/provider.go | 105 +++++ lite/proxy/verifier.go | 41 -- lite/proxy/wrapper.go | 6 +- lite/types.go | 3 + lite/verifying/provider.go | 394 ++++++++++++++++++ .../provider_test.go} | 140 ++++--- types/block.go | 33 ++ types/validator_set.go | 30 +- 12 files changed, 651 insertions(+), 405 deletions(-) delete mode 100644 lite/dynamic_verifier.go delete mode 100644 lite/proxy/verifier.go create mode 100644 lite/verifying/provider.go rename lite/{dynamic_verifier_test.go => verifying/provider_test.go} (62%) diff --git a/lite/base_verifier.go b/lite/base_verifier.go index 9eb880bb2..58227f5a7 100644 --- a/lite/base_verifier.go +++ b/lite/base_verifier.go @@ -14,7 +14,11 @@ var _ Verifier = (*BaseVerifier)(nil) // later, requiring sufficient votes (> 2/3) from the given valset. // To verify blocks produced by a blockchain with mutable validator sets, // use the DynamicVerifier. -// TODO: Handle unbonding time. +// +// NOTE: Verifier as a supported interface is deprecated, it may be reasonable +// to rename this to simply "verifier" and to remove that intereface +// declaration. See also VerifyingProvider, which is not a Verifier, but is a +// Provider. type BaseVerifier struct { chainID string height int64 diff --git a/lite/base_verifier_test.go b/lite/base_verifier_test.go index 2ef1203fb..0178c4113 100644 --- a/lite/base_verifier_test.go +++ b/lite/base_verifier_test.go @@ -9,7 +9,7 @@ import ( "github.com/tendermint/tendermint/types" ) -func TestBaseCert(t *testing.T) { +func TestBaseVerifier(t *testing.T) { assert := assert.New(t) keys := genPrivKeys(4) diff --git a/lite/dynamic_verifier.go b/lite/dynamic_verifier.go deleted file mode 100644 index 8b69d2d7c..000000000 --- a/lite/dynamic_verifier.go +++ /dev/null @@ -1,275 +0,0 @@ -package lite - -import ( - "bytes" - "fmt" - "sync" - - log "github.com/tendermint/tendermint/libs/log" - lerr "github.com/tendermint/tendermint/lite/errors" - "github.com/tendermint/tendermint/types" -) - -const sizeOfPendingMap = 1024 - -var _ Verifier = (*DynamicVerifier)(nil) - -// DynamicVerifier implements an auto-updating Verifier. It uses a -// "source" provider to obtain the needed FullCommits to securely sync with -// validator set changes. It stores properly validated data on the -// "trusted" local system. -// TODO: make this single threaded and create a new -// ConcurrentDynamicVerifier that wraps it with concurrency. -// see https://github.com/tendermint/tendermint/issues/3170 -type DynamicVerifier struct { - chainID string - logger log.Logger - - // Already validated, stored locally - trusted PersistentProvider - - // New info, like a node rpc, or other import method. - source Provider - - // pending map to synchronize concurrent verification requests - mtx sync.Mutex - pendingVerifications map[int64]chan struct{} -} - -// NewDynamicVerifier returns a new DynamicVerifier. It uses the -// trusted provider to store validated data and the source provider to -// obtain missing data (e.g. FullCommits). -// -// The trusted provider should be a DBProvider. -// The source provider should be a client.HTTPProvider. -func NewDynamicVerifier(chainID string, trusted PersistentProvider, source Provider) *DynamicVerifier { - return &DynamicVerifier{ - logger: log.NewNopLogger(), - chainID: chainID, - trusted: trusted, - source: source, - pendingVerifications: make(map[int64]chan struct{}, sizeOfPendingMap), - } -} - -func (dv *DynamicVerifier) SetLogger(logger log.Logger) { - logger = logger.With("module", "lite") - dv.logger = logger - dv.trusted.SetLogger(logger) - dv.source.SetLogger(logger) -} - -// Implements Verifier. -func (dv *DynamicVerifier) ChainID() string { - return dv.chainID -} - -// Implements Verifier. -// -// If the validators have changed since the last known time, it looks to -// dv.trusted and dv.source to prove the new validators. On success, it will -// try to store the SignedHeader in dv.trusted if the next -// validator can be sourced. -func (dv *DynamicVerifier) Verify(shdr types.SignedHeader) error { - - // Performs synchronization for multi-threads verification at the same height. - dv.mtx.Lock() - if pending := dv.pendingVerifications[shdr.Height]; pending != nil { - dv.mtx.Unlock() - <-pending // pending is chan struct{} - } else { - pending := make(chan struct{}) - dv.pendingVerifications[shdr.Height] = pending - defer func() { - close(pending) - dv.mtx.Lock() - delete(dv.pendingVerifications, shdr.Height) - dv.mtx.Unlock() - }() - dv.mtx.Unlock() - } - - //Get the exact trusted commit for h, and if it is - // equal to shdr, then it's already trusted, so - // just return nil. - trustedFCSameHeight, err := dv.trusted.LatestFullCommit(dv.chainID, shdr.Height, shdr.Height) - if err == nil { - // If loading trust commit successfully, and trust commit equal to shdr, then don't verify it, - // just return nil. - if bytes.Equal(trustedFCSameHeight.SignedHeader.Hash(), shdr.Hash()) { - dv.logger.Info(fmt.Sprintf("Load full commit at height %d from cache, there is not need to verify.", shdr.Height)) - return nil - } - } else if !lerr.IsErrCommitNotFound(err) { - // Return error if it is not CommitNotFound error - dv.logger.Info(fmt.Sprintf("Encountered unknown error in loading full commit at height %d.", shdr.Height)) - return err - } - - // Get the latest known full commit <= h-1 from our trusted providers. - // The full commit at h-1 contains the valset to sign for h. - prevHeight := shdr.Height - 1 - trustedFC, err := dv.trusted.LatestFullCommit(dv.chainID, 1, prevHeight) - if err != nil { - return err - } - - // sync up to the prevHeight and assert our latest NextValidatorSet - // is the ValidatorSet for the SignedHeader - if trustedFC.Height() == prevHeight { - // Return error if valset doesn't match. - if !bytes.Equal( - trustedFC.NextValidators.Hash(), - shdr.Header.ValidatorsHash) { - return lerr.ErrUnexpectedValidators( - trustedFC.NextValidators.Hash(), - shdr.Header.ValidatorsHash) - } - } else { - // If valset doesn't match, try to update - if !bytes.Equal( - trustedFC.NextValidators.Hash(), - shdr.Header.ValidatorsHash) { - // ... update. - trustedFC, err = dv.updateToHeight(prevHeight) - if err != nil { - return err - } - // Return error if valset _still_ doesn't match. - if !bytes.Equal(trustedFC.NextValidators.Hash(), - shdr.Header.ValidatorsHash) { - return lerr.ErrUnexpectedValidators( - trustedFC.NextValidators.Hash(), - shdr.Header.ValidatorsHash) - } - } - } - - // Verify the signed header using the matching valset. - cert := NewBaseVerifier(dv.chainID, trustedFC.Height()+1, trustedFC.NextValidators) - err = cert.Verify(shdr) - if err != nil { - return err - } - - // By now, the SignedHeader is fully validated and we're synced up to - // SignedHeader.Height - 1. To sync to SignedHeader.Height, we need - // the validator set at SignedHeader.Height + 1 so we can verify the - // SignedHeader.NextValidatorSet. - // TODO: is the ValidateFull below mostly redundant with the BaseVerifier.Verify above? - // See https://github.com/tendermint/tendermint/issues/3174. - - // Get the next validator set. - nextValset, err := dv.source.ValidatorSet(dv.chainID, shdr.Height+1) - if lerr.IsErrUnknownValidators(err) { - // Ignore this error. - return nil - } else if err != nil { - return err - } - - // Create filled FullCommit. - nfc := FullCommit{ - SignedHeader: shdr, - Validators: trustedFC.NextValidators, - NextValidators: nextValset, - } - // Validate the full commit. This checks the cryptographic - // signatures of Commit against Validators. - if err := nfc.ValidateFull(dv.chainID); err != nil { - return err - } - // Trust it. - return dv.trusted.SaveFullCommit(nfc) -} - -// verifyAndSave will verify if this is a valid source full commit given the -// best match trusted full commit, and if good, persist to dv.trusted. -// Returns ErrTooMuchChange when >2/3 of trustedFC did not sign sourceFC. -// Panics if trustedFC.Height() >= sourceFC.Height(). -func (dv *DynamicVerifier) verifyAndSave(trustedFC, sourceFC FullCommit) error { - if trustedFC.Height() >= sourceFC.Height() { - panic("should not happen") - } - err := trustedFC.NextValidators.VerifyFutureCommit( - sourceFC.Validators, - dv.chainID, sourceFC.SignedHeader.Commit.BlockID, - sourceFC.SignedHeader.Height, sourceFC.SignedHeader.Commit, - ) - if err != nil { - return err - } - - return dv.trusted.SaveFullCommit(sourceFC) -} - -// updateToHeight will use divide-and-conquer to find a path to h. -// Returns nil error iff we successfully verify and persist a full commit -// for height h, using repeated applications of bisection if necessary. -// -// Returns ErrCommitNotFound if source provider doesn't have the commit for h. -func (dv *DynamicVerifier) updateToHeight(h int64) (FullCommit, error) { - - // Fetch latest full commit from source. - sourceFC, err := dv.source.LatestFullCommit(dv.chainID, h, h) - if err != nil { - return FullCommit{}, err - } - - // If sourceFC.Height() != h, we can't do it. - if sourceFC.Height() != h { - return FullCommit{}, lerr.ErrCommitNotFound() - } - - // Validate the full commit. This checks the cryptographic - // signatures of Commit against Validators. - if err := sourceFC.ValidateFull(dv.chainID); err != nil { - return FullCommit{}, err - } - - // Verify latest FullCommit against trusted FullCommits -FOR_LOOP: - for { - // Fetch latest full commit from trusted. - trustedFC, err := dv.trusted.LatestFullCommit(dv.chainID, 1, h) - if err != nil { - return FullCommit{}, err - } - // We have nothing to do. - if trustedFC.Height() == h { - return trustedFC, nil - } - - // Try to update to full commit with checks. - err = dv.verifyAndSave(trustedFC, sourceFC) - if err == nil { - // All good! - return sourceFC, nil - } - - // Handle special case when err is ErrTooMuchChange. - if types.IsErrTooMuchChange(err) { - // Divide and conquer. - start, end := trustedFC.Height(), sourceFC.Height() - if !(start < end) { - panic("should not happen") - } - mid := (start + end) / 2 - _, err = dv.updateToHeight(mid) - if err != nil { - return FullCommit{}, err - } - // If we made it to mid, we retry. - continue FOR_LOOP - } - return FullCommit{}, err - } -} - -func (dv *DynamicVerifier) LastTrustedHeight() int64 { - fc, err := dv.trusted.LatestFullCommit(dv.chainID, 1, 1<<63-1) - if err != nil { - panic("should not happen") - } - return fc.Height() -} diff --git a/lite/errors/errors.go b/lite/errors/errors.go index 75442c726..42ce90010 100644 --- a/lite/errors/errors.go +++ b/lite/errors/errors.go @@ -41,6 +41,12 @@ func (e errEmptyTree) Error() string { return "Tree is empty" } +type errCommitExpired struct{} + +func (e errCommitExpired) Error() string { + return "Commit is too old to be trusted" +} + //---------------------------------------- // Methods for above error types @@ -109,3 +115,18 @@ func IsErrEmptyTree(err error) bool { } return false } + +//----------------- +// ErrCommitExpired + +func ErrCommitExpired() error { + return cmn.ErrorWrap(errCommitExpired{}, "") +} + +func IsErrCommitExpired(err error) bool { + if err_, ok := err.(cmn.Error); ok { + _, ok := err_.Data().(errCommitExpired) + return ok + } + return false +} diff --git a/lite/provider.go b/lite/provider.go index ebab16264..73f674aba 100644 --- a/lite/provider.go +++ b/lite/provider.go @@ -1,6 +1,9 @@ package lite import ( + "fmt" + "sync" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/types" ) @@ -30,3 +33,105 @@ type PersistentProvider interface { // SaveFullCommit saves a FullCommit (without verification). SaveFullCommit(fc FullCommit) error } + +// A provider that can update itself w/ more recent commit data. +type UpdatingProvider interface { + Provider + + // Update internal information by fetching information somehow. + // UpdateToHeight will block until the request is complete, + // or returns an error if the request cannot complete. + // Generally, one must call UpdateToHeight(h) before LatestFullCommit(_,h,h) + // will return this height. + // + // NOTE: Behavior with concurrent requests is undefined. To make + // concurrent calls safe, look at the struct `ConcurrentUpdatingProvider`. + UpdateToHeight(chainID string, height int64) error +} + +//---------------------------------------- + +type concurrentProvider struct { + UpdatingProvider + + // pending map to synchronize concurrent verification requests + mtx sync.Mutex + pendingVerifications map[pendingKey]*pendingResult +} + +// convenience to create the key for the lookup map +type pendingKey struct { + chainID string + height int64 +} + +// used to cache the result from underlying UpdatingProvider. +type pendingResult struct { + wait chan struct{} + err error // cached result. +} + +func NewConcurrentUpdatingProvider(up UpdatingProvider) concurrentProvider { + return concurrentProvider{ + UpdatingProvider: up, + pendingVerifications: make(map[pendingKey]*pendingResult), + } +} + +// Returns the unique pending request for all identical calls to +// joinConcurrency(chainID,height), and returns true for isFirstCall only for the +// first call, which should call the returned callback w/ results if any. +// The callback must be called, otherwise there will be memory leaks. +// Other subsequent calls should just return uniq.err. +// NOTE: This is a separate function, primarily to make mtx unlocking more obviously safe via defer. +func (cp concurrentProvider) joinConcurrency(chainID string, height int64) (uniq *pendingResult, isFirstCall bool, callback func(error)) { + cp.mtx.Lock() + defer cp.mtx.Unlock() + + var pkey = pendingKey{chainID, height} + cp.mtx.Lock() + if uniq = cp.pendingVerifications[pkey]; uniq != nil { + cp.mtx.Unlock() + <-uniq.wait // uniq.wait is of type `chan struct{}` + return uniq, false, nil + } else { + uniq = &pendingResult{wait: make(chan struct{}), err: nil} + cp.pendingVerifications[pkey] = uniq + cp.mtx.Unlock() + // The caller must call this, otherwise there will be memory leaks. + return uniq, true, func(err error) { + // NOTE: other result parameters can be added here. + uniq.err = err + // *After* setting the results, *then* call close(uniq.wait). + close(uniq.wait) + cp.mtx.Lock() // temporarily acquire lock to remove this iteem + delete(cp.pendingVerifications, pkey) + cp.mtx.Unlock() // and release lock + } + } +} + +func (cp concurrentProvider) UpdateToHeight(chainID string, height int64) error { + + // Performs synchronization for multi-threads verification at the same height. + var presult *pendingResult + var isFirstCall bool + var callback func(error) + presult, isFirstCall, callback = cp.joinConcurrency(chainID, height) + + if isFirstCall { + var err error + // Use a defer in case UpdateToHeight itself fails. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("Recovered from panic: %v", r) + } + callback(err) + }() + err = cp.UpdatingProvider.UpdateToHeight(chainID, height) + return err + } else { + // Is not the first call, so return the error from previous concurrent calls. + return presult.err + } +} diff --git a/lite/proxy/verifier.go b/lite/proxy/verifier.go deleted file mode 100644 index b7c11f18e..000000000 --- a/lite/proxy/verifier.go +++ /dev/null @@ -1,41 +0,0 @@ -package proxy - -import ( - cmn "github.com/tendermint/tendermint/libs/common" - dbm "github.com/tendermint/tendermint/libs/db" - log "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/lite" - lclient "github.com/tendermint/tendermint/lite/client" -) - -func NewVerifier(chainID, rootDir string, client lclient.SignStatusClient, logger log.Logger, cacheSize int) (*lite.DynamicVerifier, error) { - - logger = logger.With("module", "lite/proxy") - logger.Info("lite/proxy/NewVerifier()...", "chainID", chainID, "rootDir", rootDir, "client", client) - - memProvider := lite.NewDBProvider("trusted.mem", dbm.NewMemDB()).SetLimit(cacheSize) - lvlProvider := lite.NewDBProvider("trusted.lvl", dbm.NewDB("trust-base", dbm.LevelDBBackend, rootDir)) - trust := lite.NewMultiProvider( - memProvider, - lvlProvider, - ) - source := lclient.NewProvider(chainID, client) - cert := lite.NewDynamicVerifier(chainID, trust, source) - cert.SetLogger(logger) // Sets logger recursively. - - // TODO: Make this more secure, e.g. make it interactive in the console? - _, err := trust.LatestFullCommit(chainID, 1, 1<<63-1) - if err != nil { - logger.Info("lite/proxy/NewVerifier found no trusted full commit, initializing from source from height 1...") - fc, err := source.LatestFullCommit(chainID, 1, 1) - if err != nil { - return nil, cmn.ErrorWrap(err, "fetching source full commit @ height 1") - } - err = trust.SaveFullCommit(fc) - if err != nil { - return nil, cmn.ErrorWrap(err, "saving full commit to trusted") - } - } - - return cert, nil -} diff --git a/lite/proxy/wrapper.go b/lite/proxy/wrapper.go index 7ddb3b8ad..a787956d6 100644 --- a/lite/proxy/wrapper.go +++ b/lite/proxy/wrapper.go @@ -4,7 +4,7 @@ import ( cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/crypto/merkle" - "github.com/tendermint/tendermint/lite" + "github.com/tendermint/tendermint/lite/verifying" rpcclient "github.com/tendermint/tendermint/rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" ) @@ -15,7 +15,7 @@ var _ rpcclient.Client = Wrapper{} // provable before passing it along. Allows you to make any rpcclient fully secure. type Wrapper struct { rpcclient.Client - cert *lite.DynamicVerifier + cert *verifying.Provider prt *merkle.ProofRuntime } @@ -23,7 +23,7 @@ type Wrapper struct { // host and return a cryptographically secure rpc client. // // If it is wrapping an HTTP rpcclient, it will also wrap the websocket interface -func SecureClient(c rpcclient.Client, cert *lite.DynamicVerifier) Wrapper { +func SecureClient(c rpcclient.Client, cert *verifying.Provider) Wrapper { prt := defaultProofRuntime() wrap := Wrapper{c, cert, prt} // TODO: no longer possible as no more such interface exposed.... diff --git a/lite/types.go b/lite/types.go index 643f5ad48..0e269e508 100644 --- a/lite/types.go +++ b/lite/types.go @@ -4,6 +4,9 @@ import ( "github.com/tendermint/tendermint/types" ) +// NOTE: The Verifier interface is deprecated. BaseVerifier can continue to +// exist, but this interface isn't very useful on its own to declare here. +// // Verifier checks the votes to make sure the block really is signed properly. // Verifier must know the current or recent set of validitors by some other // means. diff --git a/lite/verifying/provider.go b/lite/verifying/provider.go new file mode 100644 index 000000000..87eb313ea --- /dev/null +++ b/lite/verifying/provider.go @@ -0,0 +1,394 @@ +package verifying + +import ( + "bytes" + "fmt" + "sync" + "time" + + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + log "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/lite" + lclient "github.com/tendermint/tendermint/lite/client" + lerr "github.com/tendermint/tendermint/lite/errors" + "github.com/tendermint/tendermint/types" +) + +type TrustOptions struct { + // Required: only trust commits up to this old. + TrustPeriod time.Duration + + // Option 1: TrustHeight and TrustHash can both be provided + // to force the trusting of a particular height and hash. + // If the latest trusted height/hash is more recent, then this option is + // ignored. + TrustHeight int64 + TrustHash []byte + + // Option 2: Callback can be set to implement a confirmation + // step if the trust store is uninitialized, or expired. + Callback func(height int64, hash []byte) error +} + +// NOTE If you retain the resulting verifier in memory for a long time, +// usage of the verifier may eventually error, but immediate usage should +// not error like that, so that e.g. cli usage never errors unexpectedly. +// TODO Move some of this initialization to a separate function. +func NewProvider(chainID, rootDir string, client lclient.SignStatusClient, logger log.Logger, cacheSize int, options TrustOptions) (*provider, error) { + + logger = logger.With("module", "lite/proxy") + logger.Info("lite/proxy/NewProvider()...", "chainID", chainID, "rootDir", rootDir, "client", client) + + memProvider := lite.NewDBProvider("trusted.mem", dbm.NewMemDB()).SetLimit(cacheSize) + lvlProvider := lite.NewDBProvider("trusted.lvl", dbm.NewDB("trust-base", dbm.LevelDBBackend, rootDir)) + trust := lite.NewMultiProvider( + memProvider, + lvlProvider, + ) + source := lclient.NewProvider(chainID, client) + vp := makeProvider(chainID, options.TrustPeriod, trust, source) + vp.SetLogger(logger) + trustPeriod := options.TrustPeriod + + // Get the latest trusted FC. + tlfc, err := trust.LatestFullCommit(chainID, 1, 1<<63-1) + if err != nil { + // Get the latest source commit, or the one provided in options. + targetCommit, err := getTargetCommit(client, options) + if err != nil { + return nil, err + } + err = vp.fillValidateAndSaveToTrust(targetCommit, nil, nil) + if err != nil { + return nil, err + } + return vp, nil + } + + // sanity check + if time.Now().Sub(tlfc.SignedHeader.Time) <= 0 { + panic(fmt.Sprintf("impossible time %v vs %v", time.Now(), tlfc.SignedHeader.Time)) + } + + if time.Now().Sub(tlfc.SignedHeader.Time) > trustPeriod { + // Get the latest source commit, or the one provided in options. + targetCommit, err := getTargetCommit(client, options) + if err != nil { + return nil, err + } + err = vp.fillValidateAndSaveToTrust(targetCommit, nil, nil) + if err != nil { + return nil, err + } + return vp, nil + } + + // Otherwise we're syncing within the unbonding period. + // NOTE: There is a duplication of fetching this latest commit (since + // UpdateToHeight() will fetch it again, and latestCommit isn't used), but + // it's only once upon initialization of a validator so it's not a big + // deal. + latestCommit, err := client.Commit(nil) + if err != nil { + return nil, err + } + err = vp.UpdateToHeight(chainID, latestCommit.SignedHeader.Height) + if err != nil { + return nil, err + } + return vp, nil +} + +// Returns the desired trusted sync point to fetch from the client. +func getTargetCommit(client lclient.SignStatusClient, options TrustOptions) (types.SignedHeader, error) { + if options.TrustHeight != 0 { + resCommit, err := client.Commit(&options.TrustHeight) + if err != nil { + return types.SignedHeader{}, err + } + targetCommit := resCommit.SignedHeader + if !bytes.Equal(targetCommit.Hash(), options.TrustHash) { + return types.SignedHeader{}, fmt.Errorf("WARNING!!! Expected height/hash %v/%X but got %X", + options.TrustHeight, options.TrustHash, targetCommit.Hash()) + } + return targetCommit, nil + } else { + resCommit, err := client.Commit(nil) + if err != nil { + return types.SignedHeader{}, err + } + targetCommit := resCommit.SignedHeader + // NOTE: This should really belong in the callback. + // WARN THE USER IN ALL CAPS THAT THE LITE CLIENT IS NEW, + // AND THAT WE WILL SYNC TO AND VERIFY LATEST COMMIT. + fmt.Printf("trusting source at height %v and hash %X...\n", + targetCommit.Height, targetCommit.Hash()) + if options.Callback != nil { + err := options.Callback(targetCommit.Height, targetCommit.Hash()) + if err != nil { + return types.SignedHeader{}, err + } + } + return targetCommit, nil + } +} + +//---------------------------------------- + +type nowFn func() time.Time + +const sizeOfPendingMap = 1024 + +var _ lite.UpdatingProvider = (*provider)(nil) + +// provider implements a persistent caching provider that +// auto-validates. It uses a "source" provider to obtain the needed +// FullCommits to securely sync with validator set changes. It stores properly +// validated data on the "trusted" local system. +// NOTE: This provider can only work with one chainID, provided upon +// instantiation. +type provider struct { + chainID string + logger log.Logger + trustPeriod time.Duration // e.g. the unbonding period, or something smaller. + now nowFn + + // Already validated, stored locally + trusted lite.PersistentProvider + + // New info, like a node rpc, or other import method. + source lite.Provider + + // pending map to synchronize concurrent verification requests + mtx sync.Mutex + pendingVerifications map[int64]chan struct{} +} + +// makeProvider returns a new verifying provider. It uses the +// trusted provider to store validated data and the source provider to +// obtain missing data (e.g. FullCommits). +// +// The trusted provider should be a DBProvider. +// The source provider should be a client.HTTPProvider. +// NOTE: The external facing constructor is called NewVerifyingProivider. +func makeProvider(chainID string, trustPeriod time.Duration, trusted lite.PersistentProvider, source lite.Provider) *provider { + if trustPeriod == 0 { + panic("VerifyingProvider must have non-zero trust period") + } + return &provider{ + logger: log.NewNopLogger(), + chainID: chainID, + trustPeriod: trustPeriod, + trusted: trusted, + source: source, + pendingVerifications: make(map[int64]chan struct{}, sizeOfPendingMap), + } +} + +func (vp *provider) SetLogger(logger log.Logger) { + logger = logger.With("module", "lite") + vp.logger = logger + vp.trusted.SetLogger(logger) + vp.source.SetLogger(logger) +} + +func (vp *provider) ChainID() string { + return vp.chainID +} + +// Implements UpdatingProvider +// +// On success, it will store the full commit (SignedHeader + Validators) in +// vp.trusted. +// NOTE: For concurreent usage, use concurrentProvider. +func (vp *provider) UpdateToHeight(chainID string, height int64) error { + + // If we alreeady have the commit, just return nil. + _, err := vp.trusted.LatestFullCommit(vp.chainID, height, height) + if err == nil { + return nil + } else if !lerr.IsErrCommitNotFound(err) { + // Return error if it is not CommitNotFound error + vp.logger.Info(fmt.Sprintf("Encountered unknown error in loading full commit at height %d.", height)) + return err + } + + // Fetch trusted FC at exactly height, while updating trust when possible. + _, err = vp.fetchAndVerifyToHeight(height) + if err != nil { + return err + } + + // Good! + return nil +} + +// If valset or nextValset are nil, fetches them. +// Then, validatees the full commit, then savees it. +func (vp *provider) fillValidateAndSaveToTrust(signedHeader types.SignedHeader, valset, nextValset *types.ValidatorSet) (err error) { + + // Get the valset. + if valset != nil { + valset, err = vp.source.ValidatorSet(vp.chainID, signedHeader.Height) + if err != nil { + return cmn.ErrorWrap(err, "fetching the valset") + } + } + + // Get the next validator set. + if nextValset != nil { + for { + nextValset, err = vp.source.ValidatorSet(vp.chainID, signedHeader.Height+1) + if lerr.IsErrUnknownValidators(err) { + // try again until we get it. + fmt.Printf("fetching validatorset for height %v...\n", + signedHeader.Height+1) + continue + } else if err != nil { + return cmn.ErrorWrap(err, "fetching the next valset") + } + } + } + + // Create filled FullCommit. + fc := lite.FullCommit{ + SignedHeader: signedHeader, + Validators: valset, + NextValidators: nextValset, + } + // Validate the full commit. This checks the cryptographic + // signatures of Commit against Validators. + if err := fc.ValidateFull(vp.chainID); err != nil { + return cmn.ErrorWrap(err, "verifying validators from source") + } + // Trust it. + err = vp.trusted.SaveFullCommit(fc) + if err != nil { + return cmn.ErrorWrap(err, "saving full commit") + } + return nil +} + +// verifyAndSave will verify if this is a valid source full commit given the +// best match trusted full commit, and persist to vp.trusted. +// +// Returns ErrTooMuchChange when >2/3 of trustedFC did not sign newFC. +// Returns ErrCommitExpired when trustedFC is too old. +// Panics if trustedFC.Height() >= newFC.Height(). +func (vp *provider) verifyAndSave(trustedFC, newFC lite.FullCommit) error { + if trustedFC.Height() >= newFC.Height() { + panic("should not happen") + } + if vp.now().Sub(trustedFC.SignedHeader.Time) > vp.trustPeriod { + return lerr.ErrCommitExpired() + } + if trustedFC.Height() == newFC.Height()-1 { + err := trustedFC.NextValidators.VerifyCommit( + vp.chainID, newFC.SignedHeader.Commit.BlockID, + newFC.SignedHeader.Height, newFC.SignedHeader.Commit, + ) + if err != nil { + return err + } + } else { + err := trustedFC.NextValidators.VerifyFutureCommit( + newFC.Validators, + vp.chainID, newFC.SignedHeader.Commit.BlockID, + newFC.SignedHeader.Height, newFC.SignedHeader.Commit, + ) + if err != nil { + return err + } + } + + if vp.now().Before(newFC.SignedHeader.Time) { + // TODO print warning + // TODO if too egregious, return error. + // return FullCommit{}, errors.New("now should not be before source time") + } + + return vp.trusted.SaveFullCommit(newFC) +} + +// fetchAndVerifyToHeight will use divide-and-conquer to find a path to h. +// Returns nil error iff we successfully verify for height h, using repeated +// applications of bisection if necessary. +// Along the way, if a recent trust is used to verify a more recent header, the +// more recent header becomes trusted. +// +// Returns ErrCommitNotFound if source provider doesn't have the commit for h. +func (vp *provider) fetchAndVerifyToHeight(h int64) (lite.FullCommit, error) { + + // Fetch latest full commit from source. + sourceFC, err := vp.source.LatestFullCommit(vp.chainID, h, h) + if err != nil { + return lite.FullCommit{}, err + } + + // If sourceFC.Height() != h, we can't do it. + if sourceFC.Height() != h { + return lite.FullCommit{}, lerr.ErrCommitNotFound() + } + + // Validate the full commit. This checks the cryptographic + // signatures of Commit against Validators. + if err := sourceFC.ValidateFull(vp.chainID); err != nil { + return lite.FullCommit{}, err + } + + // Verify latest FullCommit against trusted FullCommits + // Use a loop rather than recursion to avoid stack overflows. +FOR_LOOP: + for { + // Fetch latest full commit from trusted. + trustedFC, err := vp.trusted.LatestFullCommit(vp.chainID, 1, h) + if err != nil { + return lite.FullCommit{}, err + } + // We have nothing to do. + if trustedFC.Height() == h { + return trustedFC, nil + } + + // Update to full commit with checks. + err = vp.verifyAndSave(trustedFC, sourceFC) + // Handle special case when err is ErrTooMuchChange. + if types.IsErrTooMuchChange(err) { + // Divide and conquer. + start, end := trustedFC.Height(), sourceFC.Height() + if !(start < end) { + panic("should not happen") + } + mid := (start + end) / 2 + _, err = vp.fetchAndVerifyToHeight(mid) + if err != nil { + return lite.FullCommit{}, err + } + // If we made it to mid, we retry. + continue FOR_LOOP + } else if err != nil { + return lite.FullCommit{}, err + } + + // All good! + return sourceFC, nil + } +} + +func (vp *provider) LastTrustedHeight() int64 { + fc, err := vp.trusted.LatestFullCommit(vp.chainID, 1, 1<<63-1) + if err != nil { + panic("should not happen") + } + return fc.Height() +} + +func (vp *provider) LatestFullCommit(chainID string, minHeight, maxHeight int64) (lite.FullCommit, error) { + return vp.trusted.LatestFullCommit(chainID, minHeight, maxHeight) +} + +func (vp *provider) ValidatorSet(chainID string, height int64) (*types.ValidatorSet, error) { + // XXX try to sync? + return vp.trusted.ValidatorSet(chainID, height) +} diff --git a/lite/dynamic_verifier_test.go b/lite/verifying/provider_test.go similarity index 62% rename from lite/dynamic_verifier_test.go rename to lite/verifying/provider_test.go index 386de513c..21a8b4121 100644 --- a/lite/dynamic_verifier_test.go +++ b/lite/verifying/provider_test.go @@ -1,4 +1,4 @@ -package lite +package verifying import ( "fmt" @@ -13,10 +13,10 @@ import ( "github.com/tendermint/tendermint/types" ) -func TestInquirerValidPath(t *testing.T) { +func TestProviderValidPath(t *testing.T) { assert, require := assert.New(t), require.New(t) - trust := NewDBProvider("trust", dbm.NewMemDB()) - source := NewDBProvider("source", dbm.NewMemDB()) + trust := lite.NewDBProvider("trust", dbm.NewMemDB()) + source := lite.NewDBProvider("source", dbm.NewMemDB()) // Set up the validators to generate test blocks. var vote int64 = 10 @@ -45,35 +45,41 @@ func TestInquirerValidPath(t *testing.T) { // Initialize a Verifier with the initial state. err := trust.SaveFullCommit(fcz[0]) - require.Nil(err) - cert := NewDynamicVerifier(chainID, trust, source) - cert.SetLogger(log.TestingLogger()) + require.NoError(err) + vp := NewProvider(chainID, trust, source) + vp.SetLogger(log.TestingLogger()) - // This should fail validation: - sh := fcz[count-1].SignedHeader - err = cert.Verify(sh) - require.NotNil(err) + // The latest commit is the first one. + fc, err := vp.LatestFullCommit(chainID, 0, fcz[count-1].SignedHeader.Height) + require.NoError(err) + require.NoError(fc.ValidateFull(chainID)) + require.Equal(fcz[0].SignedHeader, fc.SignedHeader) // Adding a few commits in the middle should be insufficient. + // The latest commit is still the first one. for i := 10; i < 13; i++ { err := source.SaveFullCommit(fcz[i]) - require.Nil(err) + require.NoError(err) } - err = cert.Verify(sh) - assert.NotNil(err) + fc, err = vp.LatestFullCommit(chainID, 0, fcz[count-1].SignedHeader.Height) + require.NoError(err) + require.NoError(fc.ValidateFull(chainID)) + require.Equal(fcz[0].SignedHeader, fc.SignedHeeader) // With more info, we succeed. for i := 0; i < count; i++ { err := source.SaveFullCommit(fcz[i]) - require.Nil(err) + require.NoError(err) } - err = cert.Verify(sh) - assert.Nil(err, "%+v", err) + fc, err = vp.LatestFullCommit(chainID, 0, fcz[count-1].SignedHeader.Height) + require.NoError(err) + require.NoError(fc.ValidateFull(chainID)) + require.Equal(fcz[count-1].SignedHeader, fc.SignedHeeader) } -func TestDynamicVerify(t *testing.T) { - trust := NewDBProvider("trust", dbm.NewMemDB()) - source := NewDBProvider("source", dbm.NewMemDB()) +func TestProviderDynamicVerification(t *testing.T) { + trust := lite.NewDBProvider("trust", dbm.NewMemDB()) + source := lite.NewDBProvider("source", dbm.NewMemDB()) // 10 commits with one valset, 1 to change, // 10 commits with the next one @@ -110,18 +116,15 @@ func TestDynamicVerify(t *testing.T) { // Initialize a Verifier with the initial state. err := trust.SaveFullCommit(fcz[0]) - require.Nil(t, err) - ver := NewDynamicVerifier(chainID, trust, source) - ver.SetLogger(log.TestingLogger()) + require.NoError(t, err) + vp := NewProvider(chainID, trust, source) + vp.SetLogger(log.TestingLogger()) // fetch the latest from the source latestFC, err := source.LatestFullCommit(chainID, 1, maxHeight) require.NoError(t, err) - - // try to update to the latest - err = ver.Verify(latestFC.SignedHeader) - require.NoError(t, err) - + require.NoError(latestFC.ValidateFull(chainID)) + require.Equal(fcz[nCommits-1].SignedHeader, latestFC.SignedHeader) } func makeFullCommit(height int64, keys privKeys, vals, nextVals *types.ValidatorSet, chainID string) FullCommit { @@ -135,10 +138,10 @@ func makeFullCommit(height int64, keys privKeys, vals, nextVals *types.Validator appHash, consHash, resHash, 0, len(keys)) } -func TestInquirerVerifyHistorical(t *testing.T) { +func TestVerifingProviderHistorical(t *testing.T) { assert, require := assert.New(t), require.New(t) - trust := NewDBProvider("trust", dbm.NewMemDB()) - source := NewDBProvider("source", dbm.NewMemDB()) + trust := lite.NewDBProvider("trust", dbm.NewMemDB()) + source := lite.NewDBProvider("source", dbm.NewMemDB()) // Set up the validators to generate test blocks. var vote int64 = 10 @@ -167,9 +170,9 @@ func TestInquirerVerifyHistorical(t *testing.T) { // Initialize a Verifier with the initial state. err := trust.SaveFullCommit(fcz[0]) - require.Nil(err) - cert := NewDynamicVerifier(chainID, trust, source) - cert.SetLogger(log.TestingLogger()) + require.NoError(err) + vp := NewProvider(chainID, trust, source) + vp.SetLogger(log.TestingLogger()) // Store a few full commits as trust. for _, i := range []int{2, 5} { @@ -177,51 +180,49 @@ func TestInquirerVerifyHistorical(t *testing.T) { } // See if we can jump forward using trusted full commits. - // Souce doesn't have fcz[9] so cert.LastTrustedHeight wont' change. + // Souce doesn't have fcz[9] so vp.LastTrustedHeight wont' change. err = source.SaveFullCommit(fcz[7]) - require.Nil(err, "%+v", err) - sh := fcz[8].SignedHeader - err = cert.Verify(sh) - require.Nil(err, "%+v", err) - assert.Equal(fcz[7].Height(), cert.LastTrustedHeight()) + require.NoError(err, "%+v", err) + assert.Equal(fcz[7].Height(), vp.LastTrustedHeight()) fc_, err := trust.LatestFullCommit(chainID, fcz[8].Height(), fcz[8].Height()) - require.NotNil(err, "%+v", err) - assert.Equal(fc_, (FullCommit{})) + require.Error(err, "%+v", err) + assert.Equal((FullCommit{}), fc_) // With fcz[9] Verify will update last trusted height. err = source.SaveFullCommit(fcz[9]) - require.Nil(err, "%+v", err) - sh = fcz[8].SignedHeader - err = cert.Verify(sh) - require.Nil(err, "%+v", err) - assert.Equal(fcz[8].Height(), cert.LastTrustedHeight()) + require.NoError(err, "%+v", err) + assert.Equal(fcz[8].Height(), vp.LastTrustedHeight()) fc_, err = trust.LatestFullCommit(chainID, fcz[8].Height(), fcz[8].Height()) - require.Nil(err, "%+v", err) - assert.Equal(fc_.Height(), fcz[8].Height()) + require.NoError(err, "%+v", err) + assert.Equal(fcz[8].Height(), fc_.Height()) // Add access to all full commits via untrusted source. for i := 0; i < count; i++ { err := source.SaveFullCommit(fcz[i]) - require.Nil(err) + require.NoError(err) } - // Try to check an unknown seed in the past. - sh = fcz[3].SignedHeader - err = cert.Verify(sh) - require.Nil(err, "%+v", err) - assert.Equal(fcz[8].Height(), cert.LastTrustedHeight()) + // Try to fetch an unknown commit from the past. + fc_, err = trust.LatestFullCommit(chainID, fcz[2].Height(), fcz[3].Height()) + require.NoError(err, "%+v", err) + assert.Equal(fcz[2].Height(), fc_.Height()) + assert.Equal(fcz[8].Height(), vp.LastTrustedHeight()) + // TODO This should work for as long as the trust period hasn't passed for + // fcz[2]. Write a test that tries to retroactively fetchees fcz[3] from + // source. Initially it should fail since source doesn't have it, but it + // should succeed once source is provided it. - // Jump all the way forward again. - sh = fcz[count-1].SignedHeader - err = cert.Verify(sh) - require.Nil(err, "%+v", err) - assert.Equal(fcz[9].Height(), cert.LastTrustedHeight()) + // Try to fetch the latest known commit. + fc_, err = trust.LatestFullCommit(chainID, 0, fcz[9].Height()) + require.NoError(err, "%+v", err) + assert.Equal(fcz[9].Height(), fc_.Height()) + assert.Equal(fcz[9].Height(), vp.LastTrustedHeight()) } -func TestConcurrencyInquirerVerify(t *testing.T) { +func TestConcurrentProvider(t *testing.T) { _, require := assert.New(t), require.New(t) - trust := NewDBProvider("trust", dbm.NewMemDB()).SetLimit(10) - source := NewDBProvider("source", dbm.NewMemDB()) + trust := lite.NewDBProvider("trust", dbm.NewMemDB()).SetLimit(10) + source := lite.NewDBProvider("source", dbm.NewMemDB()) // Set up the validators to generate test blocks. var vote int64 = 10 @@ -250,13 +251,14 @@ func TestConcurrencyInquirerVerify(t *testing.T) { // Initialize a Verifier with the initial state. err := trust.SaveFullCommit(fcz[0]) - require.Nil(err) - cert := NewDynamicVerifier(chainID, trust, source) - cert.SetLogger(log.TestingLogger()) + require.NoError(err) + vp := NewProvider(chainID, trust, source) + vp.SetLogger(log.TestingLogger()) + cp := NewConcurrentProvider(vp) err = source.SaveFullCommit(fcz[7]) err = source.SaveFullCommit(fcz[8]) - require.Nil(err, "%+v", err) + require.NoError(err, "%+v", err) sh := fcz[8].SignedHeader var wg sync.WaitGroup @@ -265,12 +267,12 @@ func TestConcurrencyInquirerVerify(t *testing.T) { for i := 0; i < count; i++ { wg.Add(1) go func(index int) { - errList[index] = cert.Verify(sh) + errList[index] = cp.UpdateToHeight(chainID, fcz[8].SignedHeader.Height) defer wg.Done() }(i) } wg.Wait() for _, err := range errList { - require.Nil(err) + require.NoError(err) } } diff --git a/types/block.go b/types/block.go index 99ee3f8e1..5dba4409d 100644 --- a/types/block.go +++ b/types/block.go @@ -403,6 +403,18 @@ func (h *Header) Populate( h.ProposerAddress = proposerAddress } +// NOTE: While it's possible to make this faster via a custom implementation, +// (or naively via a struct copy, though this isn't yet a frozen design goal), +// for now use hashes in case of any issues that may arise in implementation. +func (h *Header) Equal(h2 *Header) bool { + h1Hash := h.Hash() + if h1Hash == nil { + panic("incomplete heaeders cannot be compared") + } + h2Hash := h2.Hash() + return bytes.Compare(h1Hash, h2Hash) == 0 +} + // Hash returns the hash of the header. // It computes a Merkle tree from the header fields // ordered as they appear in the Header. @@ -600,6 +612,18 @@ func (commit *Commit) ValidateBasic() error { return nil } +// NOTE: While it's possible to make this faster via a custom implementation, +// (naively via a struct copy won't work due to the volatile fields), +// for now use hashes in case of any issues that may arise in implementation. +func (commit *Commit) Equal(commit2 *Commit) bool { + c1Hash := commit.Hash() + if c1Hash == nil { + panic("incomplete commit cannot be compared") + } + c2Hash := commit2.Hash() + return bytes.Compare(c1Hash, c2Hash) == 0 +} + // Hash returns the hash of the commit func (commit *Commit) Hash() cmn.HexBytes { if commit == nil { @@ -644,6 +668,15 @@ type SignedHeader struct { Commit *Commit `json:"commit"` } +// Returns true iff both the header and commit hold identical information +// (disregarding any volatile memoized fields). +// Header and Commit must be their final immutable forms, otherwise this +// function will panic. +func (sh SignedHeader) Equal(sh2 SignedHeader) bool { + return sh.Header.Equal(sh2.Header) && + sh.Commit.Equal(sh2.Commit) +} + // ValidateBasic does basic consistency checks and makes sure the header // and commit are consistent. // diff --git a/types/validator_set.go b/types/validator_set.go index a36e1920d..68beedc79 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -416,15 +416,16 @@ func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height i return errTooMuchChange{talliedVotingPower, vals.TotalVotingPower()*2/3 + 1} } -// VerifyFutureCommit will check to see if the set would be valid with a different -// validator set. +// VerifyFutureCommit checks to see if a given future validator set has +// committed a block, and whether those who signed of this future validator set +// has sufficient overlap with this validator set. // -// vals is the old validator set that we know. Over 2/3 of the power in old -// signed this block. +// vals is the current validator set that we know. Over 2/3 of the power in +// this valset is expected to have signed this block. // -// In Tendermint, 1/3 of the voting power can halt or fork the chain, but 1/3 -// can't make arbitrary state transitions. You still need > 2/3 Byzantine to -// make arbitrary state transitions. +// Justification for the 2/3: In Tendermint, 1/3 of the voting power can halt +// or fork the chain, but 1/3 can't make arbitrary state transitions. You +// still need > 2/3 Byzantine to make arbitrary state transitions. // // To preserve this property in the light client, we also require > 2/3 of the // old vals to sign the future commit at H, that way we preserve the property @@ -433,18 +434,17 @@ func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height i // > 2/3. Otherwise, the lite client isn't providing the same security // guarantees. // -// Even if we added a slashing condition that if you sign a block header with -// the wrong validator set, then we would only need > 1/3 of signatures from -// the old vals on the new commit, it wouldn't be sufficient because the new -// vals can be arbitrary and commit some arbitrary app hash. -// // newSet is the validator set that signed this block. Only votes from new are // sufficient for 2/3 majority in the new set as well, for it to be a valid // commit. // -// NOTE: This doesn't check whether the commit is a future commit, because the -// current height isn't part of the ValidatorSet. Caller must check that the -// commit height is greater than the height for this validator set. +// NOTE: This doesn't check whether the commit is actually a future commit, +// because the current height isn't part of the ValidatorSet. Caller must +// check that the commit height is greater than the height for this validator +// set. +// +// NOTE: This function is strictly more restrictive than merely checking +// whether newSet.VerifyCommit(...), in fact it calls exactly that. func (vals *ValidatorSet) VerifyFutureCommit(newSet *ValidatorSet, chainID string, blockID BlockID, height int64, commit *Commit) error { oldVals := vals