Revert "delete everything" (includes everything non-go-crypto)

This reverts commit 96a3502

lite/client/main_test.go

@@ -0,0 +1,25 @@
+package client_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/tendermint/abci/example/kvstore"
+
+	nm "github.com/tendermint/tendermint/node"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+var node *nm.Node
+
+func TestMain(m *testing.M) {
+	// start a tendermint node (and merkleeyes) in the background to test against
+	app := kvstore.NewKVStoreApplication()
+	node = rpctest.StartTendermint(app)
+	code := m.Run()
+
+	// and shut down proper at the end
+	node.Stop()
+	node.Wait()
+	os.Exit(code)
+}

lite/client/provider.go

@@ -0,0 +1,141 @@
+/*
+Package client defines a provider that uses a rpcclient
+to get information, which is used to get new headers
+and validators directly from a node.
+*/
+package client
+
+import (
+	"bytes"
+
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+// SignStatusClient combines a SignClient and StatusClient.
+type SignStatusClient interface {
+	rpcclient.SignClient
+	rpcclient.StatusClient
+}
+
+type provider struct {
+	node       SignStatusClient
+	lastHeight int64
+}
+
+// NewProvider can wrap any rpcclient to expose it as
+// a read-only provider.
+func NewProvider(node SignStatusClient) lite.Provider {
+	return &provider{node: node}
+}
+
+// NewHTTPProvider can connect to a tendermint json-rpc endpoint
+// at the given url, and uses that as a read-only provider.
+func NewHTTPProvider(remote string) lite.Provider {
+	return &provider{
+		node: rpcclient.NewHTTP(remote, "/websocket"),
+	}
+}
+
+// StatusClient returns the internal node as a StatusClient
+func (p *provider) StatusClient() rpcclient.StatusClient {
+	return p.node
+}
+
+// StoreCommit is a noop, as clients can only read from the chain...
+func (p *provider) StoreCommit(_ lite.FullCommit) error { return nil }
+
+// GetHash gets the most recent validator and sees if it matches
+//
+// TODO: improve when the rpc interface supports more functionality
+func (p *provider) GetByHash(hash []byte) (lite.FullCommit, error) {
+	var fc lite.FullCommit
+	vals, err := p.node.Validators(nil)
+	// if we get no validators, or a different height, return an error
+	if err != nil {
+		return fc, err
+	}
+	p.updateHeight(vals.BlockHeight)
+	vhash := types.NewValidatorSet(vals.Validators).Hash()
+	if !bytes.Equal(hash, vhash) {
+		return fc, liteErr.ErrCommitNotFound()
+	}
+	return p.seedFromVals(vals)
+}
+
+// GetByHeight gets the validator set by height
+func (p *provider) GetByHeight(h int64) (fc lite.FullCommit, err error) {
+	commit, err := p.node.Commit(&h)
+	if err != nil {
+		return fc, err
+	}
+	return p.seedFromCommit(commit)
+}
+
+// LatestCommit returns the newest commit stored.
+func (p *provider) LatestCommit() (fc lite.FullCommit, err error) {
+	commit, err := p.GetLatestCommit()
+	if err != nil {
+		return fc, err
+	}
+	return p.seedFromCommit(commit)
+}
+
+// GetLatestCommit should return the most recent commit there is,
+// which handles queries for future heights as per the semantics
+// of GetByHeight.
+func (p *provider) GetLatestCommit() (*ctypes.ResultCommit, error) {
+	status, err := p.node.Status()
+	if err != nil {
+		return nil, err
+	}
+	return p.node.Commit(&status.SyncInfo.LatestBlockHeight)
+}
+
+// CommitFromResult ...
+func CommitFromResult(result *ctypes.ResultCommit) lite.Commit {
+	return (lite.Commit)(result.SignedHeader)
+}
+
+func (p *provider) seedFromVals(vals *ctypes.ResultValidators) (lite.FullCommit, error) {
+	// now get the commits and build a full commit
+	commit, err := p.node.Commit(&vals.BlockHeight)
+	if err != nil {
+		return lite.FullCommit{}, err
+	}
+	fc := lite.NewFullCommit(
+		CommitFromResult(commit),
+		types.NewValidatorSet(vals.Validators),
+	)
+	return fc, nil
+}
+
+func (p *provider) seedFromCommit(commit *ctypes.ResultCommit) (fc lite.FullCommit, err error) {
+	fc.Commit = CommitFromResult(commit)
+
+	// now get the proper validators
+	vals, err := p.node.Validators(&commit.Header.Height)
+	if err != nil {
+		return fc, err
+	}
+
+	// make sure they match the commit (as we cannot enforce height)
+	vset := types.NewValidatorSet(vals.Validators)
+	if !bytes.Equal(vset.Hash(), commit.Header.ValidatorsHash) {
+		return fc, liteErr.ErrValidatorsChanged()
+	}
+
+	p.updateHeight(commit.Header.Height)
+	fc.Validators = vset
+	return fc, nil
+}
+
+func (p *provider) updateHeight(h int64) {
+	if h > p.lastHeight {
+		p.lastHeight = h
+	}
+}

lite/client/provider_test.go

@@ -0,0 +1,63 @@
+package client
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestProvider(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	cfg := rpctest.GetConfig()
+	rpcAddr := cfg.RPC.ListenAddress
+	genDoc, _ := types.GenesisDocFromFile(cfg.GenesisFile())
+	chainID := genDoc.ChainID
+	p := NewHTTPProvider(rpcAddr)
+	require.NotNil(t, p)
+
+	// let it produce some blocks
+	err := rpcclient.WaitForHeight(p.(*provider).node, 6, nil)
+	require.Nil(err)
+
+	// let's get the highest block
+	seed, err := p.LatestCommit()
+
+	require.Nil(err, "%+v", err)
+	sh := seed.Height()
+	vhash := seed.Header.ValidatorsHash
+	assert.True(sh < 5000)
+
+	// let's check this is valid somehow
+	assert.Nil(seed.ValidateBasic(chainID))
+	cert := lite.NewStaticCertifier(chainID, seed.Validators)
+
+	// historical queries now work :)
+	lower := sh - 5
+	seed, err = p.GetByHeight(lower)
+	assert.Nil(err, "%+v", err)
+	assert.Equal(lower, seed.Height())
+
+	// also get by hash (given the match)
+	seed, err = p.GetByHash(vhash)
+	require.Nil(err, "%+v", err)
+	require.Equal(vhash, seed.Header.ValidatorsHash)
+	err = cert.Certify(seed.Commit)
+	assert.Nil(err, "%+v", err)
+
+	// get by hash fails without match
+	seed, err = p.GetByHash([]byte("foobar"))
+	assert.NotNil(err)
+	assert.True(liteErr.IsCommitNotFoundErr(err))
+
+	// storing the seed silently ignored
+	err = p.StoreCommit(seed)
+	assert.Nil(err, "%+v", err)
+}

lite/commit.go

@@ -0,0 +1,99 @@
+package lite
+
+import (
+	"bytes"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tendermint/types"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+// Certifier checks the votes to make sure the block really is signed properly.
+// Certifier must know the current set of validitors by some other means.
+type Certifier interface {
+	Certify(check Commit) error
+	ChainID() string
+}
+
+// Commit is basically the rpc /commit response, but extended
+//
+// This is the basepoint for proving anything on the blockchain. It contains
+// a signed header.  If the signatures are valid and > 2/3 of the known set,
+// we can store this checkpoint and use it to prove any number of aspects of
+// the system: such as txs, abci state, validator sets, etc...
+type Commit types.SignedHeader
+
+// FullCommit is a commit and the actual validator set,
+// the base info you need to update to a given point,
+// assuming knowledge of some previous validator set
+type FullCommit struct {
+	Commit     `json:"commit"`
+	Validators *types.ValidatorSet `json:"validator_set"`
+}
+
+// NewFullCommit returns a new FullCommit.
+func NewFullCommit(commit Commit, vals *types.ValidatorSet) FullCommit {
+	return FullCommit{
+		Commit:     commit,
+		Validators: vals,
+	}
+}
+
+// Height returns the height of the header.
+func (c Commit) Height() int64 {
+	if c.Header == nil {
+		return 0
+	}
+	return c.Header.Height
+}
+
+// ValidatorsHash returns the hash of the validator set.
+func (c Commit) ValidatorsHash() []byte {
+	if c.Header == nil {
+		return nil
+	}
+	return c.Header.ValidatorsHash
+}
+
+// ValidateBasic does basic consistency checks and makes sure the headers
+// and commits are all consistent and refer to our chain.
+//
+// Make sure to use a Verifier to validate the signatures actually provide
+// a significantly strong proof for this header's validity.
+func (c Commit) ValidateBasic(chainID string) error {
+	// make sure the header is reasonable
+	if c.Header == nil {
+		return errors.New("Commit missing header")
+	}
+	if c.Header.ChainID != chainID {
+		return errors.Errorf("Header belongs to another chain '%s' not '%s'",
+			c.Header.ChainID, chainID)
+	}
+
+	if c.Commit == nil {
+		return errors.New("Commit missing signatures")
+	}
+
+	// make sure the header and commit match (height and hash)
+	if c.Commit.Height() != c.Header.Height {
+		return liteErr.ErrHeightMismatch(c.Commit.Height(), c.Header.Height)
+	}
+	hhash := c.Header.Hash()
+	chash := c.Commit.BlockID.Hash
+	if !bytes.Equal(hhash, chash) {
+		return errors.Errorf("Commits sign block %X header is block %X",
+			chash, hhash)
+	}
+
+	// make sure the commit is reasonable
+	err := c.Commit.ValidateBasic()
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	// looks good, we just need to make sure the signatures are really from
+	// empowered validators
+	return nil
+}

lite/doc.go

@@ -0,0 +1,133 @@
+/*
+Package lite allows you to securely validate headers
+without a full node.
+
+This library pulls together all the crypto and algorithms,
+so given a relatively recent (< unbonding period) known
+validator set, one can get indisputable proof that data is in
+the chain (current state) or detect if the node is lying to
+the client.
+
+Tendermint RPC exposes a lot of info, but a malicious node
+could return any data it wants to queries, or even to block
+headers, even making up fake signatures from non-existent
+validators to justify it. This is a lot of logic to get
+right, to be contained in a small, easy to use library,
+that does this for you, so you can just build nice UI.
+
+We design for clients who have no strong trust relationship
+with any tendermint node, just the validator set as a whole.
+Beyond building nice mobile or desktop applications, the
+cosmos hub is another important example of a client,
+that needs undeniable proof without syncing the full chain,
+in order to efficiently implement IBC.
+
+Commits
+
+There are two main data structures that we pass around - Commit
+and FullCommit. Both of them mirror what information is
+exposed in tendermint rpc.
+
+Commit is a block header along with enough validator signatures
+to prove its validity (> 2/3 of the voting power). A FullCommit
+is a Commit along with the full validator set. When the
+validator set doesn't change, the Commit is enough, but since
+the block header only has a hash, we need the FullCommit to
+follow any changes to the validator set.
+
+Certifiers
+
+A Certifier validates a new Commit given the currently known
+state. There are three different types of Certifiers exposed,
+each one building on the last one, with additional complexity.
+
+Static - given the validator set upon initialization. Verifies
+all signatures against that set and if the validator set
+changes, it will reject all headers.
+
+Dynamic - This wraps Static and has the same Certify
+method. However, it adds an Update method, which can be called
+with a FullCommit when the validator set changes. If it can
+prove this is a valid transition, it will update the validator
+set.
+
+Inquiring - this wraps Dynamic and implements an auto-update
+strategy on top of the Dynamic update. If a call to
+Certify fails as the validator set has changed, then it
+attempts to find a FullCommit and Update to that header.
+To get these FullCommits, it makes use of a Provider.
+
+Providers
+
+A Provider allows us to store and retrieve the FullCommits,
+to provide memory to the Inquiring Certifier.
+
+NewMemStoreProvider - in-memory cache.
+
+files.NewProvider - disk backed storage.
+
+client.NewHTTPProvider - query tendermint rpc.
+
+NewCacheProvider - combine multiple providers.
+
+The suggested use for local light clients is
+client.NewHTTPProvider for getting new data (Source),
+and NewCacheProvider(NewMemStoreProvider(),
+files.NewProvider()) to store confirmed headers (Trusted)
+
+How We Track Validators
+
+Unless you want to blindly trust the node you talk with, you
+need to trace every response back to a hash in a block header
+and validate the commit signatures of that block header match
+the proper validator set.  If there is a contant validator
+set, you store it locally upon initialization of the client,
+and check against that every time.
+
+Once there is a dynamic validator set, the issue of
+verifying a block becomes a bit more tricky. There is
+background information in a
+github issue (https://github.com/tendermint/tendermint/issues/377).
+
+In short, if there is a block at height H with a known
+(trusted) validator set V, and another block at height H'
+(H' > H) with validator set V' != V, then we want a way to
+safely update it.
+
+First, get the new (unconfirmed) validator set V' and
+verify H' is internally consistent and properly signed by
+this V'. Assuming it is a valid block, we check that at
+least 2/3 of the validators in V also signed it, meaning
+it would also be valid under our old assumptions.
+That should be enough, but we can also check that the
+V counts for at least 2/3 of the total votes in H'
+for extra safety (we can have a discussion if this is
+strictly required). If we can verify all this,
+then we can accept H' and V' as valid and use that to
+validate all blocks X > H'.
+
+If we cannot update directly from H -> H' because there was
+too much change to the validator set, then we can look for
+some Hm (H < Hm < H') with a validator set Vm.  Then we try
+to update H -> Hm and Hm -> H' in two separate steps.
+If one of these steps doesn't work, then we continue
+bisecting, until we eventually have to externally
+validate the valdiator set changes at every block.
+
+Since we never trust any server in this protocol, only the
+signatures themselves, it doesn't matter if the seed comes
+from a (possibly malicious) node or a (possibly malicious) user.
+We can accept it or reject it based only on our trusted
+validator set and cryptographic proofs. This makes it
+extremely important to verify that you have the proper
+validator set when initializing the client, as that is the
+root of all trust.
+
+Or course, this assumes that the known block is within the
+unbonding period to avoid the "nothing at stake" problem.
+If you haven't seen the state in a few months, you will need
+to manually verify the new validator set hash using off-chain
+means (the same as getting the initial hash).
+
+*/
+package lite

lite/dynamic_certifier.go

@@ -0,0 +1,96 @@
+package lite
+
+import (
+	"github.com/tendermint/tendermint/types"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+var _ Certifier = (*DynamicCertifier)(nil)
+
+// DynamicCertifier uses a StaticCertifier for Certify, but adds an
+// Update method to allow for a change of validators.
+//
+// You can pass in a FullCommit with another validator set,
+// and if this is a provably secure transition (< 1/3 change,
+// sufficient signatures), then it will update the
+// validator set for the next Certify call.
+// For security, it will only follow validator set changes
+// going forward.
+type DynamicCertifier struct {
+	cert       *StaticCertifier
+	lastHeight int64
+}
+
+// NewDynamic returns a new dynamic certifier.
+func NewDynamicCertifier(chainID string, vals *types.ValidatorSet, height int64) *DynamicCertifier {
+	return &DynamicCertifier{
+		cert:       NewStaticCertifier(chainID, vals),
+		lastHeight: height,
+	}
+}
+
+// ChainID returns the chain id of this certifier.
+// Implements Certifier.
+func (dc *DynamicCertifier) ChainID() string {
+	return dc.cert.ChainID()
+}
+
+// Validators returns the validators of this certifier.
+func (dc *DynamicCertifier) Validators() *types.ValidatorSet {
+	return dc.cert.vSet
+}
+
+// Hash returns the hash of this certifier.
+func (dc *DynamicCertifier) Hash() []byte {
+	return dc.cert.Hash()
+}
+
+// LastHeight returns the last height of this certifier.
+func (dc *DynamicCertifier) LastHeight() int64 {
+	return dc.lastHeight
+}
+
+// Certify will verify whether the commit is valid and will update the height if it is or return an
+// error if it is not.
+// Implements Certifier.
+func (dc *DynamicCertifier) Certify(check Commit) error {
+	err := dc.cert.Certify(check)
+	if err == nil {
+		// update last seen height if input is valid
+		dc.lastHeight = check.Height()
+	}
+	return err
+}
+
+// Update will verify if this is a valid change and update
+// the certifying validator set if safe to do so.
+//
+// Returns an error if update is impossible (invalid proof or IsTooMuchChangeErr)
+func (dc *DynamicCertifier) Update(fc FullCommit) error {
+	// ignore all checkpoints in the past -> only to the future
+	h := fc.Height()
+	if h <= dc.lastHeight {
+		return liteErr.ErrPastTime()
+	}
+
+	// first, verify if the input is self-consistent....
+	err := fc.ValidateBasic(dc.ChainID())
+	if err != nil {
+		return err
+	}
+
+	// now, make sure not too much change... meaning this commit
+	// would be approved by the currently known validator set
+	// as well as the new set
+	commit := fc.Commit.Commit
+	err = dc.Validators().VerifyCommitAny(fc.Validators, dc.ChainID(), commit.BlockID, h, commit)
+	if err != nil {
+		return liteErr.ErrTooMuchChange()
+	}
+
+	// looks good, we can update
+	dc.cert = NewStaticCertifier(dc.ChainID(), fc.Validators)
+	dc.lastHeight = h
+	return nil
+}

lite/dynamic_certifier_test.go

@@ -0,0 +1,130 @@
+package lite_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/types"
+
+	"github.com/tendermint/tendermint/lite"
+	"github.com/tendermint/tendermint/lite/errors"
+)
+
+// TestDynamicCert just makes sure it still works like StaticCert
+func TestDynamicCert(t *testing.T) {
+	// assert, require := assert.New(t), require.New(t)
+	assert := assert.New(t)
+	// require := require.New(t)
+
+	keys := lite.GenValKeys(4)
+	// 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do!
+	vals := keys.ToValidators(20, 10)
+	// and a certifier based on our known set
+	chainID := "test-dyno"
+	cert := lite.NewDynamicCertifier(chainID, vals, 0)
+
+	cases := []struct {
+		keys        lite.ValKeys
+		vals        *types.ValidatorSet
+		height      int64
+		first, last int  // who actually signs
+		proper      bool // true -> expect no error
+		changed     bool // true -> expect validator change error
+	}{
+		// perfect, signed by everyone
+		{keys, vals, 1, 0, len(keys), true, false},
+		// skip little guy is okay
+		{keys, vals, 2, 1, len(keys), true, false},
+		// but not the big guy
+		{keys, vals, 3, 0, len(keys) - 1, false, false},
+		// even changing the power a little bit breaks the static validator
+		// the sigs are enough, but the validator hash is unknown
+		{keys, keys.ToValidators(20, 11), 4, 0, len(keys), false, true},
+	}
+
+	for _, tc := range cases {
+		check := tc.keys.GenCommit(chainID, tc.height, nil, tc.vals,
+			[]byte("bar"), []byte("params"), []byte("results"), tc.first, tc.last)
+		err := cert.Certify(check)
+		if tc.proper {
+			assert.Nil(err, "%+v", err)
+			assert.Equal(cert.LastHeight(), tc.height)
+		} else {
+			assert.NotNil(err)
+			if tc.changed {
+				assert.True(errors.IsValidatorsChangedErr(err), "%+v", err)
+			}
+		}
+	}
+}
+
+// TestDynamicUpdate makes sure we update safely and sanely
+func TestDynamicUpdate(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	chainID := "test-dyno-up"
+	keys := lite.GenValKeys(5)
+	vals := keys.ToValidators(20, 0)
+	cert := lite.NewDynamicCertifier(chainID, vals, 40)
+
+	// one valid block to give us a sense of time
+	h := int64(100)
+	good := keys.GenCommit(chainID, h, nil, vals, []byte("foo"), []byte("params"), []byte("results"), 0, len(keys))
+	err := cert.Certify(good)
+	require.Nil(err, "%+v", err)
+
+	// some new sets to try later
+	keys2 := keys.Extend(2)
+	keys3 := keys2.Extend(4)
+
+	// we try to update with some blocks
+	cases := []struct {
+		keys        lite.ValKeys
+		vals        *types.ValidatorSet
+		height      int64
+		first, last int  // who actually signs
+		proper      bool // true -> expect no error
+		changed     bool // true -> expect too much change error
+	}{
+		// same validator set, well signed, of course it is okay
+		{keys, vals, h + 10, 0, len(keys), true, false},
+		// same validator set, poorly signed, fails
+		{keys, vals, h + 20, 2, len(keys), false, false},
+
+		// shift the power a little, works if properly signed
+		{keys, keys.ToValidators(10, 0), h + 30, 1, len(keys), true, false},
+		// but not on a poor signature
+		{keys, keys.ToValidators(10, 0), h + 40, 2, len(keys), false, false},
+		// and not if it was in the past
+		{keys, keys.ToValidators(10, 0), h + 25, 0, len(keys), false, false},
+
+		// let's try to adjust to a whole new validator set (we have 5/7 of the votes)
+		{keys2, keys2.ToValidators(10, 0), h + 33, 0, len(keys2), true, false},
+
+		// properly signed but too much change, not allowed (only 7/11 validators known)
+		{keys3, keys3.ToValidators(10, 0), h + 50, 0, len(keys3), false, true},
+	}
+
+	for _, tc := range cases {
+		fc := tc.keys.GenFullCommit(chainID, tc.height, nil, tc.vals,
+			[]byte("bar"), []byte("params"), []byte("results"), tc.first, tc.last)
+		err := cert.Update(fc)
+		if tc.proper {
+			assert.Nil(err, "%d: %+v", tc.height, err)
+			// we update last seen height
+			assert.Equal(cert.LastHeight(), tc.height)
+			// and we update the proper validators
+			assert.EqualValues(fc.Header.ValidatorsHash, cert.Hash())
+		} else {
+			assert.NotNil(err, "%d", tc.height)
+			// we don't update the height
+			assert.NotEqual(cert.LastHeight(), tc.height)
+			if tc.changed {
+				assert.True(errors.IsTooMuchChangeErr(err),
+					"%d: %+v", tc.height, err)
+			}
+		}
+	}
+}

lite/errors/errors.go

@@ -0,0 +1,92 @@
+package errors
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	errValidatorsChanged = fmt.Errorf("Validators differ between header and certifier")
+	errCommitNotFound    = fmt.Errorf("Commit not found by provider")
+	errTooMuchChange     = fmt.Errorf("Validators change too much to safely update")
+	errPastTime          = fmt.Errorf("Update older than certifier height")
+	errNoPathFound       = fmt.Errorf("Cannot find a path of validators")
+)
+
+// IsCommitNotFoundErr checks whether an error is due to missing data
+func IsCommitNotFoundErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errCommitNotFound)
+}
+
+// ErrCommitNotFound indicates that a the requested commit was not found.
+func ErrCommitNotFound() error {
+	return errors.WithStack(errCommitNotFound)
+}
+
+// IsValidatorsChangedErr checks whether an error is due
+// to a differing validator set.
+func IsValidatorsChangedErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errValidatorsChanged)
+}
+
+// ErrValidatorsChanged indicates that the validator set was changed between two commits.
+func ErrValidatorsChanged() error {
+	return errors.WithStack(errValidatorsChanged)
+}
+
+// IsTooMuchChangeErr checks whether an error is due to too much change
+// between these validators sets.
+func IsTooMuchChangeErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errTooMuchChange)
+}
+
+// ErrTooMuchChange indicates that the underlying validator set was changed by >1/3.
+func ErrTooMuchChange() error {
+	return errors.WithStack(errTooMuchChange)
+}
+
+// IsPastTimeErr ...
+func IsPastTimeErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errPastTime)
+}
+
+// ErrPastTime ...
+func ErrPastTime() error {
+	return errors.WithStack(errPastTime)
+}
+
+// IsNoPathFoundErr checks whether an error is due to no path of
+// validators in provider from where we are to where we want to be
+func IsNoPathFoundErr(err error) bool {
+	return err != nil && (errors.Cause(err) == errNoPathFound)
+}
+
+// ErrNoPathFound ...
+func ErrNoPathFound() error {
+	return errors.WithStack(errNoPathFound)
+}
+
+//--------------------------------------------
+
+type errHeightMismatch struct {
+	h1, h2 int64
+}
+
+func (e errHeightMismatch) Error() string {
+	return fmt.Sprintf("Blocks don't match - %d vs %d", e.h1, e.h2)
+}
+
+// IsHeightMismatchErr checks whether an error is due to data from different blocks
+func IsHeightMismatchErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	_, ok := errors.Cause(err).(errHeightMismatch)
+	return ok
+}
+
+// ErrHeightMismatch returns an mismatch error with stack-trace
+func ErrHeightMismatch(h1, h2 int64) error {
+	return errors.WithStack(errHeightMismatch{h1, h2})
+}

lite/errors/errors_test.go

@@ -0,0 +1,18 @@
+package errors
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestErrorHeight(t *testing.T) {
+	e1 := ErrHeightMismatch(2, 3)
+	e1.Error()
+	assert.True(t, IsHeightMismatchErr(e1))
+
+	e2 := errors.New("foobar")
+	assert.False(t, IsHeightMismatchErr(e2))
+	assert.False(t, IsHeightMismatchErr(nil))
+}

lite/files/commit.go

@@ -0,0 +1,93 @@
+package files
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+const (
+	// MaxFullCommitSize is the maximum number of bytes we will
+	// read in for a full commit to avoid excessive allocations
+	// in the deserializer
+	MaxFullCommitSize = 1024 * 1024
+)
+
+// SaveFullCommit exports the seed in binary / go-amino style
+func SaveFullCommit(fc lite.FullCommit, path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	defer f.Close()
+
+	_, err = cdc.MarshalBinaryWriter(f, fc)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+// SaveFullCommitJSON exports the seed in a json format
+func SaveFullCommitJSON(fc lite.FullCommit, path string) error {
+	f, err := os.Create(path)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	defer f.Close()
+	bz, err := cdc.MarshalJSON(fc)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	_, err = f.Write(bz)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	return nil
+}
+
+// LoadFullCommit loads the full commit from the file system.
+func LoadFullCommit(path string) (lite.FullCommit, error) {
+	var fc lite.FullCommit
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fc, liteErr.ErrCommitNotFound()
+		}
+		return fc, errors.WithStack(err)
+	}
+	defer f.Close()
+
+	_, err = cdc.UnmarshalBinaryReader(f, &fc, 0)
+	if err != nil {
+		return fc, errors.WithStack(err)
+	}
+	return fc, nil
+}
+
+// LoadFullCommitJSON loads the commit from the file system in JSON format.
+func LoadFullCommitJSON(path string) (lite.FullCommit, error) {
+	var fc lite.FullCommit
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fc, liteErr.ErrCommitNotFound()
+		}
+		return fc, errors.WithStack(err)
+	}
+	defer f.Close()
+
+	bz, err := ioutil.ReadAll(f)
+	if err != nil {
+		return fc, errors.WithStack(err)
+	}
+	err = cdc.UnmarshalJSON(bz, &fc)
+	if err != nil {
+		return fc, errors.WithStack(err)
+	}
+	return fc, nil
+}

lite/files/commit_test.go

@@ -0,0 +1,66 @@
+package files
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/lite"
+)
+
+func tmpFile() string {
+	suffix := cmn.RandStr(16)
+	return filepath.Join(os.TempDir(), "fc-test-"+suffix)
+}
+
+func TestSerializeFullCommits(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	// some constants
+	appHash := []byte("some crazy thing")
+	chainID := "ser-ial"
+	h := int64(25)
+
+	// build a fc
+	keys := lite.GenValKeys(5)
+	vals := keys.ToValidators(10, 0)
+	fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5)
+
+	require.Equal(h, fc.Height())
+	require.Equal(vals.Hash(), fc.ValidatorsHash())
+
+	// try read/write with json
+	jfile := tmpFile()
+	defer os.Remove(jfile)
+	jseed, err := LoadFullCommitJSON(jfile)
+	assert.NotNil(err)
+	err = SaveFullCommitJSON(fc, jfile)
+	require.Nil(err)
+	jseed, err = LoadFullCommitJSON(jfile)
+	assert.Nil(err, "%+v", err)
+	assert.Equal(h, jseed.Height())
+	assert.Equal(vals.Hash(), jseed.ValidatorsHash())
+
+	// try read/write with binary
+	bfile := tmpFile()
+	defer os.Remove(bfile)
+	bseed, err := LoadFullCommit(bfile)
+	assert.NotNil(err)
+	err = SaveFullCommit(fc, bfile)
+	require.Nil(err)
+	bseed, err = LoadFullCommit(bfile)
+	assert.Nil(err, "%+v", err)
+	assert.Equal(h, bseed.Height())
+	assert.Equal(vals.Hash(), bseed.ValidatorsHash())
+
+	// make sure they don't read the other format (different)
+	_, err = LoadFullCommit(jfile)
+	assert.NotNil(err)
+	_, err = LoadFullCommitJSON(bfile)
+	assert.NotNil(err)
+}

lite/files/provider.go

@@ -0,0 +1,139 @@
+/*
+Package files defines a Provider that stores all data in the filesystem
+
+We assume the same validator hash may be reused by many different
+headers/Commits, and thus store it separately. This leaves us
+with three issues:
+
+  1. Given a validator hash, retrieve the validator set if previously stored
+  2. Given a block height, find the Commit with the highest height <= h
+  3. Given a FullCommit, store it quickly to satisfy 1 and 2
+
+Note that we do not worry about caching, as that can be achieved by
+pairing this with a MemStoreProvider and CacheProvider from certifiers
+*/
+package files
+
+import (
+	"encoding/hex"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+// nolint
+const (
+	Ext      = ".tsd"
+	ValDir   = "validators"
+	CheckDir = "checkpoints"
+	dirPerm  = os.FileMode(0755)
+	//filePerm = os.FileMode(0644)
+)
+
+type provider struct {
+	valDir   string
+	checkDir string
+}
+
+// NewProvider creates the parent dir and subdirs
+// for validators and checkpoints as needed
+func NewProvider(dir string) lite.Provider {
+	valDir := filepath.Join(dir, ValDir)
+	checkDir := filepath.Join(dir, CheckDir)
+	for _, d := range []string{valDir, checkDir} {
+		err := os.MkdirAll(d, dirPerm)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return &provider{valDir: valDir, checkDir: checkDir}
+}
+
+func (p *provider) encodeHash(hash []byte) string {
+	return hex.EncodeToString(hash) + Ext
+}
+
+func (p *provider) encodeHeight(h int64) string {
+	// pad up to 10^12 for height...
+	return fmt.Sprintf("%012d%s", h, Ext)
+}
+
+// StoreCommit saves a full commit after it has been verified.
+func (p *provider) StoreCommit(fc lite.FullCommit) error {
+	// make sure the fc is self-consistent before saving
+	err := fc.ValidateBasic(fc.Commit.Header.ChainID)
+	if err != nil {
+		return err
+	}
+
+	paths := []string{
+		filepath.Join(p.checkDir, p.encodeHeight(fc.Height())),
+		filepath.Join(p.valDir, p.encodeHash(fc.Header.ValidatorsHash)),
+	}
+	for _, path := range paths {
+		err := SaveFullCommit(fc, path)
+		// unknown error in creating or writing immediately breaks
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// GetByHeight returns the closest commit with height <= h.
+func (p *provider) GetByHeight(h int64) (lite.FullCommit, error) {
+	// first we look for exact match, then search...
+	path := filepath.Join(p.checkDir, p.encodeHeight(h))
+	fc, err := LoadFullCommit(path)
+	if liteErr.IsCommitNotFoundErr(err) {
+		path, err = p.searchForHeight(h)
+		if err == nil {
+			fc, err = LoadFullCommit(path)
+		}
+	}
+	return fc, err
+}
+
+// LatestCommit returns the newest commit stored.
+func (p *provider) LatestCommit() (fc lite.FullCommit, err error) {
+	// Note to future: please update by 2077 to avoid rollover
+	return p.GetByHeight(math.MaxInt32 - 1)
+}
+
+// search for height, looks for a file with highest height < h
+// return certifiers.ErrCommitNotFound() if not there...
+func (p *provider) searchForHeight(h int64) (string, error) {
+	d, err := os.Open(p.checkDir)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	files, err := d.Readdirnames(0)
+
+	d.Close()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	desired := p.encodeHeight(h)
+	sort.Strings(files)
+	i := sort.SearchStrings(files, desired)
+	if i == 0 {
+		return "", liteErr.ErrCommitNotFound()
+	}
+	found := files[i-1]
+	path := filepath.Join(p.checkDir, found)
+	return path, errors.WithStack(err)
+}
+
+// GetByHash returns a commit exactly matching this validator hash.
+func (p *provider) GetByHash(hash []byte) (lite.FullCommit, error) {
+	path := filepath.Join(p.valDir, p.encodeHash(hash))
+	return LoadFullCommit(path)
+}

lite/files/provider_test.go

@@ -0,0 +1,96 @@
+package files_test
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+	"github.com/tendermint/tendermint/lite/files"
+)
+
+func checkEqual(stored, loaded lite.FullCommit, chainID string) error {
+	err := loaded.ValidateBasic(chainID)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(stored.ValidatorsHash(), loaded.ValidatorsHash()) {
+		return errors.New("Different block hashes")
+	}
+	return nil
+}
+
+func TestFileProvider(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	dir, err := ioutil.TempDir("", "fileprovider-test")
+	assert.Nil(err)
+	defer os.RemoveAll(dir)
+	p := files.NewProvider(dir)
+
+	chainID := "test-files"
+	appHash := []byte("some-data")
+	keys := lite.GenValKeys(5)
+	count := 10
+
+	// make a bunch of seeds...
+	seeds := make([]lite.FullCommit, count)
+	for i := 0; i < count; i++ {
+		// two seeds for each validator, to check how we handle dups
+		// (10, 0), (10, 1), (10, 1), (10, 2), (10, 2), ...
+		vals := keys.ToValidators(10, int64(count/2))
+		h := int64(20 + 10*i)
+		check := keys.GenCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5)
+		seeds[i] = lite.NewFullCommit(check, vals)
+	}
+
+	// check provider is empty
+	seed, err := p.GetByHeight(20)
+	require.NotNil(err)
+	assert.True(liteErr.IsCommitNotFoundErr(err))
+
+	seed, err = p.GetByHash(seeds[3].ValidatorsHash())
+	require.NotNil(err)
+	assert.True(liteErr.IsCommitNotFoundErr(err))
+
+	// now add them all to the provider
+	for _, s := range seeds {
+		err = p.StoreCommit(s)
+		require.Nil(err)
+		// and make sure we can get it back
+		s2, err := p.GetByHash(s.ValidatorsHash())
+		assert.Nil(err)
+		err = checkEqual(s, s2, chainID)
+		assert.Nil(err)
+		// by height as well
+		s2, err = p.GetByHeight(s.Height())
+		err = checkEqual(s, s2, chainID)
+		assert.Nil(err)
+	}
+
+	// make sure we get the last hash if we overstep
+	seed, err = p.GetByHeight(5000)
+	if assert.Nil(err, "%+v", err) {
+		assert.Equal(seeds[count-1].Height(), seed.Height())
+		err = checkEqual(seeds[count-1], seed, chainID)
+		assert.Nil(err)
+	}
+
+	// and middle ones as well
+	seed, err = p.GetByHeight(47)
+	if assert.Nil(err, "%+v", err) {
+		// we only step by 10, so 40 must be the one below this
+		assert.EqualValues(40, seed.Height())
+	}
+
+	// and proper error for too low
+	_, err = p.GetByHeight(5)
+	assert.NotNil(err)
+	assert.True(liteErr.IsCommitNotFoundErr(err))
+}

lite/files/wire.go

@@ -0,0 +1,12 @@
+package files
+
+import (
+	"github.com/tendermint/go-amino"
+	"github.com/tendermint/tendermint/crypto"
+)
+
+var cdc = amino.NewCodec()
+
+func init() {
+	crypto.RegisterAmino(cdc)
+}

lite/helpers.go

@@ -0,0 +1,158 @@
+package lite
+
+import (
+	"time"
+
+	crypto "github.com/tendermint/tendermint/crypto"
+
+	"github.com/tendermint/tendermint/types"
+)
+
+// ValKeys is a helper for testing.
+//
+// It lets us simulate signing with many keys, either ed25519 or secp256k1.
+// The main use case is to create a set, and call GenCommit
+// to get properly signed header for testing.
+//
+// You can set different weights of validators each time you call
+// ToValidators, and can optionally extend the validator set later
+// with Extend or ExtendSecp
+type ValKeys []crypto.PrivKey
+
+// GenValKeys produces an array of private keys to generate commits.
+func GenValKeys(n int) ValKeys {
+	res := make(ValKeys, n)
+	for i := range res {
+		res[i] = crypto.GenPrivKeyEd25519()
+	}
+	return res
+}
+
+// Change replaces the key at index i.
+func (v ValKeys) Change(i int) ValKeys {
+	res := make(ValKeys, len(v))
+	copy(res, v)
+	res[i] = crypto.GenPrivKeyEd25519()
+	return res
+}
+
+// Extend adds n more keys (to remove, just take a slice).
+func (v ValKeys) Extend(n int) ValKeys {
+	extra := GenValKeys(n)
+	return append(v, extra...)
+}
+
+// GenSecpValKeys produces an array of secp256k1 private keys to generate commits.
+func GenSecpValKeys(n int) ValKeys {
+	res := make(ValKeys, n)
+	for i := range res {
+		res[i] = crypto.GenPrivKeySecp256k1()
+	}
+	return res
+}
+
+// ExtendSecp adds n more secp256k1 keys (to remove, just take a slice).
+func (v ValKeys) ExtendSecp(n int) ValKeys {
+	extra := GenSecpValKeys(n)
+	return append(v, extra...)
+}
+
+// ToValidators produces a list of validators from the set of keys
+// The first key has weight `init` and it increases by `inc` every step
+// so we can have all the same weight, or a simple linear distribution
+// (should be enough for testing).
+func (v ValKeys) ToValidators(init, inc int64) *types.ValidatorSet {
+	res := make([]*types.Validator, len(v))
+	for i, k := range v {
+		res[i] = types.NewValidator(k.PubKey(), init+int64(i)*inc)
+	}
+	return types.NewValidatorSet(res)
+}
+
+// signHeader properly signs the header with all keys from first to last exclusive.
+func (v ValKeys) signHeader(header *types.Header, first, last int) *types.Commit {
+	votes := make([]*types.Vote, len(v))
+
+	// we need this list to keep the ordering...
+	vset := v.ToValidators(1, 0)
+
+	// fill in the votes we want
+	for i := first; i < last && i < len(v); i++ {
+		vote := makeVote(header, vset, v[i])
+		votes[vote.ValidatorIndex] = vote
+	}
+
+	res := &types.Commit{
+		BlockID:    types.BlockID{Hash: header.Hash()},
+		Precommits: votes,
+	}
+	return res
+}
+
+func makeVote(header *types.Header, vals *types.ValidatorSet, key crypto.PrivKey) *types.Vote {
+	addr := key.PubKey().Address()
+	idx, _ := vals.GetByAddress(addr)
+	vote := &types.Vote{
+		ValidatorAddress: addr,
+		ValidatorIndex:   idx,
+		Height:           header.Height,
+		Round:            1,
+		Timestamp:        time.Now().UTC(),
+		Type:             types.VoteTypePrecommit,
+		BlockID:          types.BlockID{Hash: header.Hash()},
+	}
+	// Sign it
+	signBytes := vote.SignBytes(header.ChainID)
+	sig, err := key.Sign(signBytes)
+	if err != nil {
+		panic(err)
+	}
+	vote.Signature = sig
+
+	return vote
+}
+
+// Silences warning that vals can also be merkle.Hashable
+// nolint: interfacer
+func genHeader(chainID string, height int64, txs types.Txs,
+	vals *types.ValidatorSet, appHash, consHash, resHash []byte) *types.Header {
+
+	return &types.Header{
+		ChainID:  chainID,
+		Height:   height,
+		Time:     time.Now(),
+		NumTxs:   int64(len(txs)),
+		TotalTxs: int64(len(txs)),
+		// LastBlockID
+		// LastCommitHash
+		ValidatorsHash:  vals.Hash(),
+		DataHash:        txs.Hash(),
+		AppHash:         appHash,
+		ConsensusHash:   consHash,
+		LastResultsHash: resHash,
+	}
+}
+
+// GenCommit calls genHeader and signHeader and combines them into a Commit.
+func (v ValKeys) GenCommit(chainID string, height int64, txs types.Txs,
+	vals *types.ValidatorSet, appHash, consHash, resHash []byte, first, last int) Commit {
+
+	header := genHeader(chainID, height, txs, vals, appHash, consHash, resHash)
+	check := Commit{
+		Header: header,
+		Commit: v.signHeader(header, first, last),
+	}
+	return check
+}
+
+// GenFullCommit calls genHeader and signHeader and combines them into a Commit.
+func (v ValKeys) GenFullCommit(chainID string, height int64, txs types.Txs,
+	vals *types.ValidatorSet, appHash, consHash, resHash []byte, first, last int) FullCommit {
+
+	header := genHeader(chainID, height, txs, vals, appHash, consHash, resHash)
+	commit := Commit{
+		Header: header,
+		Commit: v.signHeader(header, first, last),
+	}
+	return NewFullCommit(commit, vals)
+}

lite/inquiring_certifier.go

@@ -0,0 +1,163 @@
+package lite
+
+import (
+	"github.com/tendermint/tendermint/types"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+var _ Certifier = (*InquiringCertifier)(nil)
+
+// InquiringCertifier wraps a dynamic certifier and implements an auto-update strategy. If a call
+// to Certify fails due to a change it validator set, InquiringCertifier will try and find a
+// previous FullCommit which it can use to safely update the validator set. It uses a source
+// provider to obtain the needed FullCommits. It stores properly validated data on the local system.
+type InquiringCertifier struct {
+	cert *DynamicCertifier
+	// These are only properly validated data, from local system
+	trusted Provider
+	// This is a source of new info, like a node rpc, or other import method
+	Source Provider
+}
+
+// NewInquiringCertifier returns a new Inquiring object. It uses the trusted provider to store
+// validated data and the source provider to obtain missing FullCommits.
+//
+// Example: The trusted provider should a CacheProvider, MemProvider or files.Provider. The source
+// provider should be a client.HTTPProvider.
+func NewInquiringCertifier(chainID string, fc FullCommit, trusted Provider,
+	source Provider) (*InquiringCertifier, error) {
+
+	// store the data in trusted
+	err := trusted.StoreCommit(fc)
+	if err != nil {
+		return nil, err
+	}
+
+	return &InquiringCertifier{
+		cert:    NewDynamicCertifier(chainID, fc.Validators, fc.Height()),
+		trusted: trusted,
+		Source:  source,
+	}, nil
+}
+
+// ChainID returns the chain id.
+// Implements Certifier.
+func (ic *InquiringCertifier) ChainID() string {
+	return ic.cert.ChainID()
+}
+
+// Validators returns the validator set.
+func (ic *InquiringCertifier) Validators() *types.ValidatorSet {
+	return ic.cert.cert.vSet
+}
+
+// LastHeight returns the last height.
+func (ic *InquiringCertifier) LastHeight() int64 {
+	return ic.cert.lastHeight
+}
+
+// Certify makes sure this is checkpoint is valid.
+//
+// If the validators have changed since the last know time, it looks
+// for a path to prove the new validators.
+//
+// On success, it will store the checkpoint in the store for later viewing
+// Implements Certifier.
+func (ic *InquiringCertifier) Certify(commit Commit) error {
+	err := ic.useClosestTrust(commit.Height())
+	if err != nil {
+		return err
+	}
+
+	err = ic.cert.Certify(commit)
+	if !liteErr.IsValidatorsChangedErr(err) {
+		return err
+	}
+	err = ic.updateToHash(commit.Header.ValidatorsHash)
+	if err != nil {
+		return err
+	}
+
+	err = ic.cert.Certify(commit)
+	if err != nil {
+		return err
+	}
+
+	// store the new checkpoint
+	return ic.trusted.StoreCommit(NewFullCommit(commit, ic.Validators()))
+}
+
+// Update will verify if this is a valid change and update
+// the certifying validator set if safe to do so.
+func (ic *InquiringCertifier) Update(fc FullCommit) error {
+	err := ic.useClosestTrust(fc.Height())
+	if err != nil {
+		return err
+	}
+
+	err = ic.cert.Update(fc)
+	if err == nil {
+		err = ic.trusted.StoreCommit(fc)
+	}
+	return err
+}
+
+func (ic *InquiringCertifier) useClosestTrust(h int64) error {
+	closest, err := ic.trusted.GetByHeight(h)
+	if err != nil {
+		return err
+	}
+
+	// if the best seed is not the one we currently use,
+	// let's just reset the dynamic validator
+	if closest.Height() != ic.LastHeight() {
+		ic.cert = NewDynamicCertifier(ic.ChainID(), closest.Validators, closest.Height())
+	}
+	return nil
+}
+
+// updateToHash gets the validator hash we want to update to
+// if IsTooMuchChangeErr, we try to find a path by binary search over height
+func (ic *InquiringCertifier) updateToHash(vhash []byte) error {
+	// try to get the match, and update
+	fc, err := ic.Source.GetByHash(vhash)
+	if err != nil {
+		return err
+	}
+	err = ic.cert.Update(fc)
+	// handle IsTooMuchChangeErr by using divide and conquer
+	if liteErr.IsTooMuchChangeErr(err) {
+		err = ic.updateToHeight(fc.Height())
+	}
+	return err
+}
+
+// updateToHeight will use divide-and-conquer to find a path to h
+func (ic *InquiringCertifier) updateToHeight(h int64) error {
+	// try to update to this height (with checks)
+	fc, err := ic.Source.GetByHeight(h)
+	if err != nil {
+		return err
+	}
+	start, end := ic.LastHeight(), fc.Height()
+	if end <= start {
+		return liteErr.ErrNoPathFound()
+	}
+	err = ic.Update(fc)
+
+	// we can handle IsTooMuchChangeErr specially
+	if !liteErr.IsTooMuchChangeErr(err) {
+		return err
+	}
+
+	// try to update to mid
+	mid := (start + end) / 2
+	err = ic.updateToHeight(mid)
+	if err != nil {
+		return err
+	}
+
+	// if we made it to mid, we recurse
+	return ic.updateToHeight(h)
+}

lite/inquiring_certifier_test.go

@@ -0,0 +1,173 @@
+// nolint: vetshadow
+package lite_test
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/lite"
+)
+
+func TestInquirerValidPath(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	trust := lite.NewMemStoreProvider()
+	source := lite.NewMemStoreProvider()
+
+	// set up the validators to generate test blocks
+	var vote int64 = 10
+	keys := lite.GenValKeys(5)
+
+	// construct a bunch of commits, each with one more height than the last
+	chainID := "inquiry-test"
+	consHash := []byte("params")
+	resHash := []byte("results")
+	count := 50
+	commits := make([]lite.FullCommit, count)
+	for i := 0; i < count; i++ {
+		// extend the keys by 1 each time
+		keys = keys.Extend(1)
+		vals := keys.ToValidators(vote, 0)
+		h := int64(20 + 10*i)
+		appHash := []byte(fmt.Sprintf("h=%d", h))
+		commits[i] = keys.GenFullCommit(chainID, h, nil, vals, appHash, consHash, resHash, 0,
+			len(keys))
+	}
+
+	// initialize a certifier with the initial state
+	cert, err := lite.NewInquiringCertifier(chainID, commits[0], trust, source)
+	require.Nil(err)
+
+	// this should fail validation....
+	commit := commits[count-1].Commit
+	err = cert.Certify(commit)
+	require.NotNil(err)
+
+	// adding a few commits in the middle should be insufficient
+	for i := 10; i < 13; i++ {
+		err := source.StoreCommit(commits[i])
+		require.Nil(err)
+	}
+	err = cert.Certify(commit)
+	assert.NotNil(err)
+
+	// with more info, we succeed
+	for i := 0; i < count; i++ {
+		err := source.StoreCommit(commits[i])
+		require.Nil(err)
+	}
+	err = cert.Certify(commit)
+	assert.Nil(err, "%+v", err)
+}
+
+func TestInquirerMinimalPath(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	trust := lite.NewMemStoreProvider()
+	source := lite.NewMemStoreProvider()
+
+	// set up the validators to generate test blocks
+	var vote int64 = 10
+	keys := lite.GenValKeys(5)
+
+	// construct a bunch of commits, each with one more height than the last
+	chainID := "minimal-path"
+	consHash := []byte("other-params")
+	count := 12
+	commits := make([]lite.FullCommit, count)
+	for i := 0; i < count; i++ {
+		// extend the validators, so we are just below 2/3
+		keys = keys.Extend(len(keys)/2 - 1)
+		vals := keys.ToValidators(vote, 0)
+		h := int64(5 + 10*i)
+		appHash := []byte(fmt.Sprintf("h=%d", h))
+		resHash := []byte(fmt.Sprintf("res=%d", h))
+		commits[i] = keys.GenFullCommit(chainID, h, nil, vals, appHash, consHash, resHash, 0,
+			len(keys))
+	}
+
+	// initialize a certifier with the initial state
+	cert, _ := lite.NewInquiringCertifier(chainID, commits[0], trust, source)
+
+	// this should fail validation....
+	commit := commits[count-1].Commit
+	err := cert.Certify(commit)
+	require.NotNil(err)
+
+	// add a few seed in the middle should be insufficient
+	for i := 5; i < 8; i++ {
+		err := source.StoreCommit(commits[i])
+		require.Nil(err)
+	}
+	err = cert.Certify(commit)
+	assert.NotNil(err)
+
+	// with more info, we succeed
+	for i := 0; i < count; i++ {
+		err := source.StoreCommit(commits[i])
+		require.Nil(err)
+	}
+	err = cert.Certify(commit)
+	assert.Nil(err, "%+v", err)
+}
+
+func TestInquirerVerifyHistorical(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	trust := lite.NewMemStoreProvider()
+	source := lite.NewMemStoreProvider()
+
+	// set up the validators to generate test blocks
+	var vote int64 = 10
+	keys := lite.GenValKeys(5)
+
+	// construct a bunch of commits, each with one more height than the last
+	chainID := "inquiry-test"
+	count := 10
+	consHash := []byte("special-params")
+	commits := make([]lite.FullCommit, count)
+	for i := 0; i < count; i++ {
+		// extend the keys by 1 each time
+		keys = keys.Extend(1)
+		vals := keys.ToValidators(vote, 0)
+		h := int64(20 + 10*i)
+		appHash := []byte(fmt.Sprintf("h=%d", h))
+		resHash := []byte(fmt.Sprintf("res=%d", h))
+		commits[i] = keys.GenFullCommit(chainID, h, nil, vals, appHash, consHash, resHash, 0,
+			len(keys))
+	}
+
+	// initialize a certifier with the initial state
+	cert, _ := lite.NewInquiringCertifier(chainID, commits[0], trust, source)
+
+	// store a few commits as trust
+	for _, i := range []int{2, 5} {
+		trust.StoreCommit(commits[i])
+	}
+
+	// let's see if we can jump forward using trusted commits
+	err := source.StoreCommit(commits[7])
+	require.Nil(err, "%+v", err)
+	check := commits[7].Commit
+	err = cert.Certify(check)
+	require.Nil(err, "%+v", err)
+	assert.Equal(check.Height(), cert.LastHeight())
+
+	// add access to all commits via untrusted source
+	for i := 0; i < count; i++ {
+		err := source.StoreCommit(commits[i])
+		require.Nil(err)
+	}
+
+	// try to check an unknown seed in the past
+	mid := commits[3].Commit
+	err = cert.Certify(mid)
+	require.Nil(err, "%+v", err)
+	assert.Equal(mid.Height(), cert.LastHeight())
+
+	// and jump all the way forward again
+	end := commits[count-1].Commit
+	err = cert.Certify(end)
+	require.Nil(err, "%+v", err)
+	assert.Equal(end.Height(), cert.LastHeight())
+}

lite/memprovider.go

@@ -0,0 +1,152 @@
+package lite
+
+import (
+	"encoding/hex"
+	"sort"
+	"sync"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+type memStoreProvider struct {
+	mtx sync.RWMutex
+	// byHeight is always sorted by Height... need to support range search (nil, h]
+	// btree would be more efficient for larger sets
+	byHeight fullCommits
+	byHash   map[string]FullCommit
+
+	sorted bool
+}
+
+// fullCommits just exists to allow easy sorting
+type fullCommits []FullCommit
+
+func (s fullCommits) Len() int      { return len(s) }
+func (s fullCommits) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s fullCommits) Less(i, j int) bool {
+	return s[i].Height() < s[j].Height()
+}
+
+// NewMemStoreProvider returns a new in-memory provider.
+func NewMemStoreProvider() Provider {
+	return &memStoreProvider{
+		byHeight: fullCommits{},
+		byHash:   map[string]FullCommit{},
+	}
+}
+
+func (m *memStoreProvider) encodeHash(hash []byte) string {
+	return hex.EncodeToString(hash)
+}
+
+// StoreCommit stores a FullCommit after verifying it.
+func (m *memStoreProvider) StoreCommit(fc FullCommit) error {
+	// make sure the fc is self-consistent before saving
+	err := fc.ValidateBasic(fc.Commit.Header.ChainID)
+	if err != nil {
+		return err
+	}
+
+	// store the valid fc
+	key := m.encodeHash(fc.ValidatorsHash())
+
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	m.byHash[key] = fc
+	m.byHeight = append(m.byHeight, fc)
+	m.sorted = false
+	return nil
+}
+
+// GetByHeight returns the FullCommit for height h or an error if the commit is not found.
+func (m *memStoreProvider) GetByHeight(h int64) (FullCommit, error) {
+	// By heuristics, GetByHeight with linearsearch is fast enough
+	// for about 50 keys but after that, it needs binary search.
+	// See https://github.com/tendermint/tendermint/pull/1043#issue-285188242
+	m.mtx.RLock()
+	n := len(m.byHeight)
+	m.mtx.RUnlock()
+
+	if n <= 50 {
+		return m.getByHeightLinearSearch(h)
+	}
+	return m.getByHeightBinarySearch(h)
+}
+
+func (m *memStoreProvider) sortByHeightIfNecessaryLocked() {
+	if !m.sorted {
+		sort.Sort(m.byHeight)
+		m.sorted = true
+	}
+}
+
+func (m *memStoreProvider) getByHeightLinearSearch(h int64) (FullCommit, error) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	m.sortByHeightIfNecessaryLocked()
+	// search from highest to lowest
+	for i := len(m.byHeight) - 1; i >= 0; i-- {
+		if fc := m.byHeight[i]; fc.Height() <= h {
+			return fc, nil
+		}
+	}
+	return FullCommit{}, liteErr.ErrCommitNotFound()
+}
+
+func (m *memStoreProvider) getByHeightBinarySearch(h int64) (FullCommit, error) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	m.sortByHeightIfNecessaryLocked()
+	low, high := 0, len(m.byHeight)-1
+	var mid int
+	var hmid int64
+	var midFC FullCommit
+	// Our goal is to either find:
+	//   * item ByHeight with the query
+	//   * greatest height with a height <= query
+	for low <= high {
+		mid = int(uint(low+high) >> 1) // Avoid an overflow
+		midFC = m.byHeight[mid]
+		hmid = midFC.Height()
+		switch {
+		case hmid == h:
+			return midFC, nil
+		case hmid < h:
+			low = mid + 1
+		case hmid > h:
+			high = mid - 1
+		}
+	}
+
+	if high >= 0 {
+		if highFC := m.byHeight[high]; highFC.Height() < h {
+			return highFC, nil
+		}
+	}
+	return FullCommit{}, liteErr.ErrCommitNotFound()
+}
+
+// GetByHash returns the FullCommit for the hash or an error if the commit is not found.
+func (m *memStoreProvider) GetByHash(hash []byte) (FullCommit, error) {
+	m.mtx.RLock()
+	defer m.mtx.RUnlock()
+
+	fc, ok := m.byHash[m.encodeHash(hash)]
+	if !ok {
+		return fc, liteErr.ErrCommitNotFound()
+	}
+	return fc, nil
+}
+
+// LatestCommit returns the latest FullCommit or an error if no commits exist.
+func (m *memStoreProvider) LatestCommit() (FullCommit, error) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+
+	l := len(m.byHeight)
+	if l == 0 {
+		return FullCommit{}, liteErr.ErrCommitNotFound()
+	}
+	m.sortByHeightIfNecessaryLocked()
+	return m.byHeight[l-1], nil
+}

lite/performance_test.go

@@ -0,0 +1,365 @@
+package lite
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+func TestMemStoreProvidergetByHeightBinaryAndLinearSameResult(t *testing.T) {
+	p := NewMemStoreProvider().(*memStoreProvider)
+
+	// Store a bunch of commits at specific heights
+	// and then ensure that:
+	//  * getByHeightLinearSearch
+	//  * getByHeightBinarySearch
+	// both return the exact same result
+
+	// 1. Non-existent height commits
+	nonExistent := []int64{-1000, -1, 0, 1, 10, 11, 17, 31, 67, 1000, 1e9}
+	ensureNonExistentCommitsAtHeight(t, "getByHeightLinearSearch", p.getByHeightLinearSearch, nonExistent)
+	ensureNonExistentCommitsAtHeight(t, "getByHeightBinarySearch", p.getByHeightBinarySearch, nonExistent)
+
+	// 2. Save some known height commits
+	knownHeights := []int64{0, 1, 7, 9, 12, 13, 18, 44, 23, 16, 1024, 100, 199, 1e9}
+	createAndStoreCommits(t, p, knownHeights)
+
+	// 3. Now check if those heights are retrieved
+	ensureExistentCommitsAtHeight(t, "getByHeightLinearSearch", p.getByHeightLinearSearch, knownHeights)
+	ensureExistentCommitsAtHeight(t, "getByHeightBinarySearch", p.getByHeightBinarySearch, knownHeights)
+
+	// 4. And now for the height probing to ensure that any height
+	// requested returns a fullCommit of height <= requestedHeight.
+	comparegetByHeightAlgorithms(t, p, 0, 0)
+	comparegetByHeightAlgorithms(t, p, 1, 1)
+	comparegetByHeightAlgorithms(t, p, 2, 1)
+	comparegetByHeightAlgorithms(t, p, 5, 1)
+	comparegetByHeightAlgorithms(t, p, 7, 7)
+	comparegetByHeightAlgorithms(t, p, 10, 9)
+	comparegetByHeightAlgorithms(t, p, 12, 12)
+	comparegetByHeightAlgorithms(t, p, 14, 13)
+	comparegetByHeightAlgorithms(t, p, 19, 18)
+	comparegetByHeightAlgorithms(t, p, 43, 23)
+	comparegetByHeightAlgorithms(t, p, 45, 44)
+	comparegetByHeightAlgorithms(t, p, 1025, 1024)
+	comparegetByHeightAlgorithms(t, p, 101, 100)
+	comparegetByHeightAlgorithms(t, p, 1e3, 199)
+	comparegetByHeightAlgorithms(t, p, 1e4, 1024)
+	comparegetByHeightAlgorithms(t, p, 1e9, 1e9)
+	comparegetByHeightAlgorithms(t, p, 1e9+1, 1e9)
+}
+
+func createAndStoreCommits(t *testing.T, p Provider, heights []int64) {
+	chainID := "cache-best-height-binary-and-linear"
+	appHash := []byte("0xdeadbeef")
+	keys := GenValKeys(len(heights) / 2)
+
+	for _, h := range heights {
+		vals := keys.ToValidators(10, int64(len(heights)/2))
+		fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5)
+		err := p.StoreCommit(fc)
+		require.NoError(t, err, "StoreCommit height=%d", h)
+	}
+}
+
+func comparegetByHeightAlgorithms(t *testing.T, p *memStoreProvider, ask, expect int64) {
+	algos := map[string]func(int64) (FullCommit, error){
+		"getHeightByLinearSearch": p.getByHeightLinearSearch,
+		"getHeightByBinarySearch": p.getByHeightBinarySearch,
+	}
+
+	for algo, fn := range algos {
+		fc, err := fn(ask)
+		// t.Logf("%s got=%v want=%d", algo, expect, fc.Height())
+		require.Nil(t, err, "%s: %+v", algo, err)
+		if assert.Equal(t, expect, fc.Height()) {
+			err = p.StoreCommit(fc)
+			require.Nil(t, err, "%s: %+v", algo, err)
+		}
+	}
+}
+
+var blankFullCommit FullCommit
+
+func ensureNonExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (FullCommit, error), data []int64) {
+	for i, qh := range data {
+		fc, err := fn(qh)
+		assert.NotNil(t, err, "#%d: %s: height=%d should return non-nil error", i, prefix, qh)
+		assert.Equal(t, fc, blankFullCommit, "#%d: %s: height=%d\ngot =%+v\nwant=%+v", i, prefix, qh, fc, blankFullCommit)
+	}
+}
+
+func ensureExistentCommitsAtHeight(t *testing.T, prefix string, fn func(int64) (FullCommit, error), data []int64) {
+	for i, qh := range data {
+		fc, err := fn(qh)
+		assert.Nil(t, err, "#%d: %s: height=%d should not return an error: %v", i, prefix, qh, err)
+		assert.NotEqual(t, fc, blankFullCommit, "#%d: %s: height=%d got a blankCommit", i, prefix, qh)
+	}
+}
+
+func BenchmarkGenCommit20(b *testing.B) {
+	keys := GenValKeys(20)
+	benchmarkGenCommit(b, keys)
+}
+
+func BenchmarkGenCommit100(b *testing.B) {
+	keys := GenValKeys(100)
+	benchmarkGenCommit(b, keys)
+}
+
+func BenchmarkGenCommitSec20(b *testing.B) {
+	keys := GenSecpValKeys(20)
+	benchmarkGenCommit(b, keys)
+}
+
+func BenchmarkGenCommitSec100(b *testing.B) {
+	keys := GenSecpValKeys(100)
+	benchmarkGenCommit(b, keys)
+}
+
+func benchmarkGenCommit(b *testing.B, keys ValKeys) {
+	chainID := fmt.Sprintf("bench-%d", len(keys))
+	vals := keys.ToValidators(20, 10)
+	for i := 0; i < b.N; i++ {
+		h := int64(1 + i)
+		appHash := []byte(fmt.Sprintf("h=%d", h))
+		resHash := []byte(fmt.Sprintf("res=%d", h))
+		keys.GenCommit(chainID, h, nil, vals, appHash, []byte("params"), resHash, 0, len(keys))
+	}
+}
+
+// this benchmarks generating one key
+func BenchmarkGenValKeys(b *testing.B) {
+	keys := GenValKeys(20)
+	for i := 0; i < b.N; i++ {
+		keys = keys.Extend(1)
+	}
+}
+
+// this benchmarks generating one key
+func BenchmarkGenSecpValKeys(b *testing.B) {
+	keys := GenSecpValKeys(20)
+	for i := 0; i < b.N; i++ {
+		keys = keys.Extend(1)
+	}
+}
+
+func BenchmarkToValidators20(b *testing.B) {
+	benchmarkToValidators(b, 20)
+}
+
+func BenchmarkToValidators100(b *testing.B) {
+	benchmarkToValidators(b, 100)
+}
+
+// this benchmarks constructing the validator set (.PubKey() * nodes)
+func benchmarkToValidators(b *testing.B, nodes int) {
+	keys := GenValKeys(nodes)
+	for i := 1; i <= b.N; i++ {
+		keys.ToValidators(int64(2*i), int64(i))
+	}
+}
+
+func BenchmarkToValidatorsSec100(b *testing.B) {
+	benchmarkToValidatorsSec(b, 100)
+}
+
+// this benchmarks constructing the validator set (.PubKey() * nodes)
+func benchmarkToValidatorsSec(b *testing.B, nodes int) {
+	keys := GenSecpValKeys(nodes)
+	for i := 1; i <= b.N; i++ {
+		keys.ToValidators(int64(2*i), int64(i))
+	}
+}
+
+func BenchmarkCertifyCommit20(b *testing.B) {
+	keys := GenValKeys(20)
+	benchmarkCertifyCommit(b, keys)
+}
+
+func BenchmarkCertifyCommit100(b *testing.B) {
+	keys := GenValKeys(100)
+	benchmarkCertifyCommit(b, keys)
+}
+
+func BenchmarkCertifyCommitSec20(b *testing.B) {
+	keys := GenSecpValKeys(20)
+	benchmarkCertifyCommit(b, keys)
+}
+
+func BenchmarkCertifyCommitSec100(b *testing.B) {
+	keys := GenSecpValKeys(100)
+	benchmarkCertifyCommit(b, keys)
+}
+
+func benchmarkCertifyCommit(b *testing.B, keys ValKeys) {
+	chainID := "bench-certify"
+	vals := keys.ToValidators(20, 10)
+	cert := NewStaticCertifier(chainID, vals)
+	check := keys.GenCommit(chainID, 123, nil, vals, []byte("foo"), []byte("params"), []byte("res"), 0, len(keys))
+	for i := 0; i < b.N; i++ {
+		err := cert.Certify(check)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+}
+
+type algo bool
+
+const (
+	linearSearch = true
+	binarySearch = false
+)
+
+// Lazy load the commits
+var fcs5, fcs50, fcs100, fcs500, fcs1000 []FullCommit
+var h5, h50, h100, h500, h1000 []int64
+var commitsOnce sync.Once
+
+func lazyGenerateFullCommits(b *testing.B) {
+	b.Logf("Generating FullCommits")
+	commitsOnce.Do(func() {
+		fcs5, h5 = genFullCommits(nil, nil, 5)
+		b.Logf("Generated 5 FullCommits")
+		fcs50, h50 = genFullCommits(fcs5, h5, 50)
+		b.Logf("Generated 50 FullCommits")
+		fcs100, h100 = genFullCommits(fcs50, h50, 100)
+		b.Logf("Generated 100 FullCommits")
+		fcs500, h500 = genFullCommits(fcs100, h100, 500)
+		b.Logf("Generated 500 FullCommits")
+		fcs1000, h1000 = genFullCommits(fcs500, h500, 1000)
+		b.Logf("Generated 1000 FullCommits")
+	})
+}
+
+func BenchmarkMemStoreProviderGetByHeightLinearSearch5(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs5, h5, linearSearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightLinearSearch50(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs50, h50, linearSearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightLinearSearch100(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs100, h100, linearSearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightLinearSearch500(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs500, h500, linearSearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightLinearSearch1000(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs1000, h1000, linearSearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightBinarySearch5(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs5, h5, binarySearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightBinarySearch50(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs50, h50, binarySearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightBinarySearch100(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs100, h100, binarySearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightBinarySearch500(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs500, h500, binarySearch)
+}
+
+func BenchmarkMemStoreProviderGetByHeightBinarySearch1000(b *testing.B) {
+	benchmarkMemStoreProvidergetByHeight(b, fcs1000, h1000, binarySearch)
+}
+
+var rng = rand.New(rand.NewSource(10))
+
+func benchmarkMemStoreProvidergetByHeight(b *testing.B, fcs []FullCommit, fHeights []int64, algo algo) {
+	lazyGenerateFullCommits(b)
+
+	b.StopTimer()
+	mp := NewMemStoreProvider()
+	for i, fc := range fcs {
+		if err := mp.StoreCommit(fc); err != nil {
+			b.Fatalf("FullCommit #%d: err: %v", i, err)
+		}
+	}
+	qHeights := make([]int64, len(fHeights))
+	copy(qHeights, fHeights)
+	// Append some non-existent heights to trigger the worst cases.
+	qHeights = append(qHeights, 19, -100, -10000, 1e7, -17, 31, -1e9)
+
+	memP := mp.(*memStoreProvider)
+	searchFn := memP.getByHeightLinearSearch
+	if algo == binarySearch { // nolint
+		searchFn = memP.getByHeightBinarySearch
+	}
+
+	hPerm := rng.Perm(len(qHeights))
+	b.StartTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for _, j := range hPerm {
+			h := qHeights[j]
+			if _, err := searchFn(h); err != nil {
+			}
+		}
+	}
+	b.ReportAllocs()
+}
+
+func genFullCommits(prevFC []FullCommit, prevH []int64, want int) ([]FullCommit, []int64) {
+	fcs := make([]FullCommit, len(prevFC))
+	copy(fcs, prevFC)
+	heights := make([]int64, len(prevH))
+	copy(heights, prevH)
+
+	appHash := []byte("benchmarks")
+	chainID := "benchmarks-gen-full-commits"
+	n := want
+	keys := GenValKeys(2 + (n / 3))
+	for i := 0; i < n; i++ {
+		vals := keys.ToValidators(10, int64(n/2))
+		h := int64(20 + 10*i)
+		fcs = append(fcs, keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5))
+		heights = append(heights, h)
+	}
+	return fcs, heights
+}
+
+func TestMemStoreProviderLatestCommitAlwaysUsesSorted(t *testing.T) {
+	p := NewMemStoreProvider().(*memStoreProvider)
+	// 1. With no commits yet stored, it should return ErrCommitNotFound
+	got, err := p.LatestCommit()
+	require.Equal(t, err.Error(), liteErr.ErrCommitNotFound().Error(), "should return ErrCommitNotFound()")
+	require.Equal(t, got, blankFullCommit, "With no fullcommits, it should return a blank FullCommit")
+
+	// 2. Generate some full commits now and we'll add them unsorted.
+	genAndStoreCommitsOfHeight(t, p, 27, 100, 1, 12, 1000, 17, 91)
+	fc, err := p.LatestCommit()
+	require.Nil(t, err, "with commits saved no error expected")
+	require.NotEqual(t, fc, blankFullCommit, "with commits saved no blank FullCommit")
+	require.Equal(t, fc.Height(), int64(1000), "the latest commit i.e. the largest expected")
+}
+
+func genAndStoreCommitsOfHeight(t *testing.T, p Provider, heights ...int64) {
+	n := len(heights)
+	appHash := []byte("tests")
+	chainID := "tests-gen-full-commits"
+	keys := GenValKeys(2 + (n / 3))
+	for i := 0; i < n; i++ {
+		h := heights[i]
+		vals := keys.ToValidators(10, int64(n/2))
+		fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5)
+		err := p.StoreCommit(fc)
+		require.NoError(t, err, "StoreCommit height=%d", h)
+	}
+}

lite/provider.go

@@ -0,0 +1,103 @@
+package lite
+
+// Provider is used to get more validators by other means.
+//
+// Examples: MemProvider, files.Provider, client.Provider, CacheProvider....
+type Provider interface {
+	// StoreCommit saves a FullCommit after we have verified it,
+	// so we can query for it later. Important for updating our
+	// store of trusted commits.
+	StoreCommit(fc FullCommit) error
+	// GetByHeight returns the closest commit with height <= h.
+	GetByHeight(h int64) (FullCommit, error)
+	// GetByHash returns a commit exactly matching this validator hash.
+	GetByHash(hash []byte) (FullCommit, error)
+	// LatestCommit returns the newest commit stored.
+	LatestCommit() (FullCommit, error)
+}
+
+// cacheProvider allows you to place one or more caches in front of a source
+// Provider.  It runs through them in order until a match is found.
+// So you can keep a local cache, and check with the network if
+// no data is there.
+type cacheProvider struct {
+	Providers []Provider
+}
+
+// NewCacheProvider returns a new provider which wraps multiple other providers.
+func NewCacheProvider(providers ...Provider) Provider {
+	return cacheProvider{
+		Providers: providers,
+	}
+}
+
+// StoreCommit tries to add the seed to all providers.
+//
+// Aborts on first error it encounters (closest provider)
+func (c cacheProvider) StoreCommit(fc FullCommit) (err error) {
+	for _, p := range c.Providers {
+		err = p.StoreCommit(fc)
+		if err != nil {
+			break
+		}
+	}
+	return err
+}
+
+// GetByHeight should return the closest possible match from all providers.
+//
+// The Cache is usually organized in order from cheapest call (memory)
+// to most expensive calls (disk/network). However, since GetByHeight returns
+// a FullCommit at h' <= h, if the memory has a seed at h-10, but the network would
+// give us the exact match, a naive "stop at first non-error" would hide
+// the actual desired results.
+//
+// Thus, we query each provider in order until we find an exact match
+// or we finished querying them all.  If at least one returned a non-error,
+// then this returns the best match (minimum h-h').
+func (c cacheProvider) GetByHeight(h int64) (fc FullCommit, err error) {
+	for _, p := range c.Providers {
+		var tfc FullCommit
+		tfc, err = p.GetByHeight(h)
+		if err == nil {
+			if tfc.Height() > fc.Height() {
+				fc = tfc
+			}
+			if tfc.Height() == h {
+				break
+			}
+		}
+	}
+	// even if the last one had an error, if any was a match, this is good
+	if fc.Height() > 0 {
+		err = nil
+	}
+	return fc, err
+}
+
+// GetByHash returns the FullCommit for the hash or an error if the commit is not found.
+func (c cacheProvider) GetByHash(hash []byte) (fc FullCommit, err error) {
+	for _, p := range c.Providers {
+		fc, err = p.GetByHash(hash)
+		if err == nil {
+			break
+		}
+	}
+	return fc, err
+}
+
+// LatestCommit returns the latest FullCommit or an error if no commit exists.
+func (c cacheProvider) LatestCommit() (fc FullCommit, err error) {
+	for _, p := range c.Providers {
+		var tfc FullCommit
+		tfc, err = p.LatestCommit()
+		if err == nil && tfc.Height() > fc.Height() {
+			fc = tfc
+		}
+	}
+	// even if the last one had an error, if any was a match, this is good
+	if fc.Height() > 0 {
+		err = nil
+	}
+	return fc, err
+}

lite/provider_test.go

@@ -0,0 +1,149 @@
+// nolint: vetshadow
+package lite_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+// missingProvider doesn't store anything, always a miss
+// Designed as a mock for testing
+type missingProvider struct{}
+
+// NewMissingProvider returns a provider which does not store anything and always misses.
+func NewMissingProvider() lite.Provider {
+	return missingProvider{}
+}
+
+func (missingProvider) StoreCommit(lite.FullCommit) error { return nil }
+func (missingProvider) GetByHeight(int64) (lite.FullCommit, error) {
+	return lite.FullCommit{}, liteErr.ErrCommitNotFound()
+}
+func (missingProvider) GetByHash([]byte) (lite.FullCommit, error) {
+	return lite.FullCommit{}, liteErr.ErrCommitNotFound()
+}
+func (missingProvider) LatestCommit() (lite.FullCommit, error) {
+	return lite.FullCommit{}, liteErr.ErrCommitNotFound()
+}
+
+func TestMemProvider(t *testing.T) {
+	p := lite.NewMemStoreProvider()
+	checkProvider(t, p, "test-mem", "empty")
+}
+
+func TestCacheProvider(t *testing.T) {
+	p := lite.NewCacheProvider(
+		NewMissingProvider(),
+		lite.NewMemStoreProvider(),
+		NewMissingProvider(),
+	)
+	checkProvider(t, p, "test-cache", "kjfhekfhkewhgit")
+}
+
+func checkProvider(t *testing.T, p lite.Provider, chainID, app string) {
+	assert, require := assert.New(t), require.New(t)
+	appHash := []byte(app)
+	keys := lite.GenValKeys(5)
+	count := 10
+
+	// make a bunch of commits...
+	commits := make([]lite.FullCommit, count)
+	for i := 0; i < count; i++ {
+		// two commits for each validator, to check how we handle dups
+		// (10, 0), (10, 1), (10, 1), (10, 2), (10, 2), ...
+		vals := keys.ToValidators(10, int64(count/2))
+		h := int64(20 + 10*i)
+		commits[i] = keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5)
+	}
+
+	// check provider is empty
+	fc, err := p.GetByHeight(20)
+	require.NotNil(err)
+	assert.True(liteErr.IsCommitNotFoundErr(err))
+
+	fc, err = p.GetByHash(commits[3].ValidatorsHash())
+	require.NotNil(err)
+	assert.True(liteErr.IsCommitNotFoundErr(err))
+
+	// now add them all to the provider
+	for _, s := range commits {
+		err = p.StoreCommit(s)
+		require.Nil(err)
+		// and make sure we can get it back
+		s2, err := p.GetByHash(s.ValidatorsHash())
+		assert.Nil(err)
+		assert.Equal(s, s2)
+		// by height as well
+		s2, err = p.GetByHeight(s.Height())
+		assert.Nil(err)
+		assert.Equal(s, s2)
+	}
+
+	// make sure we get the last hash if we overstep
+	fc, err = p.GetByHeight(5000)
+	if assert.Nil(err) {
+		assert.Equal(commits[count-1].Height(), fc.Height())
+		assert.Equal(commits[count-1], fc)
+	}
+
+	// and middle ones as well
+	fc, err = p.GetByHeight(47)
+	if assert.Nil(err) {
+		// we only step by 10, so 40 must be the one below this
+		assert.EqualValues(40, fc.Height())
+	}
+
+}
+
+// this will make a get height, and if it is good, set the data as well
+func checkGetHeight(t *testing.T, p lite.Provider, ask, expect int64) {
+	fc, err := p.GetByHeight(ask)
+	require.Nil(t, err, "GetByHeight")
+	if assert.Equal(t, expect, fc.Height()) {
+		err = p.StoreCommit(fc)
+		require.Nil(t, err, "StoreCommit")
+	}
+}
+
+func TestCacheGetsBestHeight(t *testing.T) {
+	// assert, require := assert.New(t), require.New(t)
+	require := require.New(t)
+
+	// we will write data to the second level of the cache (p2),
+	// and see what gets cached, stored in
+	p := lite.NewMemStoreProvider()
+	p2 := lite.NewMemStoreProvider()
+	cp := lite.NewCacheProvider(p, p2)
+
+	chainID := "cache-best-height"
+	appHash := []byte("01234567")
+	keys := lite.GenValKeys(5)
+	count := 10
+
+	// set a bunch of commits
+	for i := 0; i < count; i++ {
+		vals := keys.ToValidators(10, int64(count/2))
+		h := int64(10 * (i + 1))
+		fc := keys.GenFullCommit(chainID, h, nil, vals, appHash, []byte("params"), []byte("results"), 0, 5)
+		err := p2.StoreCommit(fc)
+		require.NoError(err)
+	}
+
+	// let's get a few heights from the cache and set them proper
+	checkGetHeight(t, cp, 57, 50)
+	checkGetHeight(t, cp, 33, 30)
+
+	// make sure they are set in p as well (but nothing else)
+	checkGetHeight(t, p, 44, 30)
+	checkGetHeight(t, p, 50, 50)
+	checkGetHeight(t, p, 99, 50)
+
+	// now, query the cache for a higher value
+	checkGetHeight(t, p2, 99, 90)
+	checkGetHeight(t, cp, 99, 90)
+}

lite/proxy/block.go

@@ -0,0 +1,49 @@
+package proxy
+
+import (
+	"bytes"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tendermint/lite"
+	certerr "github.com/tendermint/tendermint/lite/errors"
+	"github.com/tendermint/tendermint/types"
+)
+
+func ValidateBlockMeta(meta *types.BlockMeta, check lite.Commit) error {
+	if meta == nil {
+		return errors.New("expecting a non-nil BlockMeta")
+	}
+	// TODO: check the BlockID??
+	return ValidateHeader(meta.Header, check)
+}
+
+func ValidateBlock(meta *types.Block, check lite.Commit) error {
+	if meta == nil {
+		return errors.New("expecting a non-nil Block")
+	}
+	err := ValidateHeader(meta.Header, check)
+	if err != nil {
+		return err
+	}
+	if !bytes.Equal(meta.Data.Hash(), meta.Header.DataHash) {
+		return errors.New("Data hash doesn't match header")
+	}
+	return nil
+}
+
+func ValidateHeader(head *types.Header, check lite.Commit) error {
+	if head == nil {
+		return errors.New("expecting a non-nil Header")
+	}
+	// make sure they are for the same height (obvious fail)
+	if head.Height != check.Height() {
+		return certerr.ErrHeightMismatch(head.Height, check.Height())
+	}
+	// check if they are equal by using hashes
+	chead := check.Header
+	if !bytes.Equal(head.Hash(), chead.Hash()) {
+		return errors.New("Headers don't match")
+	}
+	return nil
+}

lite/proxy/certifier.go

@@ -0,0 +1,35 @@
+package proxy
+
+import (
+	"github.com/tendermint/tendermint/lite"
+	certclient "github.com/tendermint/tendermint/lite/client"
+	"github.com/tendermint/tendermint/lite/files"
+)
+
+func GetCertifier(chainID, rootDir, nodeAddr string) (*lite.InquiringCertifier, error) {
+	trust := lite.NewCacheProvider(
+		lite.NewMemStoreProvider(),
+		files.NewProvider(rootDir),
+	)
+
+	source := certclient.NewHTTPProvider(nodeAddr)
+
+	// XXX: total insecure hack to avoid `init`
+	fc, err := source.LatestCommit()
+	/* XXX
+	// this gets the most recent verified commit
+	fc, err := trust.LatestCommit()
+	if certerr.IsCommitNotFoundErr(err) {
+		return nil, errors.New("Please run init first to establish a root of trust")
+	}*/
+	if err != nil {
+		return nil, err
+	}
+
+	cert, err := lite.NewInquiringCertifier(chainID, fc, trust, source)
+	if err != nil {
+		return nil, err
+	}
+
+	return cert, nil
+}

lite/proxy/errors.go

@@ -0,0 +1,22 @@
+package proxy
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+//--------------------------------------------
+
+var errNoData = fmt.Errorf("No data returned for query")
+
+// IsNoDataErr checks whether an error is due to a query returning empty data
+func IsNoDataErr(err error) bool {
+	return errors.Cause(err) == errNoData
+}
+
+func ErrNoData() error {
+	return errors.WithStack(errNoData)
+}
+
+//--------------------------------------------

lite/proxy/errors_test.go

@@ -0,0 +1,17 @@
+package proxy
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestErrorNoData(t *testing.T) {
+	e1 := ErrNoData()
+	assert.True(t, IsNoDataErr(e1))
+
+	e2 := errors.New("foobar")
+	assert.False(t, IsNoDataErr(e2))
+	assert.False(t, IsNoDataErr(nil))
+}

lite/proxy/proxy.go

@@ -0,0 +1,77 @@
+package proxy
+
+import (
+	"net/http"
+
+	"github.com/tendermint/go-amino"
+	"github.com/tendermint/tmlibs/log"
+
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/rpc/core"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpc "github.com/tendermint/tendermint/rpc/lib/server"
+)
+
+const (
+	wsEndpoint = "/websocket"
+)
+
+// StartProxy will start the websocket manager on the client,
+// set up the rpc routes to proxy via the given client,
+// and start up an http/rpc server on the location given by bind (eg. :1234)
+func StartProxy(c rpcclient.Client, listenAddr string, logger log.Logger) error {
+	err := c.Start()
+	if err != nil {
+		return err
+	}
+
+	cdc := amino.NewCodec()
+	ctypes.RegisterAmino(cdc)
+	r := RPCRoutes(c)
+
+	// build the handler...
+	mux := http.NewServeMux()
+	rpc.RegisterRPCFuncs(mux, r, cdc, logger)
+
+	wm := rpc.NewWebsocketManager(r, cdc, rpc.EventSubscriber(c))
+	wm.SetLogger(logger)
+	core.SetLogger(logger)
+	mux.HandleFunc(wsEndpoint, wm.WebsocketHandler)
+
+	_, err = rpc.StartHTTPServer(listenAddr, mux, logger)
+
+	return err
+}
+
+// RPCRoutes just routes everything to the given client, as if it were
+// a tendermint fullnode.
+//
+// if we want security, the client must implement it as a secure client
+func RPCRoutes(c rpcclient.Client) map[string]*rpc.RPCFunc {
+
+	return map[string]*rpc.RPCFunc{
+		// Subscribe/unsubscribe are reserved for websocket events.
+		// We can just use the core tendermint impl, which uses the
+		// EventSwitch we registered in NewWebsocketManager above
+		"subscribe":   rpc.NewWSRPCFunc(core.Subscribe, "query"),
+		"unsubscribe": rpc.NewWSRPCFunc(core.Unsubscribe, "query"),
+
+		// info API
+		"status":     rpc.NewRPCFunc(c.Status, ""),
+		"blockchain": rpc.NewRPCFunc(c.BlockchainInfo, "minHeight,maxHeight"),
+		"genesis":    rpc.NewRPCFunc(c.Genesis, ""),
+		"block":      rpc.NewRPCFunc(c.Block, "height"),
+		"commit":     rpc.NewRPCFunc(c.Commit, "height"),
+		"tx":         rpc.NewRPCFunc(c.Tx, "hash,prove"),
+		"validators": rpc.NewRPCFunc(c.Validators, ""),
+
+		// broadcast API
+		"broadcast_tx_commit": rpc.NewRPCFunc(c.BroadcastTxCommit, "tx"),
+		"broadcast_tx_sync":   rpc.NewRPCFunc(c.BroadcastTxSync, "tx"),
+		"broadcast_tx_async":  rpc.NewRPCFunc(c.BroadcastTxAsync, "tx"),
+
+		// abci API
+		"abci_query": rpc.NewRPCFunc(c.ABCIQuery, "path,data,prove"),
+		"abci_info":  rpc.NewRPCFunc(c.ABCIInfo, ""),
+	}
+}

lite/proxy/query.go

@@ -0,0 +1,152 @@
+package proxy
+
+import (
+	"github.com/pkg/errors"
+
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/lite"
+	"github.com/tendermint/tendermint/lite/client"
+	certerr "github.com/tendermint/tendermint/lite/errors"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+// KeyProof represents a proof of existence or absence of a single key.
+// Copied from iavl repo. TODO
+type KeyProof interface {
+	// Verify verfies the proof is valid. To verify absence,
+	// the value should be nil.
+	Verify(key, value, root []byte) error
+
+	// Root returns the root hash of the proof.
+	Root() []byte
+
+	// Serialize itself
+	Bytes() []byte
+}
+
+// GetWithProof will query the key on the given node, and verify it has
+// a valid proof, as defined by the certifier.
+//
+// If there is any error in checking, returns an error.
+// If val is non-empty, proof should be KeyExistsProof
+// If val is empty, proof should be KeyMissingProof
+func GetWithProof(key []byte, reqHeight int64, node rpcclient.Client,
+	cert lite.Certifier) (
+	val cmn.HexBytes, height int64, proof KeyProof, err error) {
+
+	if reqHeight < 0 {
+		err = errors.Errorf("Height cannot be negative")
+		return
+	}
+
+	_resp, proof, err := GetWithProofOptions("/key", key,
+		rpcclient.ABCIQueryOptions{Height: int64(reqHeight)},
+		node, cert)
+	if _resp != nil {
+		resp := _resp.Response
+		val, height = resp.Value, resp.Height
+	}
+	return val, height, proof, err
+}
+
+// GetWithProofOptions is useful if you want full access to the ABCIQueryOptions
+func GetWithProofOptions(path string, key []byte, opts rpcclient.ABCIQueryOptions,
+	node rpcclient.Client, cert lite.Certifier) (
+	*ctypes.ResultABCIQuery, KeyProof, error) {
+
+	_resp, err := node.ABCIQueryWithOptions(path, key, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	resp := _resp.Response
+
+	// make sure the proof is the proper height
+	if resp.IsErr() {
+		err = errors.Errorf("Query error for key %d: %d", key, resp.Code)
+		return nil, nil, err
+	}
+	if len(resp.Key) == 0 || len(resp.Proof) == 0 {
+		return nil, nil, ErrNoData()
+	}
+	if resp.Height == 0 {
+		return nil, nil, errors.New("Height returned is zero")
+	}
+
+	// AppHash for height H is in header H+1
+	commit, err := GetCertifiedCommit(resp.Height+1, node, cert)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	_ = commit
+	return &ctypes.ResultABCIQuery{Response: resp}, nil, nil
+
+	/* // TODO refactor so iavl stuff is not in tendermint core
+	   // https://github.com/tendermint/tendermint/issues/1183
+	if len(resp.Value) > 0 {
+		// The key was found, construct a proof of existence.
+		proof, err := iavl.ReadKeyProof(resp.Proof)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "Error reading proof")
+		}
+
+		eproof, ok := proof.(*iavl.KeyExistsProof)
+		if !ok {
+			return nil, nil, errors.New("Expected KeyExistsProof for non-empty value")
+		}
+
+		// Validate the proof against the certified header to ensure data integrity.
+		err = eproof.Verify(resp.Key, resp.Value, commit.Header.AppHash)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "Couldn't verify proof")
+		}
+		return &ctypes.ResultABCIQuery{Response: resp}, eproof, nil
+	}
+
+	// The key wasn't found, construct a proof of non-existence.
+	proof, err := iavl.ReadKeyProof(resp.Proof)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Error reading proof")
+	}
+
+	aproof, ok := proof.(*iavl.KeyAbsentProof)
+	if !ok {
+		return nil, nil, errors.New("Expected KeyAbsentProof for empty Value")
+	}
+
+	// Validate the proof against the certified header to ensure data integrity.
+	err = aproof.Verify(resp.Key, nil, commit.Header.AppHash)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "Couldn't verify proof")
+	}
+	return &ctypes.ResultABCIQuery{Response: resp}, aproof, ErrNoData()
+	*/
+}
+
+// GetCertifiedCommit gets the signed header for a given height
+// and certifies it.  Returns error if unable to get a proven header.
+func GetCertifiedCommit(h int64, node rpcclient.Client, cert lite.Certifier) (lite.Commit, error) {
+
+	// FIXME: cannot use cert.GetByHeight for now, as it also requires
+	// Validators and will fail on querying tendermint for non-current height.
+	// When this is supported, we should use it instead...
+	rpcclient.WaitForHeight(node, h, nil)
+	cresp, err := node.Commit(&h)
+	if err != nil {
+		return lite.Commit{}, err
+	}
+
+	commit := client.CommitFromResult(cresp)
+	// validate downloaded checkpoint with our request and trust store.
+	if commit.Height() != h {
+		return lite.Commit{}, certerr.ErrHeightMismatch(h, commit.Height())
+	}
+
+	if err = cert.Certify(commit); err != nil {
+		return lite.Commit{}, err
+	}
+
+	return commit, nil
+}

lite/proxy/query_test.go

@@ -0,0 +1,139 @@
+package proxy
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/abci/example/kvstore"
+
+	"github.com/tendermint/tendermint/lite"
+	certclient "github.com/tendermint/tendermint/lite/client"
+	nm "github.com/tendermint/tendermint/node"
+	"github.com/tendermint/tendermint/rpc/client"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+	"github.com/tendermint/tendermint/types"
+)
+
+var node *nm.Node
+
+// TODO fix tests!!
+
+func TestMain(m *testing.M) {
+	app := kvstore.NewKVStoreApplication()
+
+	node = rpctest.StartTendermint(app)
+
+	code := m.Run()
+
+	node.Stop()
+	node.Wait()
+	os.Exit(code)
+}
+
+func kvstoreTx(k, v []byte) []byte {
+	return []byte(fmt.Sprintf("%s=%s", k, v))
+}
+
+func _TestAppProofs(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	cl := client.NewLocal(node)
+	client.WaitForHeight(cl, 1, nil)
+
+	k := []byte("my-key")
+	v := []byte("my-value")
+
+	tx := kvstoreTx(k, v)
+	br, err := cl.BroadcastTxCommit(tx)
+	require.NoError(err, "%+v", err)
+	require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx)
+	require.EqualValues(0, br.DeliverTx.Code)
+	brh := br.Height
+
+	// This sets up our trust on the node based on some past point.
+	source := certclient.NewProvider(cl)
+	seed, err := source.GetByHeight(brh - 2)
+	require.NoError(err, "%+v", err)
+	cert := lite.NewStaticCertifier("my-chain", seed.Validators)
+
+	client.WaitForHeight(cl, 3, nil)
+	latest, err := source.LatestCommit()
+	require.NoError(err, "%+v", err)
+	rootHash := latest.Header.AppHash
+
+	// verify a query before the tx block has no data (and valid non-exist proof)
+	bs, height, proof, err := GetWithProof(k, brh-1, cl, cert)
+	fmt.Println(bs, height, proof, err)
+	require.NotNil(err)
+	require.True(IsNoDataErr(err), err.Error())
+	require.Nil(bs)
+
+	// but given that block it is good
+	bs, height, proof, err = GetWithProof(k, brh, cl, cert)
+	require.NoError(err, "%+v", err)
+	require.NotNil(proof)
+	require.True(height >= int64(latest.Header.Height))
+
+	// Alexis there is a bug here, somehow the above code gives us rootHash = nil
+	// and proof.Verify doesn't care, while proofNotExists.Verify fails.
+	// I am hacking this in to make it pass, but please investigate further.
+	rootHash = proof.Root()
+
+	//err = wire.ReadBinaryBytes(bs, &data)
+	//require.NoError(err, "%+v", err)
+	assert.EqualValues(v, bs)
+	err = proof.Verify(k, bs, rootHash)
+	assert.NoError(err, "%+v", err)
+
+	// Test non-existing key.
+	missing := []byte("my-missing-key")
+	bs, _, proof, err = GetWithProof(missing, 0, cl, cert)
+	require.True(IsNoDataErr(err))
+	require.Nil(bs)
+	require.NotNil(proof)
+	err = proof.Verify(missing, nil, rootHash)
+	assert.NoError(err, "%+v", err)
+	err = proof.Verify(k, nil, rootHash)
+	assert.Error(err)
+}
+
+func _TestTxProofs(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	cl := client.NewLocal(node)
+	client.WaitForHeight(cl, 1, nil)
+
+	tx := kvstoreTx([]byte("key-a"), []byte("value-a"))
+	br, err := cl.BroadcastTxCommit(tx)
+	require.NoError(err, "%+v", err)
+	require.EqualValues(0, br.CheckTx.Code, "%#v", br.CheckTx)
+	require.EqualValues(0, br.DeliverTx.Code)
+	brh := br.Height
+
+	source := certclient.NewProvider(cl)
+	seed, err := source.GetByHeight(brh - 2)
+	require.NoError(err, "%+v", err)
+	cert := lite.NewStaticCertifier("my-chain", seed.Validators)
+
+	// First let's make sure a bogus transaction hash returns a valid non-existence proof.
+	key := types.Tx([]byte("bogus")).Hash()
+	res, err := cl.Tx(key, true)
+	require.NotNil(err)
+	require.Contains(err.Error(), "not found")
+
+	// Now let's check with the real tx hash.
+	key = types.Tx(tx).Hash()
+	res, err = cl.Tx(key, true)
+	require.NoError(err, "%+v", err)
+	require.NotNil(res)
+	err = res.Proof.Validate(key)
+	assert.NoError(err, "%+v", err)
+
+	commit, err := GetCertifiedCommit(br.Height, cl, cert)
+	require.Nil(err, "%+v", err)
+	require.Equal(res.Proof.RootHash, commit.Header.DataHash)
+}

lite/proxy/validate_test.go

@@ -0,0 +1,218 @@
+package proxy_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/lite"
+	"github.com/tendermint/tendermint/lite/proxy"
+	"github.com/tendermint/tendermint/types"
+)
+
+var (
+	deadBeefTxs  = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")}
+	deadBeefHash = deadBeefTxs.Hash()
+	testTime1    = time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC)
+	testTime2    = time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC)
+)
+
+var hdrHeight11 = &types.Header{
+	Height:         11,
+	Time:           testTime1,
+	ValidatorsHash: []byte("Tendermint"),
+}
+
+func TestValidateBlock(t *testing.T) {
+	tests := []struct {
+		block   *types.Block
+		commit  lite.Commit
+		wantErr string
+	}{
+		{
+			block: nil, wantErr: "non-nil Block",
+		},
+		{
+			block: &types.Block{}, wantErr: "nil Header",
+		},
+		{
+			block: &types.Block{Header: new(types.Header)},
+		},
+
+		// Start Header.Height mismatch test
+		{
+			block:   &types.Block{Header: &types.Header{Height: 10}},
+			commit:  lite.Commit{Header: &types.Header{Height: 11}},
+			wantErr: "don't match - 10 vs 11",
+		},
+
+		{
+			block:  &types.Block{Header: &types.Header{Height: 11}},
+			commit: lite.Commit{Header: &types.Header{Height: 11}},
+		},
+		// End Header.Height mismatch test
+
+		// Start Header.Hash mismatch test
+		{
+			block:   &types.Block{Header: hdrHeight11},
+			commit:  lite.Commit{Header: &types.Header{Height: 11}},
+			wantErr: "Headers don't match",
+		},
+
+		{
+			block:  &types.Block{Header: hdrHeight11},
+			commit: lite.Commit{Header: hdrHeight11},
+		},
+		// End Header.Hash mismatch test
+
+		// Start Header.Data hash mismatch test
+		{
+			block: &types.Block{
+				Header: &types.Header{Height: 11},
+				Data:   &types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}},
+			},
+			commit: lite.Commit{
+				Header: &types.Header{Height: 11},
+				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("0xDEADBEEF")}},
+			},
+			wantErr: "Data hash doesn't match header",
+		},
+		{
+			block: &types.Block{
+				Header: &types.Header{Height: 11, DataHash: deadBeefHash},
+				Data:   &types.Data{Txs: deadBeefTxs},
+			},
+			commit: lite.Commit{
+				Header: &types.Header{Height: 11},
+				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
+			},
+		},
+		// End Header.Data hash mismatch test
+	}
+
+	for i, tt := range tests {
+		err := proxy.ValidateBlock(tt.block, tt.commit)
+		if tt.wantErr != "" {
+			if err == nil {
+				assert.FailNowf(t, "Unexpectedly passed", "#%d", i)
+			} else {
+				assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i)
+			}
+			continue
+		}
+
+		assert.Nil(t, err, "#%d: expecting a nil error", i)
+	}
+}
+
+func TestValidateBlockMeta(t *testing.T) {
+	tests := []struct {
+		meta    *types.BlockMeta
+		commit  lite.Commit
+		wantErr string
+	}{
+		{
+			meta: nil, wantErr: "non-nil BlockMeta",
+		},
+		{
+			meta: &types.BlockMeta{}, wantErr: "non-nil Header",
+		},
+		{
+			meta: &types.BlockMeta{Header: new(types.Header)},
+		},
+
+		// Start Header.Height mismatch test
+		{
+			meta:    &types.BlockMeta{Header: &types.Header{Height: 10}},
+			commit:  lite.Commit{Header: &types.Header{Height: 11}},
+			wantErr: "don't match - 10 vs 11",
+		},
+
+		{
+			meta:   &types.BlockMeta{Header: &types.Header{Height: 11}},
+			commit: lite.Commit{Header: &types.Header{Height: 11}},
+		},
+		// End Header.Height mismatch test
+
+		// Start Headers don't match test
+		{
+			meta:    &types.BlockMeta{Header: hdrHeight11},
+			commit:  lite.Commit{Header: &types.Header{Height: 11}},
+			wantErr: "Headers don't match",
+		},
+
+		{
+			meta:   &types.BlockMeta{Header: hdrHeight11},
+			commit: lite.Commit{Header: hdrHeight11},
+		},
+
+		{
+			meta: &types.BlockMeta{
+				Header: &types.Header{
+					Height:         11,
+					ValidatorsHash: []byte("lite-test"),
+					// TODO: should be able to use empty time after Amino upgrade
+					Time: testTime1,
+				},
+			},
+			commit: lite.Commit{
+				Header: &types.Header{Height: 11, DataHash: deadBeefHash},
+			},
+			wantErr: "Headers don't match",
+		},
+
+		{
+			meta: &types.BlockMeta{
+				Header: &types.Header{
+					Height: 11, DataHash: deadBeefHash,
+					ValidatorsHash: []byte("Tendermint"),
+					Time:           testTime1,
+				},
+			},
+			commit: lite.Commit{
+				Header: &types.Header{
+					Height: 11, DataHash: deadBeefHash,
+					ValidatorsHash: []byte("Tendermint"),
+					Time:           testTime2,
+				},
+				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
+			},
+			wantErr: "Headers don't match",
+		},
+
+		{
+			meta: &types.BlockMeta{
+				Header: &types.Header{
+					Height: 11, DataHash: deadBeefHash,
+					ValidatorsHash: []byte("Tendermint"),
+					Time:           testTime2,
+				},
+			},
+			commit: lite.Commit{
+				Header: &types.Header{
+					Height: 11, DataHash: deadBeefHash,
+					ValidatorsHash: []byte("Tendermint-x"),
+					Time:           testTime2,
+				},
+				Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}},
+			},
+			wantErr: "Headers don't match",
+		},
+		// End Headers don't match test
+	}
+
+	for i, tt := range tests {
+		err := proxy.ValidateBlockMeta(tt.meta, tt.commit)
+		if tt.wantErr != "" {
+			if err == nil {
+				assert.FailNowf(t, "Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr)
+			} else {
+				assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i)
+			}
+			continue
+		}
+
+		assert.Nil(t, err, "#%d: expecting a nil error", i)
+	}
+}

lite/proxy/wrapper.go

@@ -0,0 +1,187 @@
+package proxy
+
+import (
+	cmn "github.com/tendermint/tmlibs/common"
+
+	"github.com/tendermint/tendermint/lite"
+	certclient "github.com/tendermint/tendermint/lite/client"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+var _ rpcclient.Client = Wrapper{}
+
+// Wrapper wraps a rpcclient with a Certifier and double-checks any input that is
+// provable before passing it along. Allows you to make any rpcclient fully secure.
+type Wrapper struct {
+	rpcclient.Client
+	cert *lite.InquiringCertifier
+}
+
+// SecureClient uses a given certifier to wrap an connection to an untrusted
+// host and return a cryptographically secure rpc client.
+//
+// If it is wrapping an HTTP rpcclient, it will also wrap the websocket interface
+func SecureClient(c rpcclient.Client, cert *lite.InquiringCertifier) Wrapper {
+	wrap := Wrapper{c, cert}
+	// TODO: no longer possible as no more such interface exposed....
+	// if we wrap http client, then we can swap out the event switch to filter
+	// if hc, ok := c.(*rpcclient.HTTP); ok {
+	// 	evt := hc.WSEvents.EventSwitch
+	// 	hc.WSEvents.EventSwitch = WrappedSwitch{evt, wrap}
+	// }
+	return wrap
+}
+
+// ABCIQueryWithOptions exposes all options for the ABCI query and verifies the returned proof
+func (w Wrapper) ABCIQueryWithOptions(path string, data cmn.HexBytes,
+	opts rpcclient.ABCIQueryOptions) (*ctypes.ResultABCIQuery, error) {
+
+	res, _, err := GetWithProofOptions(path, data, opts, w.Client, w.cert)
+	return res, err
+}
+
+// ABCIQuery uses default options for the ABCI query and verifies the returned proof
+func (w Wrapper) ABCIQuery(path string, data cmn.HexBytes) (*ctypes.ResultABCIQuery, error) {
+	return w.ABCIQueryWithOptions(path, data, rpcclient.DefaultABCIQueryOptions)
+}
+
+// Tx queries for a given tx and verifies the proof if it was requested
+func (w Wrapper) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {
+	res, err := w.Client.Tx(hash, prove)
+	if !prove || err != nil {
+		return res, err
+	}
+	h := int64(res.Height)
+	check, err := GetCertifiedCommit(h, w.Client, w.cert)
+	if err != nil {
+		return res, err
+	}
+	err = res.Proof.Validate(check.Header.DataHash)
+	return res, err
+}
+
+// BlockchainInfo requests a list of headers and verifies them all...
+// Rather expensive.
+//
+// TODO: optimize this if used for anything needing performance
+func (w Wrapper) BlockchainInfo(minHeight, maxHeight int64) (*ctypes.ResultBlockchainInfo, error) {
+	r, err := w.Client.BlockchainInfo(minHeight, maxHeight)
+	if err != nil {
+		return nil, err
+	}
+
+	// go and verify every blockmeta in the result....
+	for _, meta := range r.BlockMetas {
+		// get a checkpoint to verify from
+		c, err := w.Commit(&meta.Header.Height)
+		if err != nil {
+			return nil, err
+		}
+		check := certclient.CommitFromResult(c)
+		err = ValidateBlockMeta(meta, check)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return r, nil
+}
+
+// Block returns an entire block and verifies all signatures
+func (w Wrapper) Block(height *int64) (*ctypes.ResultBlock, error) {
+	r, err := w.Client.Block(height)
+	if err != nil {
+		return nil, err
+	}
+	// get a checkpoint to verify from
+	c, err := w.Commit(height)
+	if err != nil {
+		return nil, err
+	}
+	check := certclient.CommitFromResult(c)
+
+	// now verify
+	err = ValidateBlockMeta(r.BlockMeta, check)
+	if err != nil {
+		return nil, err
+	}
+	err = ValidateBlock(r.Block, check)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// Commit downloads the Commit and certifies it with the lite.
+//
+// This is the foundation for all other verification in this module
+func (w Wrapper) Commit(height *int64) (*ctypes.ResultCommit, error) {
+	rpcclient.WaitForHeight(w.Client, *height, nil)
+	r, err := w.Client.Commit(height)
+	// if we got it, then certify it
+	if err == nil {
+		check := certclient.CommitFromResult(r)
+		err = w.cert.Certify(check)
+	}
+	return r, err
+}
+
+// // WrappedSwitch creates a websocket connection that auto-verifies any info
+// // coming through before passing it along.
+// //
+// // Since the verification takes 1-2 rpc calls, this is obviously only for
+// // relatively low-throughput situations that can tolerate a bit extra latency
+// type WrappedSwitch struct {
+// 	types.EventSwitch
+// 	client rpcclient.Client
+// }
+
+// // FireEvent verifies any block or header returned from the eventswitch
+// func (s WrappedSwitch) FireEvent(event string, data events.EventData) {
+// 	tm, ok := data.(types.TMEventData)
+// 	if !ok {
+// 		fmt.Printf("bad type %#v\n", data)
+// 		return
+// 	}
+
+// 	// check to validate it if possible, and drop if not valid
+// 	switch t := tm.(type) {
+// 	case types.EventDataNewBlockHeader:
+// 		err := verifyHeader(s.client, t.Header)
+// 		if err != nil {
+// 			fmt.Printf("Invalid header: %#v\n", err)
+// 			return
+// 		}
+// 	case types.EventDataNewBlock:
+// 		err := verifyBlock(s.client, t.Block)
+// 		if err != nil {
+// 			fmt.Printf("Invalid block: %#v\n", err)
+// 			return
+// 		}
+// 		// TODO: can we verify tx as well? anything else
+// 	}
+
+// 	// looks good, we fire it
+// 	s.EventSwitch.FireEvent(event, data)
+// }
+
+// func verifyHeader(c rpcclient.Client, head *types.Header) error {
+// 	// get a checkpoint to verify from
+// 	commit, err := c.Commit(&head.Height)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	check := certclient.CommitFromResult(commit)
+// 	return ValidateHeader(head, check)
+// }
+//
+// func verifyBlock(c rpcclient.Client, block *types.Block) error {
+// 	// get a checkpoint to verify from
+// 	commit, err := c.Commit(&block.Height)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	check := certclient.CommitFromResult(commit)
+// 	return ValidateBlock(block, check)
+// }

lite/static_certifier.go

@@ -0,0 +1,73 @@
+package lite
+
+import (
+	"bytes"
+
+	"github.com/pkg/errors"
+
+	"github.com/tendermint/tendermint/types"
+
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+var _ Certifier = (*StaticCertifier)(nil)
+
+// StaticCertifier assumes a static set of validators, set on
+// initilization and checks against them.
+// The signatures on every header is checked for > 2/3 votes
+// against the known validator set upon Certify
+//
+// Good for testing or really simple chains.  Building block
+// to support real-world functionality.
+type StaticCertifier struct {
+	chainID string
+	vSet    *types.ValidatorSet
+	vhash   []byte
+}
+
+// NewStaticCertifier returns a new certifier with a static validator set.
+func NewStaticCertifier(chainID string, vals *types.ValidatorSet) *StaticCertifier {
+	return &StaticCertifier{
+		chainID: chainID,
+		vSet:    vals,
+	}
+}
+
+// ChainID returns the chain id.
+// Implements Certifier.
+func (sc *StaticCertifier) ChainID() string {
+	return sc.chainID
+}
+
+// Validators returns the validator set.
+func (sc *StaticCertifier) Validators() *types.ValidatorSet {
+	return sc.vSet
+}
+
+// Hash returns the hash of the validator set.
+func (sc *StaticCertifier) Hash() []byte {
+	if len(sc.vhash) == 0 {
+		sc.vhash = sc.vSet.Hash()
+	}
+	return sc.vhash
+}
+
+// Certify makes sure that the commit is valid.
+// Implements Certifier.
+func (sc *StaticCertifier) Certify(commit Commit) error {
+	// do basic sanity checks
+	err := commit.ValidateBasic(sc.chainID)
+	if err != nil {
+		return err
+	}
+
+	// make sure it has the same validator set we have (static means static)
+	if !bytes.Equal(sc.Hash(), commit.Header.ValidatorsHash) {
+		return liteErr.ErrValidatorsChanged()
+	}
+
+	// then make sure we have the proper signatures for this
+	err = sc.vSet.VerifyCommit(sc.chainID, commit.Commit.BlockID,
+		commit.Header.Height, commit.Commit)
+	return errors.WithStack(err)
+}

lite/static_certifier_test.go

@@ -0,0 +1,59 @@
+package lite_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/types"
+
+	"github.com/tendermint/tendermint/lite"
+	liteErr "github.com/tendermint/tendermint/lite/errors"
+)
+
+func TestStaticCert(t *testing.T) {
+	// assert, require := assert.New(t), require.New(t)
+	assert := assert.New(t)
+	// require := require.New(t)
+
+	keys := lite.GenValKeys(4)
+	// 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do!
+	vals := keys.ToValidators(20, 10)
+	// and a certifier based on our known set
+	chainID := "test-static"
+	cert := lite.NewStaticCertifier(chainID, vals)
+
+	cases := []struct {
+		keys        lite.ValKeys
+		vals        *types.ValidatorSet
+		height      int64
+		first, last int  // who actually signs
+		proper      bool // true -> expect no error
+		changed     bool // true -> expect validator change error
+	}{
+		// perfect, signed by everyone
+		{keys, vals, 1, 0, len(keys), true, false},
+		// skip little guy is okay
+		{keys, vals, 2, 1, len(keys), true, false},
+		// but not the big guy
+		{keys, vals, 3, 0, len(keys) - 1, false, false},
+		// even changing the power a little bit breaks the static validator
+		// the sigs are enough, but the validator hash is unknown
+		{keys, keys.ToValidators(20, 11), 4, 0, len(keys), false, true},
+	}
+
+	for _, tc := range cases {
+		check := tc.keys.GenCommit(chainID, tc.height, nil, tc.vals,
+			[]byte("foo"), []byte("params"), []byte("results"), tc.first, tc.last)
+		err := cert.Certify(check)
+		if tc.proper {
+			assert.Nil(err, "%+v", err)
+		} else {
+			assert.NotNil(err)
+			if tc.changed {
+				assert.True(liteErr.IsValidatorsChangedErr(err), "%+v", err)
+			}
+		}
+	}
+
+}