Merge branch 'master' into marko/int64-

CHANGELOG_PENDING.md

@@ -30,6 +30,9 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
   - [libs/os] Kill() and {Must,}{Read,Write}File() functions have been removed. (@alessio)
   - [store] \#5848 Remove block store state in favor of using the db iterators directly (@cmwaters)
   - [state] \#5864 Use an iterator when pruning state (@cmwaters)
+  - [types] \#6023 Remove `tm2pb.Header`, `tm2pb.BlockID`, `tm2pb.PartSetHeader` and `tm2pb.NewValidatorUpdate`.
+    - Each of the above types has a `ToProto` and `FromProto` method or function which replaced this logic.
+  - [rpc/client/http] \#6022 Change `timeout` type to `time.Duration` in `NewWithTimeout`
 
 - Blockchain Protocol
 
@@ -42,7 +45,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 
 - [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778)
 - [privval] \#5603 Add `--key` to `init`, `gen_validator`, `testnet` & `unsafe_reset_priv_validator` for use in generating `secp256k1` keys.
-- [privval] \#5725 Add gRPC support to private validator. 
+- [privval] \#5725 Add gRPC support to private validator.
 - [privval] \#5876 `tendermint show-validator` will query the remote signer if gRPC is being used (@marbar3778)
 - [abci/client] \#5673 `Async` requests return an error if queue is full (@melekes)
 - [mempool] \#5673 Cancel `CheckTx` requests if RPC client disconnects or times out (@melekes)
@@ -62,3 +65,4 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi
 - [privval] \#5638 Increase read/write timeout to 5s and calculate ping interval based on it (@JoeKash)
 - [blockchain/v1] [\#5701](https://github.com/tendermint/tendermint/pull/5701) Handle peers without blocks (@melekes)
 - [blockchain/v1] \#5711 Fix deadlock (@melekes)
+- [light] \#6022 Fix a bug when the number of validators equals 100 (@melekes)

light/provider/http/http.go

@@ -14,10 +14,12 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-// This is very brittle, see: https://github.com/tendermint/tendermint/issues/4740
 var (
+	// This is very brittle, see: https://github.com/tendermint/tendermint/issues/4740
 	regexpMissingHeight = regexp.MustCompile(`height \d+ (must be less than or equal to|is not available)`)
-	maxRetryAttempts    = 10
+
+	maxRetryAttempts = 10
+	timeout          = 5 * time.Second
 )
 
 // http provider uses an RPC client to obtain the necessary information.
@@ -28,14 +30,14 @@ type http struct {
 
 // New creates a HTTP provider, which is using the rpchttp.HTTP client under
 // the hood. If no scheme is provided in the remote URL, http will be used by
-// default.
+// default. The 5s timeout is used for all requests.
 func New(chainID, remote string) (provider.Provider, error) {
 	// Ensure URL scheme is set (default HTTP) when not provided.
 	if !strings.Contains(remote, "://") {
 		remote = "http://" + remote
 	}
 
-	httpClient, err := rpchttp.New(remote, "/websocket")
+	httpClient, err := rpchttp.NewWithTimeout(remote, "/websocket", timeout)
 	if err != nil {
 		return nil, err
 	}
@@ -93,15 +95,21 @@ func (p *http) ReportEvidence(ctx context.Context, ev types.Evidence) error {
 }
 
 func (p *http) validatorSet(ctx context.Context, height *int64) (*types.ValidatorSet, error) {
+	// Since the malicious node could report a massive number of pages, making us
+	// spend a considerable time iterating, we restrict the number of pages here.
+	// => 10000 validators max
+	const maxPages = 100
+
 	var (
-		maxPerPage = 100
+		perPage = 100
-		vals       = []*types.Validator{}
+		vals    = []*types.Validator{}
-		page       = 1
+		page    = 1
+		total   = -1
 	)
 
-	for len(vals)%maxPerPage == 0 {
+	for len(vals) != total && page <= maxPages {
 		for attempt := 1; attempt <= maxRetryAttempts; attempt++ {
-			res, err := p.client.Validators(ctx, height, &page, &maxPerPage)
+			res, err := p.client.Validators(ctx, height, &page, &perPage)
 			if err != nil {
 				// TODO: standardize errors on the RPC side
 				if regexpMissingHeight.MatchString(err.Error()) {
@@ -115,18 +123,28 @@ func (p *http) validatorSet(ctx context.Context, height *int64) (*types.Validato
 				time.Sleep(backoffTimeout(uint16(attempt)))
 				continue
 			}
-			if len(res.Validators) == 0 { // no more validators left
+
-				valSet, err := types.ValidatorSetFromExistingValidators(vals)
+			// Validate response.
-				if err != nil {
+			if len(res.Validators) == 0 {
-					return nil, provider.ErrBadLightBlock{Reason: err}
+				return nil, provider.ErrBadLightBlock{
+					Reason: fmt.Errorf("validator set is empty (height: %d, page: %d, per_page: %d)",
+						height, page, perPage),
 				}
-				return valSet, nil
 			}
+			if res.Total <= 0 {
+				return nil, provider.ErrBadLightBlock{
+					Reason: fmt.Errorf("total number of vals is <= 0: %d (height: %d, page: %d, per_page: %d)",
+						res.Total, height, page, perPage),
+				}
+			}
+
+			total = res.Total
 			vals = append(vals, res.Validators...)
 			page++
 			break
 		}
 	}
+
 	valSet, err := types.ValidatorSetFromExistingValidators(vals)
 	if err != nil {
 		return nil, provider.ErrBadLightBlock{Reason: err}

rpc/client/http/http.go

@@ -118,12 +118,12 @@ func New(remote, wsEndpoint string) (*HTTP, error) {
 
 // NewWithTimeout does the same thing as New, except you can set a Timeout for
 // http.Client. A Timeout of zero means no timeout.
-func NewWithTimeout(remote, wsEndpoint string, timeout uint) (*HTTP, error) {
+func NewWithTimeout(remote, wsEndpoint string, timeout time.Duration) (*HTTP, error) {
 	httpClient, err := jsonrpcclient.DefaultHTTPClient(remote)
 	if err != nil {
 		return nil, err
 	}
-	httpClient.Timeout = time.Duration(timeout) * time.Second
+	httpClient.Timeout = timeout
 	return NewWithClient(remote, wsEndpoint, httpClient)
 }
 

rpc/client/rpc_test.go

@@ -40,7 +40,7 @@ func getHTTPClient() *rpchttp.HTTP {
 	return c
 }
 
-func getHTTPClientWithTimeout(timeout uint) *rpchttp.HTTP {
+func getHTTPClientWithTimeout(timeout time.Duration) *rpchttp.HTTP {
 	rpcAddr := rpctest.GetConfig().RPC.ListenAddress
 	c, err := rpchttp.NewWithTimeout(rpcAddr, "/websocket", timeout)
 	if err != nil {
@@ -497,8 +497,7 @@ func TestTx(t *testing.T) {
 }
 
 func TestTxSearchWithTimeout(t *testing.T) {
-	// Get a client with a time-out of 10 secs.
+	timeoutClient := getHTTPClientWithTimeout(10 * time.Second)
-	timeoutClient := getHTTPClientWithTimeout(10)
 
 	_, _, tx := MakeTxKV()
 	_, err := timeoutClient.BroadcastTxCommit(context.Background(), tx)

state/helpers_test.go

@@ -10,6 +10,7 @@ import (
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/ed25519"
+	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
 	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
@@ -163,10 +164,18 @@ func makeHeaderPartsResponsesValPubKeyChange(
 	// If the pubkey is new, remove the old and add the new.
 	_, val := state.NextValidators.GetByIndex(0)
 	if !bytes.Equal(pubkey.Bytes(), val.PubKey.Bytes()) {
+		vPbPk, err := cryptoenc.PubKeyToProto(val.PubKey)
+		if err != nil {
+			panic(err)
+		}
+		pbPk, err := cryptoenc.PubKeyToProto(pubkey)
+		if err != nil {
+			panic(err)
+		}
 		abciResponses.EndBlock = &abci.ResponseEndBlock{
 			ValidatorUpdates: []abci.ValidatorUpdate{
-				types.TM2PB.NewValidatorUpdate(val.PubKey, 0),
+				{PubKey: vPbPk, Power: 0},
-				types.TM2PB.NewValidatorUpdate(pubkey, 10),
+				{PubKey: pbPk, Power: 10},
 			},
 		}
 	}
@@ -188,9 +197,13 @@ func makeHeaderPartsResponsesValPowerChange(
 	// If the pubkey is new, remove the old and add the new.
 	_, val := state.NextValidators.GetByIndex(0)
 	if val.VotingPower != power {
+		vPbPk, err := cryptoenc.PubKeyToProto(val.PubKey)
+		if err != nil {
+			panic(err)
+		}
 		abciResponses.EndBlock = &abci.ResponseEndBlock{
 			ValidatorUpdates: []abci.ValidatorUpdate{
-				types.TM2PB.NewValidatorUpdate(val.PubKey, power),
+				{PubKey: vPbPk, Power: power},
 			},
 		}
 	}

state/state_test.go

@@ -108,11 +108,11 @@ func TestABCIResponsesSaveLoad1(t *testing.T) {
 
 	abciResponses.DeliverTxs[0] = &abci.ResponseDeliverTx{Data: []byte("foo"), Events: nil}
 	abciResponses.DeliverTxs[1] = &abci.ResponseDeliverTx{Data: []byte("bar"), Log: "ok", Events: nil}
-	abciResponses.EndBlock = &abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{
+	pbpk, err := cryptoenc.PubKeyToProto(ed25519.GenPrivKey().PubKey())
-		types.TM2PB.NewValidatorUpdate(ed25519.GenPrivKey().PubKey(), 10),
+	require.NoError(t, err)
-	}}
+	abciResponses.EndBlock = &abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{{PubKey: pbpk, Power: 10}}}
 
-	err := stateStore.SaveABCIResponses(block.Height, abciResponses)
+	err = stateStore.SaveABCIResponses(block.Height, abciResponses)
 	require.NoError(t, err)
 	loadedABCIResponses, err := stateStore.LoadABCIResponses(block.Height)
 	assert.Nil(err)

types/params.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/tendermint/tendermint/crypto/ed25519"
+	"github.com/tendermint/tendermint/crypto/secp256k1"
 	"github.com/tendermint/tendermint/crypto/tmhash"
 	tmstrings "github.com/tendermint/tendermint/libs/strings"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
@@ -19,8 +21,16 @@ const (
 
 	// MaxBlockPartsCount is the maximum number of block parts.
 	MaxBlockPartsCount = (MaxBlockSizeBytes / BlockPartSizeBytes) + 1
+
+	ABCIPubKeyTypeEd25519   = ed25519.KeyType
+	ABCIPubKeyTypeSecp256k1 = secp256k1.KeyType
 )
 
+var ABCIPubKeyTypesToNames = map[string]string{
+	ABCIPubKeyTypeEd25519:   ed25519.PubKeyName,
+	ABCIPubKeyTypeSecp256k1: secp256k1.PubKeyName,
+}
+
 // ConsensusParams contains consensus critical parameters that determine the
 // validity of blocks.
 type ConsensusParams struct {

types/protobuf.go

@@ -2,28 +2,9 @@ package types
 
 import (
 	abci "github.com/tendermint/tendermint/abci/types"
-	"github.com/tendermint/tendermint/crypto"
-	"github.com/tendermint/tendermint/crypto/ed25519"
 	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
-	"github.com/tendermint/tendermint/crypto/secp256k1"
-	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 )
 
-//-------------------------------------------------------
-// Use strings to distinguish types in ABCI messages
-
-const (
-	ABCIPubKeyTypeEd25519   = ed25519.KeyType
-	ABCIPubKeyTypeSecp256k1 = secp256k1.KeyType
-)
-
-// TODO: Make non-global by allowing for registration of more pubkey types
-
-var ABCIPubKeyTypesToNames = map[string]string{
-	ABCIPubKeyTypeEd25519:   ed25519.PubKeyName,
-	ABCIPubKeyTypeSecp256k1: secp256k1.PubKeyName,
-}
-
 //-------------------------------------------------------
 
 // TM2PB is used for converting Tendermint ABCI to protobuf ABCI.
@@ -32,29 +13,6 @@ var TM2PB = tm2pb{}
 
 type tm2pb struct{}
 
-func (tm2pb) Header(header *Header) tmproto.Header {
-	return tmproto.Header{
-		Version: header.Version.ToProto(),
-		ChainID: header.ChainID,
-		Height:  header.Height,
-		Time:    header.Time,
-
-		LastBlockId: header.LastBlockID.ToProto(),
-
-		LastCommitHash: header.LastCommitHash,
-		DataHash:       header.DataHash,
-
-		ValidatorsHash:     header.ValidatorsHash,
-		NextValidatorsHash: header.NextValidatorsHash,
-		ConsensusHash:      header.ConsensusHash,
-		AppHash:            header.AppHash,
-		LastResultsHash:    header.LastResultsHash,
-
-		EvidenceHash:    header.EvidenceHash,
-		ProposerAddress: header.ProposerAddress,
-	}
-}
-
 func (tm2pb) Validator(val *Validator) abci.Validator {
 	return abci.Validator{
 		Address: val.PubKey.Address(),
@@ -62,20 +20,6 @@ func (tm2pb) Validator(val *Validator) abci.Validator {
 	}
 }
 
-func (tm2pb) BlockID(blockID BlockID) tmproto.BlockID {
-	return tmproto.BlockID{
-		Hash:          blockID.Hash,
-		PartSetHeader: TM2PB.PartSetHeader(blockID.PartSetHeader),
-	}
-}
-
-func (tm2pb) PartSetHeader(header PartSetHeader) tmproto.PartSetHeader {
-	return tmproto.PartSetHeader{
-		Total: header.Total,
-		Hash:  header.Hash,
-	}
-}
-
 // XXX: panics on unknown pubkey type
 func (tm2pb) ValidatorUpdate(val *Validator) abci.ValidatorUpdate {
 	pk, err := cryptoenc.PubKeyToProto(val.PubKey)
@@ -97,18 +41,6 @@ func (tm2pb) ValidatorUpdates(vals *ValidatorSet) []abci.ValidatorUpdate {
 	return validators
 }
 
-// XXX: panics on nil or unknown pubkey type
-func (tm2pb) NewValidatorUpdate(pubkey crypto.PubKey, power int64) abci.ValidatorUpdate {
-	pubkeyABCI, err := cryptoenc.PubKeyToProto(pubkey)
-	if err != nil {
-		panic(err)
-	}
-	return abci.ValidatorUpdate{
-		PubKey: pubkeyABCI,
-		Power:  power,
-	}
-}
-
 //----------------------------------------------------------------------------
 
 // PB2TM is used for converting protobuf ABCI to Tendermint ABCI.

types/protobuf_test.go

@@ -52,25 +52,6 @@ func TestABCIValidators(t *testing.T) {
 	assert.Equal(t, tmValExpected, tmVals[0])
 }
 
-type pubKeyEddie struct{}
-
-func (pubKeyEddie) Address() Address                            { return []byte{} }
-func (pubKeyEddie) Bytes() []byte                               { return []byte{} }
-func (pubKeyEddie) VerifySignature(msg []byte, sig []byte) bool { return false }
-func (pubKeyEddie) Equals(crypto.PubKey) bool                   { return false }
-func (pubKeyEddie) String() string                              { return "" }
-func (pubKeyEddie) Type() string                                { return "pubKeyEddie" }
-
-func TestABCIValidatorFromPubKeyAndPower(t *testing.T) {
-	pubkey := ed25519.GenPrivKey().PubKey()
-
-	abciVal := TM2PB.NewValidatorUpdate(pubkey, 10)
-	assert.Equal(t, int64(10), abciVal.Power)
-
-	assert.Panics(t, func() { TM2PB.NewValidatorUpdate(nil, 10) })
-	assert.Panics(t, func() { TM2PB.NewValidatorUpdate(pubKeyEddie{}, 10) })
-}
-
 func TestABCIValidatorWithoutPubKey(t *testing.T) {
 	pkEd := ed25519.GenPrivKey().PubKey()
 

types/validator_set.go

@@ -991,16 +991,21 @@ func ValidatorSetFromProto(vp *tmproto.ValidatorSet) (*ValidatorSet, error) {
 	return vals, vals.ValidateBasic()
 }
 
-// ValidatorSetFromExistingValidators takes an existing array of validators and rebuilds
+// ValidatorSetFromExistingValidators takes an existing array of validators and
-// the exact same validator set that corresponds to it without changing the proposer priority or power
+// rebuilds the exact same validator set that corresponds to it without
-// if any of the validators fail validate basic then an empty set is returned.
+// changing the proposer priority or power if any of the validators fail
+// validate basic then an empty set is returned.
 func ValidatorSetFromExistingValidators(valz []*Validator) (*ValidatorSet, error) {
+	if len(valz) == 0 {
+		return nil, errors.New("validator set is empty")
+	}
 	for _, val := range valz {
 		err := val.ValidateBasic()
 		if err != nil {
 			return nil, fmt.Errorf("can't create validator set: %w", err)
 		}
 	}
+
 	vals := &ValidatorSet{
 		Validators: valz,
 	}