add params proto messages

This commit is contained in:
Callum Waters
2021-08-06 14:48:25 +02:00
parent e5f9dd2736
commit 2bde896c13
14 changed files with 929 additions and 336 deletions
+64 -31
View File
@@ -3,7 +3,6 @@ package statesync
import (
"context"
"errors"
"fmt"
"sync"
"time"
@@ -26,12 +25,19 @@ var (
// blocks. NOTE: It is not the responsibility of the dispatcher to verify the
// light blocks.
type dispatcher struct {
// a pool of peers to send light block request too
availablePeers *peerlist
requestCh chan<- p2p.Envelope
timeout time.Duration
// timeout for light block delivery (immutable)
timeout time.Duration
mtx sync.Mutex
calls map[types.NodeID]chan *types.LightBlock
mtx sync.Mutex
// the set of providers that the dispatcher is providing for (is distinct
// from available peers)
providers map[types.NodeID]struct{}
// all pending calls that have been dispatched and are awaiting an answer
calls map[types.NodeID]chan *types.LightBlock
// signals whether the underlying reactor is still running
running bool
}
@@ -40,8 +46,8 @@ func newDispatcher(requestCh chan<- p2p.Envelope, timeout time.Duration) *dispat
availablePeers: newPeerList(),
timeout: timeout,
requestCh: requestCh,
providers: make(map[types.NodeID]struct{}),
calls: make(map[types.NodeID]chan *types.LightBlock),
running: true,
}
}
@@ -49,6 +55,10 @@ func newDispatcher(requestCh chan<- p2p.Envelope, timeout time.Duration) *dispat
// in a list, tracks the call and waits for the reactor to pass along the response
func (d *dispatcher) LightBlock(ctx context.Context, height int64) (*types.LightBlock, types.NodeID, error) {
d.mtx.Lock()
// check that the dispatcher is connected to the reactor
if !d.running {
return nil, "", errDisconnected
}
// check to see that the dispatcher is connected to at least one peer
if d.availablePeers.Len() == 0 && len(d.calls) == 0 {
d.mtx.Unlock()
@@ -59,27 +69,36 @@ func (d *dispatcher) LightBlock(ctx context.Context, height int64) (*types.Light
// fetch the next peer id in the list and request a light block from that
// peer
peer := d.availablePeers.Pop(ctx)
defer d.release(peer)
lb, err := d.lightBlock(ctx, height, peer)
return lb, peer, err
}
// Providers turns the dispatcher into a set of providers (per peer) which can
// be used by a light client
func (d *dispatcher) Providers(chainID string, timeout time.Duration) []provider.Provider {
func (d *dispatcher) Providers(chainID string) []provider.Provider {
providers := make([]provider.Provider, d.availablePeers.Len())
for i := 0; i < cap(providers); i++ {
peer := d.availablePeers.Pop(context.Background())
providers[i] = d.CreateProvider(peer, chainID)
}
return providers
}
// Creates an individual provider from a peer id that the dispatcher is
// connected with.
func (d *dispatcher) CreateProvider(peer types.NodeID, chainID string) provider.Provider {
d.mtx.Lock()
defer d.mtx.Unlock()
providers := make([]provider.Provider, d.availablePeers.Len())
peers := d.availablePeers.Peers()
for index, peer := range peers {
providers[index] = &blockProvider{
peer: peer,
dispatcher: d,
chainID: chainID,
timeout: timeout,
}
d.availablePeers.Remove(peer)
d.providers[peer] = struct{}{}
return &blockProvider{
peer: peer,
dispatcher: d,
chainID: chainID,
timeout: d.timeout,
}
return providers
}
func (d *dispatcher) stop() {
@@ -111,11 +130,9 @@ func (d *dispatcher) lightBlock(ctx context.Context, height int64, peer types.No
return resp, nil
case <-ctx.Done():
d.release(peer)
return nil, nil
return nil, ctx.Err()
case <-time.After(d.timeout):
d.release(peer)
return nil, errNoResponse
}
}
@@ -132,10 +149,6 @@ func (d *dispatcher) respond(lb *proto.LightBlock, peer types.NodeID) error {
// this can also happen if the response came in after the timeout
return errUnsolicitedResponse
}
// release the peer after returning the response
defer d.availablePeers.Append(peer)
defer close(answerCh)
defer delete(d.calls, peer)
if lb == nil {
answerCh <- nil
@@ -144,7 +157,7 @@ func (d *dispatcher) respond(lb *proto.LightBlock, peer types.NodeID) error {
block, err := types.LightBlockFromProto(lb)
if err != nil {
fmt.Println("error with converting light block")
answerCh <- nil
return err
}
@@ -152,20 +165,36 @@ func (d *dispatcher) respond(lb *proto.LightBlock, peer types.NodeID) error {
return nil
}
// addPeer adds a peer to the dispatcher
func (d *dispatcher) addPeer(peer types.NodeID) {
d.availablePeers.Append(peer)
}
// removePeer removes a peer from the dispatcher
func (d *dispatcher) removePeer(peer types.NodeID) {
d.mtx.Lock()
defer d.mtx.Unlock()
if _, ok := d.calls[peer]; ok {
if call, ok := d.calls[peer]; ok {
call <- nil
close(call)
delete(d.calls, peer)
} else {
d.availablePeers.Remove(peer)
}
}
// peerCount returns the amount of peers that the dispatcher is connected with
func (d *dispatcher) peerCount() int {
return d.availablePeers.Len()
}
func (d *dispatcher) isConnected(peer types.NodeID) bool {
d.mtx.Lock()
defer d.mtx.Unlock()
_, ok := d.providers[peer]
return ok
}
// dispatch takes a peer and allocates it a channel so long as it's not already
// busy and the receiving channel is still running. It then dispatches the message
func (d *dispatcher) dispatch(peer types.NodeID, height int64) (chan *types.LightBlock, error) {
@@ -223,15 +252,19 @@ type blockProvider struct {
}
func (p *blockProvider) LightBlock(ctx context.Context, height int64) (*types.LightBlock, error) {
// FIXME: The provider doesn't know if the dispatcher is still connected to
// that peer. If the connection is dropped for whatever reason the
// dispatcher needs to be able to relay this back to the provider so it can
// return ErrConnectionClosed instead of ErrNoResponse
// check if the underlying reactor is still connected with the peer
if !p.dispatcher.isConnected(p.peer) {
return nil, provider.ErrConnectionClosed
}
ctx, cancel := context.WithTimeout(ctx, p.timeout)
defer cancel()
lb, _ := p.dispatcher.lightBlock(ctx, height, p.peer)
lb, err := p.dispatcher.lightBlock(ctx, height, p.peer)
if err != nil {
return nil, provider.ErrUnreliableProvider{Reason: err.Error()}
}
if lb == nil {
return nil, provider.ErrNoResponse
return nil, provider.ErrLightBlockNotFound
}
if err := lb.ValidateBasic(p.chainID); err != nil {
+38 -6
View File
@@ -49,6 +49,9 @@ func TestDispatcherBasic(t *testing.T) {
}(int64(i))
}
wg.Wait()
// we should finish with as many peers as we started out with
assert.Equal(t, 5, d.peerCount())
}
func TestDispatcherReturnsNoBlock(t *testing.T) {
@@ -99,7 +102,7 @@ func TestDispatcherReturnsBlockOncePeerAvailable(t *testing.T) {
lb, peerResult, err := d.LightBlock(wrapped, 1)
require.Nil(t, lb)
require.Equal(t, peerFromSet, peerResult)
require.Nil(t, err)
require.Equal(t, context.Canceled, err)
// calls to dispatcher.Lightblock write into the dispatcher's requestCh.
// we read from the requestCh here to unblock the requestCh for future
@@ -134,7 +137,7 @@ func TestDispatcherProviders(t *testing.T) {
closeCh := make(chan struct{})
defer close(closeCh)
d := newDispatcher(ch, 1*time.Second)
d := newDispatcher(ch, 5*time.Second)
go handleRequests(t, d, ch, closeCh)
@@ -143,16 +146,17 @@ func TestDispatcherProviders(t *testing.T) {
d.addPeer(peer)
}
providers := d.Providers(chainID, 5*time.Second)
providers := d.Providers(chainID)
require.Len(t, providers, 5)
for i, p := range providers {
bp, ok := p.(*blockProvider)
require.True(t, ok)
assert.Equal(t, bp.String(), string(peers[i]))
assert.Equal(t, string(peers[i]), bp.String(), i)
lb, err := p.LightBlock(context.Background(), 10)
assert.Error(t, err)
assert.Nil(t, lb)
}
require.Equal(t, 0, d.peerCount())
}
func TestPeerListBasic(t *testing.T) {
@@ -178,13 +182,22 @@ func TestPeerListBasic(t *testing.T) {
}
assert.Equal(t, half, peerList.Len())
// removing a peer that doesn't exist should not change the list
peerList.Remove(types.NodeID("lp"))
assert.Equal(t, half, peerList.Len())
// removing a peer that exists should decrease the list size by one
peerList.Remove(peerSet[half])
half++
assert.Equal(t, peerSet[half], peerList.Pop(ctx))
assert.Equal(t, numPeers-half-1, peerList.Len())
// popping the next peer should work as expected
assert.Equal(t, peerSet[half+1], peerList.Pop(ctx))
assert.Equal(t, numPeers-half-2, peerList.Len())
// append the two peers back
peerList.Append(peerSet[half])
peerList.Append(peerSet[half+1])
assert.Equal(t, half, peerList.Len())
}
func TestPeerListBlocksWhenEmpty(t *testing.T) {
@@ -277,6 +290,25 @@ func TestPeerListConcurrent(t *testing.T) {
}
}
func TestPeerListRemove(t *testing.T) {
peerList := newPeerList()
numPeers := 10
peerSet := createPeerSet(numPeers)
for _, peer := range peerSet {
peerList.Append(peer)
}
for _, peer := range peerSet {
peerList.Remove(peer)
for _, p := range peerList.Peers() {
require.NotEqual(t, p, peer)
}
numPeers--
require.Equal(t, numPeers, peerList.Len())
}
}
// handleRequests is a helper function usually run in a separate go routine to
// imitate the expected responses of the reactor wired to the dispatcher
func handleRequests(t *testing.T, d *dispatcher, ch chan p2p.Envelope, closeCh chan struct{}) {
-50
View File
@@ -1,50 +0,0 @@
package statesync
import (
"context"
"time"
mock "github.com/stretchr/testify/mock"
state "github.com/tendermint/tendermint/state"
)
// MockSyncReactor is an autogenerated mock type for the SyncReactor type.
// Because of the stateprovider uses in Sync(), we use package statesync instead of mocks.
type MockSyncReactor struct {
mock.Mock
}
// Backfill provides a mock function with given fields: _a0
func (_m *MockSyncReactor) Backfill(_a0 state.State) error {
ret := _m.Called(_a0)
var r0 error
if rf, ok := ret.Get(0).(func(state.State) error); ok {
r0 = rf(_a0)
} else {
r0 = ret.Error(0)
}
return r0
}
// Sync provides a mock function with given fields: _a0, _a1, _a2
func (_m *MockSyncReactor) Sync(_a0 context.Context, _a1 StateProvider, _a2 time.Duration) (state.State, error) {
ret := _m.Called(_a0, _a1, _a2)
var r0 state.State
if rf, ok := ret.Get(0).(func(context.Context, StateProvider, time.Duration) state.State); ok {
r0 = rf(_a0, _a1, _a2)
} else {
r0 = ret.Get(0).(state.State)
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, StateProvider, time.Duration) error); ok {
r1 = rf(_a0, _a1, _a2)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
+57 -18
View File
@@ -16,6 +16,7 @@ import (
"github.com/tendermint/tendermint/internal/p2p"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/libs/service"
"github.com/tendermint/tendermint/light"
ssproto "github.com/tendermint/tendermint/proto/tendermint/statesync"
"github.com/tendermint/tendermint/proxy"
sm "github.com/tendermint/tendermint/state"
@@ -81,6 +82,9 @@ const (
// LightBlockChannel exchanges light blocks
LightBlockChannel = p2p.ChannelID(0x62)
// ParamsChannel exchanges consensus params
ParamsChannel = p2p.ChannelID(0x63)
// recentSnapshots is the number of recent snapshots to send and receive per peer.
recentSnapshots = 10
@@ -102,12 +106,6 @@ const (
maxLightBlockRequestRetries = 20
)
// SyncReactor defines an interface used for testing abilities of node.startStateSync.
type SyncReactor interface {
Sync(context.Context, StateProvider, time.Duration) (sm.State, error)
Backfill(sm.State) error
}
// Reactor handles state sync, both restoring snapshots for the local node and
// serving snapshots for other nodes.
type Reactor struct {
@@ -214,13 +212,14 @@ func (r *Reactor) OnStop() {
}
// Sync runs a state sync, fetching snapshots and providing chunks to the
// application. It also saves tendermint state and runs a backfill process to
// retrieve the necessary amount of headers, commits and validators sets to be
// able to process evidence and participate in consensus.
// application. At the close of the operation, Sync will bootstrap the state
// store and persist the commit at that height so that either consensus or
// blocksync can commence. It will then proceed to backfill the necessary amount
// of historical blocks before participating in consensus
func (r *Reactor) Sync(
ctx context.Context,
stateProvider StateProvider,
discoveryTime time.Duration,
chainID string,
initialHeight int64,
) (sm.State, error) {
r.mtx.Lock()
if r.syncer != nil {
@@ -228,9 +227,31 @@ func (r *Reactor) Sync(
return sm.State{}, errors.New("a state sync is already in progress")
}
if stateProvider == nil {
r.mtx.Unlock()
return sm.State{}, errors.New("the stateProvider should not be nil when doing the state sync")
to := light.TrustOptions{
Period: r.cfg.TrustPeriod,
Height: r.cfg.TrustHeight,
Hash: r.cfg.TrustHashBytes(),
}
spLogger := r.Logger.With("module", "stateprovider")
var (
stateProvider StateProvider
err error
)
if r.cfg.UseP2P {
// state provider needs at least two connected peers to initialize
spLogger.Info("Generating P2P state provider")
r.waitForEnoughPeers(ctx, 2)
stateProvider, err = NewP2PStateProvider(ctx, chainID, initialHeight, r.Dispatcher(), to, spLogger)
if err != nil {
return sm.State{}, err
}
spLogger.Info("Finished generating P2P state provider")
} else {
stateProvider, err = NewRPCStateProvider(ctx, chainID, initialHeight, r.cfg.RPCServers, to, spLogger)
if err != nil {
return sm.State{}, err
}
}
r.syncer = newSyncer(
@@ -253,7 +274,7 @@ func (r *Reactor) Sync(
}
}
state, commit, err := r.syncer.SyncAny(ctx, discoveryTime, requestSnapshotsHook)
state, commit, err := r.syncer.SyncAny(ctx, r.cfg.DiscoveryTime, requestSnapshotsHook)
if err != nil {
return sm.State{}, err
}
@@ -272,6 +293,11 @@ func (r *Reactor) Sync(
return sm.State{}, fmt.Errorf("failed to store last seen commit: %w", err)
}
err = r.Backfill(ctx, state)
if err != nil {
return sm.State{}, err
}
return state, nil
}
@@ -279,7 +305,7 @@ func (r *Reactor) Sync(
// order. It does not stop verifying blocks until reaching a block with a height
// and time that is less or equal to the stopHeight and stopTime. The
// trustedBlockID should be of the header at startHeight.
func (r *Reactor) Backfill(state sm.State) error {
func (r *Reactor) Backfill(ctx context.Context, state sm.State) error {
params := state.ConsensusParams.Evidence
stopHeight := state.LastBlockHeight - params.MaxAgeNumBlocks
stopTime := state.LastBlockTime.Add(-params.MaxAgeDuration)
@@ -290,7 +316,7 @@ func (r *Reactor) Backfill(state sm.State) error {
stopTime = state.LastBlockTime
}
return r.backfill(
context.Background(),
ctx,
state.ChainID,
state.LastBlockHeight,
stopHeight,
@@ -732,7 +758,7 @@ func (r *Reactor) processCh(ch *p2p.Channel, chName string) {
// processPeerUpdate processes a PeerUpdate, returning an error upon failing to
// handle the PeerUpdate or if a panic is recovered.
func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
r.Logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
r.Logger.Info("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
r.mtx.RLock()
defer r.mtx.RUnlock()
@@ -750,6 +776,7 @@ func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
}
r.dispatcher.removePeer(peerUpdate.NodeID)
}
r.Logger.Info("processed peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
}
// processPeerUpdates initiates a blocking process where we listen for and handle
@@ -839,5 +866,17 @@ func (r *Reactor) fetchLightBlock(height uint64) (*types.LightBlock, error) {
},
ValidatorSet: vals,
}, nil
}
func (r *Reactor) waitForEnoughPeers(ctx context.Context, numPeers int) {
for {
select {
case <-ctx.Done():
return
case <-time.After(200 * time.Millisecond):
if r.dispatcher.peerCount() >= numPeers {
return
}
}
}
}
+38 -1
View File
@@ -18,6 +18,7 @@ import (
"github.com/tendermint/tendermint/internal/statesync/mocks"
"github.com/tendermint/tendermint/internal/test/factory"
"github.com/tendermint/tendermint/libs/log"
"github.com/tendermint/tendermint/light"
"github.com/tendermint/tendermint/light/provider"
ssproto "github.com/tendermint/tendermint/proto/tendermint/statesync"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
@@ -388,8 +389,9 @@ func TestReactor_Dispatcher(t *testing.T) {
go handleLightBlockRequests(t, chain, rts.blockOutCh, rts.blockInCh, closeCh, 0)
dispatcher := rts.reactor.Dispatcher()
providers := dispatcher.Providers(factory.DefaultTestChainID, 5*time.Second)
providers := dispatcher.Providers(factory.DefaultTestChainID)
require.Len(t, providers, 2)
require.Equal(t, 2, dispatcher.peerCount())
wg := sync.WaitGroup{}
@@ -416,6 +418,41 @@ func TestReactor_Dispatcher(t *testing.T) {
t.Fail()
case <-ctx.Done():
}
t.Log(dispatcher.availablePeers.Peers())
require.Equal(t, 2, dispatcher.peerCount())
rts.peerUpdateCh <- p2p.PeerUpdate{
NodeID: types.NodeID("cc"),
Status: p2p.PeerStatusUp,
}
require.Equal(t, 3, dispatcher.peerCount())
// we now test the p2p state provider
lb, _, err := dispatcher.LightBlock(ctx, 2)
require.NoError(t, err)
to := light.TrustOptions{
Period: 24 * time.Hour,
Height: lb.Height,
Hash: lb.Hash(),
}
p2pStateProvider, err := NewP2PStateProvider(ctx, "testchain", 1, rts.reactor.Dispatcher(), to, log.TestingLogger())
require.NoError(t, err)
appHash, err := p2pStateProvider.AppHash(ctx, 5)
require.NoError(t, err)
require.Len(t, appHash, 20)
state, err := p2pStateProvider.State(ctx, 6)
require.NoError(t, err)
require.Equal(t, appHash, state.AppHash)
commit, err := p2pStateProvider.Commit(ctx, 5)
require.NoError(t, err)
require.Equal(t, commit.BlockID, state.LastBlockID)
}
func TestReactor_Backfill(t *testing.T) {
+22 -24
View File
@@ -18,6 +18,7 @@ import (
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/types"
"github.com/tendermint/tendermint/version"
)
//go:generate ../../scripts/mockery_generate.sh StateProvider
@@ -33,20 +34,21 @@ type StateProvider interface {
State(ctx context.Context, height uint64) (sm.State, error)
}
// lightClientStateProvider is a state provider using the light client.
type lightClientStateProvider struct {
// stateProvider implements the above interface using a light client to fetch and
// verify the AppHash, Commit, and Tendermint State from trusted data in order
// to bootstrap a node. The stateProvider can either be initated using RPC or
// P2P Providers.
type stateProvider struct {
tmsync.Mutex // light.Client is not concurrency-safe
lc *light.Client
version sm.Version
initialHeight int64
providers map[lightprovider.Provider]string
}
// NewLightClientStateProvider creates a new StateProvider using a light client and RPC clients.
func NewLightClientStateProvider(
// NewRPCStateProvider creates a new StateProvider using a light client and RPC clients.
func NewRPCStateProvider(
ctx context.Context,
chainID string,
version sm.Version,
initialHeight int64,
servers []string,
trustOptions light.TrustOptions,
@@ -75,26 +77,24 @@ func NewLightClientStateProvider(
if err != nil {
return nil, err
}
return &lightClientStateProvider{
return &stateProvider{
lc: lc,
version: version,
initialHeight: initialHeight,
providers: providerRemotes,
}, nil
}
// NewLightClientStateProviderFromDispatcher creates a light client state
// provider but uses a p2p connected dispatched instead of RPC endpoints
func NewLightClientStateProviderFromDispatcher(
// NewP2PStateProvider creates a light client state
// provider but uses a dispatcher connected to the P2P layer
func NewP2PStateProvider(
ctx context.Context,
chainID string,
version sm.Version,
initialHeight int64,
dispatcher *dispatcher,
trustOptions light.TrustOptions,
logger log.Logger,
) (StateProvider, error) {
providers := dispatcher.Providers(chainID, 30*time.Second)
providers := dispatcher.Providers(chainID)
if len(providers) < 2 {
return nil, fmt.Errorf("at least 2 peers are required, got %d", len(providers))
}
@@ -110,16 +110,15 @@ func NewLightClientStateProviderFromDispatcher(
return nil, err
}
return &lightClientStateProvider{
return &stateProvider{
lc: lc,
version: version,
initialHeight: initialHeight,
providers: providersMap,
}, nil
}
// AppHash implements StateProvider.
func (s *lightClientStateProvider) AppHash(ctx context.Context, height uint64) ([]byte, error) {
func (s *stateProvider) AppHash(ctx context.Context, height uint64) ([]byte, error) {
s.Lock()
defer s.Unlock()
@@ -128,7 +127,7 @@ func (s *lightClientStateProvider) AppHash(ctx context.Context, height uint64) (
if err != nil {
return nil, err
}
// We also try to fetch the blocks at height H and H+2, since we need these
// We also try to fetch the blocks at H+2, since we need these
// when building the state while restoring the snapshot. This avoids the race
// condition where we try to restore a snapshot before H+2 exists.
//
@@ -140,15 +139,11 @@ func (s *lightClientStateProvider) AppHash(ctx context.Context, height uint64) (
if err != nil {
return nil, err
}
_, err = s.lc.VerifyLightBlockAtHeight(ctx, int64(height), time.Now())
if err != nil {
return nil, err
}
return header.AppHash, nil
}
// Commit implements StateProvider.
func (s *lightClientStateProvider) Commit(ctx context.Context, height uint64) (*types.Commit, error) {
func (s *stateProvider) Commit(ctx context.Context, height uint64) (*types.Commit, error) {
s.Lock()
defer s.Unlock()
header, err := s.lc.VerifyLightBlockAtHeight(ctx, int64(height), time.Now())
@@ -159,13 +154,12 @@ func (s *lightClientStateProvider) Commit(ctx context.Context, height uint64) (*
}
// State implements StateProvider.
func (s *lightClientStateProvider) State(ctx context.Context, height uint64) (sm.State, error) {
func (s *stateProvider) State(ctx context.Context, height uint64) (sm.State, error) {
s.Lock()
defer s.Unlock()
state := sm.State{
ChainID: s.lc.ChainID(),
Version: s.version,
InitialHeight: s.initialHeight,
}
if state.InitialHeight == 0 {
@@ -193,6 +187,10 @@ func (s *lightClientStateProvider) State(ctx context.Context, height uint64) (sm
return sm.State{}, err
}
state.Version = sm.Version{
Consensus: currentLightBlock.Version,
Software: version.TMVersion,
}
state.LastBlockHeight = lastLightBlock.Height
state.LastBlockTime = lastLightBlock.Time
state.LastBlockID = lastLightBlock.Commit.BlockID