lite2: replace primary provider with alternative when unavailable (#4354)

Closes issue #4338

Uses a wrapper function around both the signedHeader and validatorSet calls to the primary provider which attempts to retrieve the information 5 times before deeming the provider unavailable and replacing the primary provider with the first alternative before trying recursively again (until all alternatives are depleted)

Employs a mutex lock for any operations involving the providers of the light client to ensure no operations occurs whilst the new primary is chosen.

Commits:

* created swapProvider function

* eliminates old primary provider after replacement. Uses a mutex when changing providers

* renamed to replaceProvider

* created wrapped functions for signed header and val set

* created test for primary provider replacement

* implemented suggested revisions

* created Witnesses() and Primary()

* modified backoffAndJitterTime

* modified backoffAndJitterTime

* changed backoff base and jitter to functional arguments

* implemented suggested changes

* removed backoff function

* changed exp function to match go version

* halved the backoff time

* removed seeding and added comments

* fixed incorrect test

* extract backoff timeout calc into a function

Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>

lite2/provider/mock/mock.go

@@ -1,19 +1,20 @@
 package mock
 
 import (
+	"github.com/pkg/errors"
+
 	"github.com/tendermint/tendermint/lite2/provider"
 	"github.com/tendermint/tendermint/types"
 )
 
-// mock provider allows to directly set headers & vals, which can be handy when
-// testing.
 type mock struct {
 	chainID string
 	headers map[int64]*types.SignedHeader
 	vals    map[int64]*types.ValidatorSet
 }
 
-// New creates a mock provider.
+// New creates a mock provider with the given set of headers and validator
+// sets.
 func New(chainID string, headers map[int64]*types.SignedHeader, vals map[int64]*types.ValidatorSet) provider.Provider {
 	return &mock{
 		chainID: chainID,
@@ -45,3 +46,24 @@ func (p *mock) ValidatorSet(height int64) (*types.ValidatorSet, error) {
 	}
 	return nil, provider.ErrValidatorSetNotFound
 }
+
+type deadMock struct {
+	chainID string
+}
+
+// NewDeadMock creates a mock provider that always errors.
+func NewDeadMock(chainID string) provider.Provider {
+	return &deadMock{chainID: chainID}
+}
+
+func (p *deadMock) ChainID() string {
+	return p.chainID
+}
+
+func (p *deadMock) SignedHeader(height int64) (*types.SignedHeader, error) {
+	return nil, errors.New("no response from provider")
+}
+
+func (p *deadMock) ValidatorSet(height int64) (*types.ValidatorSet, error) {
+	return nil, errors.New("no response from provider")
+}