lite2: replace primary provider with alternative when unavailable (#4354)

Closes issue #4338

Uses a wrapper function around both the signedHeader and validatorSet calls to the primary provider which attempts to retrieve the information 5 times before deeming the provider unavailable and replacing the primary provider with the first alternative before trying recursively again (until all alternatives are depleted)

Employs a mutex lock for any operations involving the providers of the light client to ensure no operations occurs whilst the new primary is chosen.

Commits:

* created swapProvider function

* eliminates old primary provider after replacement. Uses a mutex when changing providers

* renamed to replaceProvider

* created wrapped functions for signed header and val set

* created test for primary provider replacement

* implemented suggested revisions

* created Witnesses() and Primary()

* modified backoffAndJitterTime

* modified backoffAndJitterTime

* changed backoff base and jitter to functional arguments

* implemented suggested changes

* removed backoff function

* changed exp function to match go version

* halved the backoff time

* removed seeding and added comments

* fixed incorrect test

* extract backoff timeout calc into a function

Co-authored-by: Anton Kaliaev <anton.kalyaev@gmail.com>

lite2/client_test.go

@@ -867,3 +867,59 @@ func TestClient_Concurrency(t *testing.T) {
 
 	wg.Wait()
 }
+
+func TestProvider_Replacement(t *testing.T) {
+	const (
+		chainID = "TestProvider_Replacement"
+	)
+
+	var (
+		keys = genPrivKeys(4)
+		// 20, 30, 40, 50 - the first 3 don't have 2/3, the last 3 do!
+		vals     = keys.ToValidators(20, 10)
+		bTime, _ = time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+		header   = keys.GenSignedHeader(chainID, 1, bTime, nil, vals, vals,
+			[]byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys))
+		primary = mockp.NewDeadMock(chainID)
+		witness = mockp.New(
+			chainID,
+			map[int64]*types.SignedHeader{
+				// trusted header
+				1: header,
+				// interim header (3/3 signed)
+				2: keys.GenSignedHeader(chainID, 2, bTime.Add(30*time.Minute), nil, vals, vals,
+					[]byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)),
+				// last header (3/3 signed)
+				3: keys.GenSignedHeader(chainID, 3, bTime.Add(1*time.Hour), nil, vals, vals,
+					[]byte("app_hash"), []byte("cons_hash"), []byte("results_hash"), 0, len(keys)),
+			},
+			map[int64]*types.ValidatorSet{
+				1: vals,
+				2: vals,
+				3: vals,
+				4: vals,
+			},
+		)
+	)
+
+	c, err := NewClient(
+		chainID,
+		TrustOptions{
+			Period: 4 * time.Hour,
+			Height: 1,
+			Hash:   header.Hash(),
+		},
+		primary,
+		[]provider.Provider{witness},
+		dbs.New(dbm.NewMemDB(), chainID),
+		UpdatePeriod(0),
+		Logger(log.TestingLogger()),
+	)
+	require.NoError(t, err)
+	err = c.Start()
+	require.NoError(t, err)
+	defer c.Stop()
+	assert.NotEqual(t, c.Primary(), primary)
+	assert.Equal(t, 0, len(c.Witnesses()))
+
+}