light: improve provider handling (#6053)

Introduces heuristics that track the amount of no responses or unavailable blocks a provider has for more robust provider handling by the light client. Use concurrent calls to all witnesses when a new primary is needed.
2026-09-30 11:45:37 +00:00 · 2021-03-01 12:04:02 +01:00
parent e2384a00ce
commit 37d36cd5bc
9 changed files with 334 additions and 178 deletions
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"sort"
+	"sync"
 	"time"

 	"github.com/tendermint/tendermint/libs/log"
@@ -126,8 +127,10 @@ type Client struct {
 	pruningSize uint16
 	// See ConfirmationFunction option
 	confirmationFn func(action string) bool
-
-	quit chan struct{}
+	// The light client keeps track of how many times it has requested a light
+	// block from it's providers. When this exceeds the amount of witnesses the
+	// light client will just return the last error sent by the providers
+	// repeatRequests uint16

 	logger log.Logger
 }
@@ -161,14 +164,14 @@ func NewClient(
 	}

 	if c.latestTrustedBlock != nil {
-		c.logger.Info("Checking trusted light block using options")
+		c.logger.Info("checking trusted light block using options")
 		if err := c.checkTrustedHeaderUsingOptions(ctx, trustOptions); err != nil {
 			return nil, err
 		}
 	}

 	if c.latestTrustedBlock == nil || c.latestTrustedBlock.Height < trustOptions.Height {
-		c.logger.Info("Downloading trusted light block using options")
+		c.logger.Info("downloading trusted light block using options")
 		if err := c.initializeWithTrustOptions(ctx, trustOptions); err != nil {
 			return nil, err
 		}
@@ -199,7 +202,6 @@ func NewClientFromTrustedStore(
 		trustedStore:     trustedStore,
 		pruningSize:      defaultPruningSize,
 		confirmationFn:   func(action string) bool { return true },
-		quit:             make(chan struct{}),
 		logger:           log.NewNopLogger(),
 	}

@@ -237,7 +239,7 @@ func (c *Client) restoreTrustedLightBlock() error {
 			return fmt.Errorf("can't get last trusted light block: %w", err)
 		}
 		c.latestTrustedBlock = trustedBlock
-		c.logger.Info("Restored trusted light block", "height", lastHeight)
+		c.logger.Info("restored trusted light block", "height", lastHeight)
 	}

 	return nil
@@ -273,7 +275,7 @@ func (c *Client) checkTrustedHeaderUsingOptions(ctx context.Context, options Tru
 	case options.Height == c.latestTrustedBlock.Height:
 		primaryHash = options.Hash
 	case options.Height < c.latestTrustedBlock.Height:
-		c.logger.Info("Client initialized with old header (trusted is more recent)",
+		c.logger.Info("client initialized with old header (trusted is more recent)",
 			"old", options.Height,
 			"trustedHeight", c.latestTrustedBlock.Height,
 			"trustedHash", c.latestTrustedBlock.Hash())
@@ -299,11 +301,11 @@ func (c *Client) checkTrustedHeaderUsingOptions(ctx context.Context, options Tru
 	}

 	if !bytes.Equal(primaryHash, c.latestTrustedBlock.Hash()) {
-		c.logger.Info("Prev. trusted header's hash (h1) doesn't match hash from primary provider (h2)",
+		c.logger.Info("previous trusted header's hash (h1) doesn't match hash from primary provider (h2)",
 			"h1", c.latestTrustedBlock.Hash(), "h2", primaryHash)

 		action := fmt.Sprintf(
-			"Prev. trusted header's hash %X doesn't match hash %X from primary provider. Remove all the stored light blocks?",
+			"Previous trusted header's hash %X doesn't match hash %X from primary provider. Remove all the stored light blocks?",
 			c.latestTrustedBlock.Hash(), primaryHash)
 		if c.confirmationFn(action) {
 			err := c.Cleanup()
@@ -415,7 +417,7 @@ func (c *Client) Update(ctx context.Context, now time.Time) (*types.LightBlock,
 		if err != nil {
 			return nil, err
 		}
-		c.logger.Info("Advanced to new state", "height", latestBlock.Height, "hash", latestBlock.Hash())
+		c.logger.Info("advanced to new state", "height", latestBlock.Height, "hash", latestBlock.Hash())
 		return latestBlock, nil
 	}

@@ -440,7 +442,7 @@ func (c *Client) VerifyLightBlockAtHeight(ctx context.Context, height int64, now
 	// Check if the light block is already verified.
 	h, err := c.TrustedLightBlock(height)
 	if err == nil {
-		c.logger.Info("Header has already been verified", "height", height, "hash", h.Hash())
+		c.logger.Debug("header has already been verified", "height", height, "hash", h.Hash())
 		// Return already trusted light block
 		return h, nil
 	}
@@ -497,7 +499,7 @@ func (c *Client) VerifyHeader(ctx context.Context, newHeader *types.Header, now
 		if !bytes.Equal(l.Hash(), newHeader.Hash()) {
 			return fmt.Errorf("existing trusted header %X does not match newHeader %X", l.Hash(), newHeader.Hash())
 		}
-		c.logger.Info("Header has already been verified",
+		c.logger.Debug("header has already been verified",
 			"height", newHeader.Height, "hash", newHeader.Hash())
 		return nil
 	}
@@ -516,7 +518,7 @@ func (c *Client) VerifyHeader(ctx context.Context, newHeader *types.Header, now
 }

 func (c *Client) verifyLightBlock(ctx context.Context, newLightBlock *types.LightBlock, now time.Time) error {
-	c.logger.Info("VerifyHeader", "height", newLightBlock.Height, "hash", newLightBlock.Hash())
+	c.logger.Info("verify light block", "height", newLightBlock.Height, "hash", newLightBlock.Hash())

 	var (
 		verifyFunc func(ctx context.Context, trusted *types.LightBlock, new *types.LightBlock, now time.Time) error
@@ -561,7 +563,7 @@ func (c *Client) verifyLightBlock(ctx context.Context, newLightBlock *types.Ligh
 		err = verifyFunc(ctx, closestBlock, newLightBlock, now)
 	}
 	if err != nil {
-		c.logger.Error("Can't verify", "err", err)
+		c.logger.Error("failed to verify", "err", err)
 		return err
 	}

@@ -595,7 +597,7 @@ func (c *Client) verifySequential(
 		}

 		// 2) Verify them
-		c.logger.Debug("Verify adjacent newLightBlock against verifiedBlock",
+		c.logger.Debug("verify adjacent newLightBlock against verifiedBlock",
 			"trustedHeight", verifiedBlock.Height,
 			"trustedHash", verifiedBlock.Hash(),
 			"newHeight", interimBlock.Height,
@@ -610,32 +612,21 @@ func (c *Client) verifySequential(
 			case ErrInvalidHeader:
 				// If the target header is invalid, return immediately.
 				if err.To == newLightBlock.Height {
-					c.logger.Debug("Target header is invalid", "err", err)
+					c.logger.Debug("target header is invalid", "err", err)
 					return err
 				}

-				// If some intermediate header is invalid, replace the primary and try
-				// again.
-				c.logger.Error("primary sent invalid header -> replacing", "err", err, "primary", c.primary)
-				replaceErr := c.replacePrimaryProvider()
-				if replaceErr != nil {
-					c.logger.Error("Can't replace primary", "err", replaceErr)
-					// return original error
-					return err
-				}
+				// If some intermediate header is invalid, remove the primary and try again.
+				c.logger.Error("primary sent invalid header -> removing", "err", err, "primary", c.primary)

-				replacementBlock, fErr := c.lightBlockFromPrimary(ctx, newLightBlock.Height)
-				if fErr != nil {
-					c.logger.Error("Can't fetch light block from primary", "err", fErr)
-					// return original error
+				replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true)
+				if removeErr != nil {
+					c.logger.Debug("failed to replace primary. Returning original error", "err", removeErr)
 					return err
 				}

 				if !bytes.Equal(replacementBlock.Hash(), newLightBlock.Hash()) {
-					c.logger.Error("Replacement provider has a different light block",
-						"newHash", newLightBlock.Hash(),
-						"replHash", replacementBlock.Hash())
-					// return original error
+					c.logger.Debug("replaced primary but new primary has a different block to the initial one")
 					return err
 				}

@@ -686,7 +677,7 @@ func (c *Client) verifySkipping(
 	)

 	for {
-		c.logger.Debug("Verify non-adjacent newHeader against verifiedBlock",
+		c.logger.Debug("verify non-adjacent newHeader against verifiedBlock",
 			"trustedHeight", verifiedBlock.Height,
 			"trustedHash", verifiedBlock.Hash(),
 			"newHeight", blockCache[depth].Height,
@@ -716,10 +707,20 @@ func (c *Client) verifySkipping(
 				pivotHeight := verifiedBlock.Height + (blockCache[depth].Height-verifiedBlock.
 					Height)*verifySkippingNumerator/verifySkippingDenominator
 				interimBlock, providerErr := source.LightBlock(ctx, pivotHeight)
-				if providerErr != nil {
+				switch providerErr {
+				case nil:
+					blockCache = append(blockCache, interimBlock)
+
+				// if the error is benign, the client does not need to replace the primary
+				case provider.ErrLightBlockNotFound, provider.ErrNoResponse:
+					return nil, err
+
+				// all other errors such as ErrBadLightBlock or ErrUnreliableProvider are seen as malevolent and the
+				// provider is removed
+				default:
 					return nil, ErrVerificationFailed{From: verifiedBlock.Height, To: pivotHeight, Reason: providerErr}
 				}
-				blockCache = append(blockCache, interimBlock)
+
 			}
 			depth++

@@ -744,32 +745,20 @@ func (c *Client) verifySkippingAgainstPrimary(
 		// If the target header is invalid, return immediately.
 		invalidHeaderHeight := err.(ErrVerificationFailed).To
 		if invalidHeaderHeight == newLightBlock.Height {
-			c.logger.Debug("Target header is invalid", "err", err)
+			c.logger.Debug("target header is invalid", "err", err)
 			return err
 		}

-		// If some intermediate header is invalid, replace the primary and try
-		// again.
+		// If some intermediate header is invalid, remove the primary and try again.
 		c.logger.Error("primary sent invalid header -> replacing", "err", err, "primary", c.primary)
-		replaceErr := c.replacePrimaryProvider()
-		if replaceErr != nil {
-			c.logger.Error("Can't replace primary", "err", replaceErr)
-			// return original error
-			return err
-		}
-
-		replacementBlock, fErr := c.lightBlockFromPrimary(ctx, newLightBlock.Height)
-		if fErr != nil {
-			c.logger.Error("Can't fetch light block from primary", "err", fErr)
-			// return original error
+		replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true)
+		if removeErr != nil {
+			c.logger.Error("failed to replace primary. Returning original error", "err", removeErr)
 			return err
 		}

 		if !bytes.Equal(replacementBlock.Hash(), newLightBlock.Hash()) {
-			c.logger.Error("Replacement provider has a different light block",
-				"newHash", newLightBlock.Hash(),
-				"replHash", replacementBlock.Hash())
-			// return original error
+			c.logger.Debug("replaced primary but new primary has a different block to the initial one. Returning original error")
 			return err
 		}

@@ -835,7 +824,7 @@ func (c *Client) Witnesses() []provider.Provider {
 // Cleanup removes all the data (headers and validator sets) stored. Note: the
 // client must be stopped at this point.
 func (c *Client) Cleanup() error {
-	c.logger.Info("Removing all light blocks")
+	c.logger.Info("removing all light blocks")
 	c.latestTrustedBlock = nil
 	return c.trustedStore.Prune(0)
 }
@@ -908,21 +897,31 @@ func (c *Client) backwards(
 			return fmt.Errorf("failed to obtain the header at height #%d: %w", verifiedHeader.Height-1, err)
 		}
 		interimHeader = interimBlock.Header
-		c.logger.Debug("Verify newHeader against verifiedHeader",
+		c.logger.Debug("verify newHeader against verifiedHeader",
 			"trustedHeight", verifiedHeader.Height,
 			"trustedHash", verifiedHeader.Hash(),
 			"newHeight", interimHeader.Height,
 			"newHash", interimHeader.Hash())
 		if err := VerifyBackwards(interimHeader, verifiedHeader); err != nil {
-			c.logger.Error("primary sent invalid header -> replacing", "err", err, "primary", c.primary)
-			if replaceErr := c.replacePrimaryProvider(); replaceErr != nil {
-				c.logger.Error("Can't replace primary", "err", replaceErr)
-				// return original error
-				return fmt.Errorf("verify backwards from %d to %d failed: %w",
-					verifiedHeader.Height, interimHeader.Height, err)
+			// verification has failed
+			c.logger.Error("backwards verification failed, replacing primary...", "err", err, "primary", c.primary)
+
+			// the client tries to see if it can get a witness to continue with the request
+			newPrimarysBlock, replaceErr := c.findNewPrimary(ctx, newHeader.Height, true)
+			if replaceErr != nil {
+				c.logger.Debug("failed to replace primary. Returning original error", "err", replaceErr)
+				return err
 			}
-			// we need to verify the header at the same height again
-			continue
+
+			// before continuing we must check that they have the same target header to validate
+			if !bytes.Equal(newPrimarysBlock.Hash(), newHeader.Hash()) {
+				c.logger.Debug("replaced primary but new primary has a different block to the initial one")
+				// return the original error
+				return err
+			}
+
+			// try again with the new primary
+			return c.backwards(ctx, verifiedHeader, newPrimarysBlock.Header)
 		}
 		verifiedHeader = interimHeader
 	}
@@ -930,55 +929,145 @@ func (c *Client) backwards(
 	return nil
 }

-// NOTE: requires a providerMutex locked.
-func (c *Client) removeWitness(idx int) {
-	switch len(c.witnesses) {
-	case 0:
-		panic(fmt.Sprintf("wanted to remove %d element from empty witnesses slice", idx))
-	case 1:
-		c.witnesses = make([]provider.Provider, 0)
-	default:
-		c.witnesses[idx] = c.witnesses[len(c.witnesses)-1]
-		c.witnesses = c.witnesses[:len(c.witnesses)-1]
-	}
-}
-
-// replaceProvider takes the first alternative provider and promotes it as the
-// primary provider.
-func (c *Client) replacePrimaryProvider() error {
-	c.providerMutex.Lock()
-	defer c.providerMutex.Unlock()
-
-	if len(c.witnesses) <= 1 {
-		return ErrNoWitnesses
-	}
-	c.primary = c.witnesses[0]
-	c.witnesses = c.witnesses[1:]
-	c.logger.Info("Replacing primary with the first witness", "new_primary", c.primary)
-
-	return nil
-}
-
 // lightBlockFromPrimary retrieves the lightBlock from the primary provider
-// at the specified height. Handles dropout by the primary provider by swapping
-// with an alternative provider.
+// at the specified height. This method also handles provider behavior as follows:
+//
+// 1. If the provider does not respond or does not have the block, it tries again
+//    with a different provider
+// 2. If all providers return the same error, the light client forwards the error to
+//    where the initial request came from
+// 3. If the provider provides an invalid light block, is deemed unreliable or returns
+//    any other error, the primary is permanently dropped and is replaced by a witness.
 func (c *Client) lightBlockFromPrimary(ctx context.Context, height int64) (*types.LightBlock, error) {
 	c.providerMutex.Lock()
 	l, err := c.primary.LightBlock(ctx, height)
 	c.providerMutex.Unlock()
-	if err != nil {
-		c.logger.Debug("Error on light block request from primary", "error", err, "primary", c.primary)
-		replaceErr := c.replacePrimaryProvider()
-		if replaceErr != nil {
-			return nil, fmt.Errorf("%v. Tried to replace primary but: %w", err.Error(), replaceErr)
-		}
-		// replace primary and request a light block again
-		return c.lightBlockFromPrimary(ctx, height)
+
+	switch err {
+	case nil:
+		// Everything went smoothly. We reset the lightBlockRequests and return the light block
+		return l, nil
+
+	case provider.ErrNoResponse, provider.ErrLightBlockNotFound:
+		// we find a new witness to replace the primary
+		c.logger.Debug("error from light block request from primary, replacing...", "error", err, "primary", c.primary)
+		return c.findNewPrimary(ctx, height, false)
+
+	default:
+		// The light client has most likely received either provider.ErrUnreliableProvider or provider.ErrBadLightBlock
+		// These errors mean that the light client should drop the primary and try with another provider instead
+		c.logger.Error("error from light block request from primary, removing...", "error", err, "primary", c.primary)
+		return c.findNewPrimary(ctx, height, true)
 	}
-	return l, err
 }

-// compareFirstHeaderWithWitnesses compares h with all witnesses. If any
+// NOTE: requires a providerMutex lock
+func (c *Client) removeWitnesses(indexes []int) error {
+	// check that we will still have witnesses remaining
+	if len(c.witnesses) <= len(indexes) {
+		return ErrNoWitnesses
+	}
+
+	// we need to make sure that we remove witnesses by index in the reverse
+	// order so as to not affect the indexes themselves
+	sort.Ints(indexes)
+	for i := len(indexes) - 1; i >= 0; i-- {
+		c.witnesses[indexes[i]] = c.witnesses[len(c.witnesses)-1]
+		c.witnesses = c.witnesses[:len(c.witnesses)-1]
+	}
+
+	return nil
+}
+
+type witnessResponse struct {
+	lb           *types.LightBlock
+	witnessIndex int
+	err          error
+}
+
+// findNewPrimary concurrently sends a light block request, promoting the first witness to return
+// a valid light block as the new primary. The remove option indicates whether the primary should be
+// entire removed or just appended to the back of the witnesses list. This method also handles witness
+// errors. If no witness is available, it returns the last error of the witness.
+func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) (*types.LightBlock, error) {
+	c.providerMutex.Lock()
+	defer c.providerMutex.Unlock()
+
+	if len(c.witnesses) <= 1 {
+		return nil, ErrNoWitnesses
+	}
+
+	var (
+		witnessResponsesC = make(chan witnessResponse, len(c.witnesses))
+		witnessesToRemove []int
+		lastError         error
+		wg                sync.WaitGroup
+	)
+
+	// send out a light block request to all witnesses
+	subctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	for index := range c.witnesses {
+		wg.Add(1)
+		go func(witnessIndex int, witnessResponsesC chan witnessResponse) {
+			defer wg.Done()
+
+			lb, err := c.witnesses[witnessIndex].LightBlock(subctx, height)
+			witnessResponsesC <- witnessResponse{lb, witnessIndex, err}
+		}(index, witnessResponsesC)
+	}
+
+	// process all the responses as they come in
+	for i := 0; i < cap(witnessResponsesC); i++ {
+		response := <-witnessResponsesC
+		switch response.err {
+		// success! We have found a new primary
+		case nil:
+			cancel() // cancel all remaining requests to other witnesses
+
+			wg.Wait() // wait for all goroutines to finish
+
+			// if we are not intending on removing the primary then append the old primary to the end of the witness slice
+			if !remove {
+				c.witnesses = append(c.witnesses, c.primary)
+			}
+
+			// promote respondent as the new primary
+			c.logger.Debug("found new primary", "primary", c.witnesses[response.witnessIndex])
+			c.primary = c.witnesses[response.witnessIndex]
+
+			// add promoted witness to the list of witnesses to be removed
+			witnessesToRemove = append(witnessesToRemove, response.witnessIndex)
+
+			// remove witnesses marked as bad (the client must do this before we alter the witness slice and change the indexes
+			// of witnesses). Removal is done in descending order
+			if err := c.removeWitnesses(witnessesToRemove); err != nil {
+				return nil, err
+			}
+
+			// return the light block that new primary responded with
+			return response.lb, nil
+
+		// process benign errors by logging them only
+		case provider.ErrNoResponse, provider.ErrLightBlockNotFound:
+			lastError = response.err
+			c.logger.Debug("error on light block request from witness",
+				"error", response.err, "primary", c.witnesses[response.witnessIndex])
+			continue
+
+		// process malevolent errors like ErrUnreliableProvider and ErrBadLightBlock by removing the witness
+		default:
+			lastError = response.err
+			c.logger.Error("error on light block request from witness, removing...",
+				"error", response.err, "primary", c.witnesses[response.witnessIndex])
+			witnessesToRemove = append(witnessesToRemove, response.witnessIndex)
+		}
+	}
+
+	return nil, lastError
+}
+
+// compareFirstHeaderWithWitnesses concurrently compares h with all witnesses. If any
 // witness reports a different header than h, the function returns an error.
 func (c *Client) compareFirstHeaderWithWitnesses(ctx context.Context, h *types.SignedHeader) error {
 	compareCtx, cancel := context.WithCancel(ctx)
@@ -1006,26 +1095,20 @@ func (c *Client) compareFirstHeaderWithWitnesses(ctx context.Context, h *types.S
 		case nil:
 			continue
 		case errConflictingHeaders:
-			c.logger.Error(fmt.Sprintf(`Witness #%d has a different header. Please check primary is correct
-and remove witness. Otherwise, use the different primary`, e.WitnessIndex), "witness", c.witnesses[e.WitnessIndex])
+			c.logger.Error(fmt.Sprintf(`witness #%d has a different header. Please check primary is correct
+and remove witness. Otherwise, use a different primary`, e.WitnessIndex), "witness", c.witnesses[e.WitnessIndex])
 			return err
 		case errBadWitness:
 			// If witness sent us an invalid header, then remove it. If it didn't
 			// respond or couldn't find the block, then we ignore it and move on to
 			// the next witness.
 			if _, ok := e.Reason.(provider.ErrBadLightBlock); ok {
-				c.logger.Info("Witness sent us invalid header / vals -> removing it", "witness", c.witnesses[e.WitnessIndex])
+				c.logger.Info("Witness sent an invalid light block, removing...", "witness", c.witnesses[e.WitnessIndex])
 				witnessesToRemove = append(witnessesToRemove, e.WitnessIndex)
 			}
 		}
 	}

-	// we need to make sure that we remove witnesses by index in the reverse
-	// order so as to not affect the indexes themselves
-	sort.Ints(witnessesToRemove)
-	for i := len(witnessesToRemove) - 1; i >= 0; i-- {
-		c.removeWitness(witnessesToRemove[i])
-	}
-
-	return nil
+	// remove all witnesses that misbehaved
+	return c.removeWitnesses(witnessesToRemove)
 }