seaweedfs/weed/util/retry.go

package util

import (
	"context"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
)

var RetryWaitTime = 6 * time.Second

func Retry(name string, job func() error) (err error) {
	waitTime := time.Second
	hasErr := false
	for waitTime < RetryWaitTime {
		err = job()
		if err == nil {
			if hasErr {
				glog.V(0).Infof("retry %s successfully", name)
			}
			waitTime = time.Second
			break
		}
		if strings.Contains(err.Error(), "transport") {
			hasErr = true
			glog.V(0).Infof("retry %s: err: %v", name, err)
		} else {
			break
		}
		time.Sleep(waitTime)
		waitTime += waitTime / 2
	}
	return err
}

func MultiRetry(name string, errList []string, job func() error) (err error) {
	waitTime := time.Second
	hasErr := false
	for waitTime < RetryWaitTime {
		err = job()
		if err == nil {
			if hasErr {
				glog.V(0).Infof("retry %s successfully", name)
			}
			waitTime = time.Second
			break
		}
		if containErr(err.Error(), errList) {
			hasErr = true
			glog.V(0).Infof("retry %s: err: %v", name, err)
		} else {
			break
		}
		time.Sleep(waitTime)
		waitTime += waitTime / 2
	}
	return err
}

// RetryOnError retries job with the same bounded backoff as MultiRetry, but
// decides retriability with a predicate instead of an error-substring list.
func RetryOnError(name string, shouldRetry func(error) bool, job func() error) (err error) {
	waitTime := time.Second
	hasErr := false
	for waitTime < RetryWaitTime {
		err = job()
		if err == nil {
			if hasErr {
				glog.V(0).Infof("retry %s successfully", name)
			}
			break
		}
		if shouldRetry(err) {
			hasErr = true
			glog.V(0).Infof("retry %s: err: %v", name, err)
		} else {
			break
		}
		time.Sleep(waitTime)
		waitTime += waitTime / 2
	}
	return err
}

// RetryUntil retries until the job returns no error or onErrFn returns false
func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) error {
	waitTime := time.Second
	for {
		err := job()
		if err == nil {
			waitTime = time.Second
			return nil
		}
		if onErrFn(err) {
			if strings.Contains(err.Error(), "transport") || strings.Contains(err.Error(), "ResourceExhausted") || strings.Contains(err.Error(), "Unavailable") {
				glog.V(0).Infof("retry %s: err: %v", name, err)
			}
			time.Sleep(waitTime)
			if waitTime < RetryWaitTime {
				waitTime += waitTime / 2
			}
			continue
		} else {
			return err
		}
	}
}

// RetryWithBackoff retries an operation on codes.Unavailable errors with exponential
// backoff, respecting context cancellation and a maximum retry duration.
// Returns nil on success, ctx.Err() on context cancellation, or the last error
// when maxDuration is exceeded or a non-retriable error occurs.
func RetryWithBackoff(ctx context.Context, name string, maxDuration time.Duration, shouldRetry func(error) bool, operation func() error) error {
	waitTime := time.Second
	maxWaitTime := RetryWaitTime
	deadline := time.Now().Add(maxDuration)
	var lastErr error
	for {
		if ctx.Err() != nil {
			return ctx.Err()
		}
		if time.Until(deadline) <= 0 {
			if lastErr != nil {
				glog.V(0).Infof("retry %s: giving up after %v: %v", name, maxDuration, lastErr)
				return lastErr
			}
		}
		err := operation()
		if err == nil {
			return nil
		}
		lastErr = err
		if !shouldRetry(err) {
			return err
		}
		remaining := time.Until(deadline)
		if remaining <= 0 {
			glog.V(0).Infof("retry %s: giving up after %v: %v", name, maxDuration, err)
			return err
		}
		sleepTime := waitTime
		if sleepTime > maxWaitTime {
			sleepTime = maxWaitTime
		}
		if sleepTime > remaining {
			sleepTime = remaining
		}
		glog.V(1).Infof("retry %s: retrying in %v: %v", name, sleepTime, err)
		timer := time.NewTimer(sleepTime)
		select {
		case <-ctx.Done():
			if !timer.Stop() {
				<-timer.C
			}
			return ctx.Err()
		case <-timer.C:
		}
		waitTime += waitTime / 2
		if waitTime > maxWaitTime {
			waitTime = maxWaitTime
		}
	}
}

// Nvl return the first non-empty string
func Nvl(values ...string) string {
	for _, s := range values {
		if s != "" {
			return s
		}
	}
	return ""
}

func containErr(err string, errList []string) bool {
	for _, e := range errList {
		if strings.Contains(err, e) {
			return true
		}
	}
	return false
}