Allow longer retries in Retrier for Spec11 (#3209)

We're getting some 429 (too many requests) responses from the
SafeBrowsing API which is causing the Spec11 pipeline to fail. Currently
the retrier is set to have a backoff starting at 100ms and only have a
couple retries before failing (the latter is already configurable). For
429s, we want to wait significantly longer. Let's start at one second of
waiting and allow four doublings.
This commit is contained in:
gbrodman
2026-08-14 20:15:30 +00:00
committed by GitHub
parent 84811d5624
commit 1423474fb1
4 changed files with 33 additions and 11 deletions
@@ -19,6 +19,7 @@ import static com.google.common.base.Throwables.throwIfUnchecked;
import static com.google.common.math.IntMath.pow;
import static google.registry.util.PredicateUtils.supertypeOf;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.common.flogger.FluentLogger;
import jakarta.inject.Inject;
@@ -41,6 +42,7 @@ public class Retrier implements Serializable {
private final Sleeper sleeper;
private final int attempts;
private final long baseIntervalMillis;
/** Holds functions to call whenever the code being retried fails. */
public interface FailureReporter {
@@ -55,11 +57,21 @@ public class Retrier implements Serializable {
void beforeRetry(Throwable thrown, int failures, int maxAttempts);
}
@VisibleForTesting
public Retrier(Sleeper sleeper, int transientFailureRetries) {
this(sleeper, transientFailureRetries, 100L);
}
@Inject
public Retrier(Sleeper sleeper, @Named("transientFailureRetries") int transientFailureRetries) {
public Retrier(
Sleeper sleeper,
@Named("transientFailureRetries") int transientFailureRetries,
@Named("transientFailureBaseIntervalMillis") long baseIntervalMillis) {
this.sleeper = sleeper;
checkArgument(transientFailureRetries > 0, "Number of attempts must be positive");
this.attempts = transientFailureRetries;
checkArgument(baseIntervalMillis > 0, "Base interval millis must be positive");
this.baseIntervalMillis = baseIntervalMillis;
}
/**
@@ -160,8 +172,8 @@ public class Retrier implements Serializable {
throw new RuntimeException(e);
}
failureReporter.beforeRetry(e, failures, attempts);
// Wait (skewed) 100ms on the first attempt, doubling on each subsequent attempt.
long backoffMillis = pow(2, failures) * 100L;
// Wait (skewed) baseIntervalMillis on the first attempt, doubling on each attempt
long backoffMillis = pow(2, failures) * baseIntervalMillis;
long sleepDurationMillis = Math.round(randomForSkew.nextDouble(0.8, 1.2) * backoffMillis);
try {
sleeper.sleep(Duration.ofMillis(sleepDurationMillis));