Bump Spec11 transient attempts to 9 (#3217)

We're still getting 429 errors after increasing the number of retries to
four. Using 9 means that the maximum number of seconds we wait will be
255 -- 8 wait intervals starting at 1 second means
1+2+4+8+16+32+64+128=255 seconds.

Also log to see if they give us a useful retry-after header

core/src/main/java/google/registry/beam/spec11/SafeBrowsingTransforms.java

@@ -15,7 +15,9 @@
 package google.registry.beam.spec11;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.http.HttpHeaders.RETRY_AFTER;
 import static org.apache.http.HttpStatus.SC_OK;
+import static org.apache.http.HttpStatus.SC_TOO_MANY_REQUESTS;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
@@ -33,6 +35,7 @@ import java.util.function.Supplier;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindow;
 import org.apache.beam.sdk.values.KV;
+import org.apache.http.Header;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.utils.URIBuilder;
@@ -229,6 +232,14 @@ public class SafeBrowsingTransforms {
         throws IOException {
       int statusCode = response.getStatusLine().getStatusCode();
       if (statusCode != SC_OK) {
+        if (statusCode == SC_TOO_MANY_REQUESTS) {
+          Header retryAfterHeader = response.getFirstHeader(RETRY_AFTER);
+          if (retryAfterHeader != null) {
+            logger.atWarning().log(
+                "SafeBrowsing API returned 429 with Retry-After header: %s",
+                retryAfterHeader.getValue());
+          }
+        }
         throw new IOException(
             String.format("Got unexpected status code %s from response.", statusCode));
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -241,7 +241,7 @@ public class Spec11Pipeline implements Serializable {
     EvaluateSafeBrowsingFn provideSafeBrowsingFn(
         Spec11PipelineOptions options, Clock clock, Sleeper sleeper) {
       // Have a noticeably longer backoff for SafeBrowsing retries to mitigate any 429s
-      Retrier safeBrowsingRetrier = new Retrier(sleeper, 4, 1000L);
+      Retrier safeBrowsingRetrier = new Retrier(sleeper, 9, 1000L);
       return new EvaluateSafeBrowsingFn(
           options.getSafeBrowsingApiKey(), safeBrowsingRetrier, clock);
     }