Revise host.inet_addresses query to use gin index (#1550)

* Revise host.inet_addresses query to use gin index

build.gradle

@@ -55,7 +55,7 @@ plugins {
 
 node {
   download = true
-  version = "14.15.5"
+  version = "16.14.0"
   npmVersion = "6.14.11"
 }
 

core/src/main/java/google/registry/backup/SyncDatastoreToSqlSnapshotAction.java

@@ -29,6 +29,8 @@ import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Sleeper;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -114,9 +116,22 @@ public class SyncDatastoreToSqlSnapshotAction implements Runnable {
       response.setPayload(
           String.format(SUCCESS_RESPONSE_TEMPLATE, sqlSnapshotId, checkpoint.getCheckpointTime()));
       return;
-    } catch (Exception e) {
+    } catch (Throwable e) {
+      logger.atSevere().withCause(e).log("Failed to sync Datastore to SQL.");
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setPayload(e.getMessage());
+      response.setPayload(getStackTrace(e));
+    }
+  }
+
+  private static String getStackTrace(Throwable e) {
+    try {
+      ByteArrayOutputStream bis = new ByteArrayOutputStream();
+      PrintStream printStream = new PrintStream(bis);
+      e.printStackTrace(printStream);
+      printStream.close();
+      return bis.toString();
+    } catch (RuntimeException re) {
+      return re.getMessage();
     }
   }
 

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -22,14 +22,17 @@ import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Multimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
-import google.registry.util.AppEngineServiceUtils;
+import google.registry.request.Action.Service;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.Retrier;
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -58,25 +61,23 @@ public final class AsyncTaskEnqueuer {
   private static final Duration MAX_ASYNC_ETA = Duration.standardDays(30);
 
   private final Duration asyncDeleteDelay;
-  private final Queue asyncActionsPushQueue;
   private final Queue asyncDeletePullQueue;
   private final Queue asyncDnsRefreshPullQueue;
-  private final AppEngineServiceUtils appEngineServiceUtils;
   private final Retrier retrier;
 
+  private CloudTasksUtils cloudTasksUtils;
+
   @Inject
   public AsyncTaskEnqueuer(
-      @Named(QUEUE_ASYNC_ACTIONS) Queue asyncActionsPushQueue,
       @Named(QUEUE_ASYNC_DELETE) Queue asyncDeletePullQueue,
       @Named(QUEUE_ASYNC_HOST_RENAME) Queue asyncDnsRefreshPullQueue,
       @Config("asyncDeleteFlowMapreduceDelay") Duration asyncDeleteDelay,
-      AppEngineServiceUtils appEngineServiceUtils,
+      CloudTasksUtils cloudTasksUtils,
       Retrier retrier) {
-    this.asyncActionsPushQueue = asyncActionsPushQueue;
     this.asyncDeletePullQueue = asyncDeletePullQueue;
     this.asyncDnsRefreshPullQueue = asyncDnsRefreshPullQueue;
     this.asyncDeleteDelay = asyncDeleteDelay;
-    this.appEngineServiceUtils = appEngineServiceUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
     this.retrier = retrier;
   }
 
@@ -102,19 +103,17 @@ public final class AsyncTaskEnqueuer {
           entityKey, firstResave, MAX_ASYNC_ETA);
       return;
     }
-    logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
-    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
-    TaskOptions task =
-        TaskOptions.Builder.withUrl(ResaveEntityAction.PATH)
-            .method(Method.POST)
-            .header("Host", backendHostname)
-            .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.stringify())
-            .param(PARAM_REQUESTED_TIME, now.toString());
+    Multimap<String, String> params = ArrayListMultimap.create();
+    params.put(PARAM_RESOURCE_KEY, entityKey.stringify());
+    params.put(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
-      task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
+      params.put(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
     }
-    addTaskToQueueWithRetry(asyncActionsPushQueue, task);
+    logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
+    cloudTasksUtils.enqueue(
+        QUEUE_ASYNC_ACTIONS,
+        cloudTasksUtils.createPostTaskWithDelay(
+            ResaveEntityAction.PATH, Service.BACKEND.toString(), params, etaDuration));
   }
 
   /** Enqueues a task to asynchronously delete a contact or host, by key. */

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -301,7 +301,6 @@ public class DomainFlowUtils {
             String.format(
                 "A maximum of %s DS records are allowed per domain.", MAX_DS_RECORDS_PER_DOMAIN));
       }
-      // TODO(sarahbot@): Add signature length verification
       ImmutableList<DelegationSignerData> invalidAlgorithms =
           dsData.stream()
               .filter(ds -> !validateAlgorithm(ds.getAlgorithm()))
@@ -322,6 +321,20 @@ public class DomainFlowUtils {
                 "Domain contains DS record(s) with an invalid digest type: %s",
                 invalidDigestTypes));
       }
+      ImmutableList<DelegationSignerData> digestsWithInvalidDigestLength =
+          dsData.stream()
+              .filter(
+                  ds ->
+                      DigestType.fromWireValue(ds.getDigestType()).isPresent()
+                          && (ds.getDigest().length
+                              != DigestType.fromWireValue(ds.getDigestType()).get().getBytes()))
+              .collect(toImmutableList());
+      if (!digestsWithInvalidDigestLength.isEmpty()) {
+        throw new InvalidDsRecordException(
+            String.format(
+                "Domain contains DS record(s) with an invalid digest length: %s",
+                digestsWithInvalidDigestLength));
+      }
     }
   }
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -36,6 +36,17 @@ import javax.persistence.AccessType;
 @javax.persistence.Entity(name = "Host")
 @javax.persistence.Table(
     name = "Host",
+    /**
+     * A gin index defined on the inet_addresses field ({@link HostBase#inetAddresses} cannot be
+     * declared here because JPA/Hibernate does not support index type specification. As a result,
+     * the hibernate-generated schema (which is for reference only) does not have this index.
+     *
+     * <p>There are Hibernate-specific solutions for adding this index to Hibernate's domain model.
+     * We could either declare the index in hibernate.cfg.xml or add it to the {@link
+     * org.hibernate.cfg.Configuration} instance for {@link SessionFactory} instantiation (which
+     * would prevent us from using JPA standard bootstrapping). For now, there is no obvious benefit
+     * doing either.
+     */
     indexes = {@javax.persistence.Index(columnList = "hostName")})
 @ExternalMessagingName("host")
 @WithStringVKey

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -127,15 +127,16 @@ public class ReplicateToDatastoreAction implements Runnable {
                 // Reload the last transaction id, which could possibly have changed.
                 LastSqlTransaction lastSqlTxn = LastSqlTransaction.load();
                 long nextTxnId = lastSqlTxn.getTransactionId() + 1;
-                if (nextTxnId < txnEntity.getId()) {
-                  // We're missing a transaction.  This is bad.  Transaction ids are supposed to
-                  // increase monotonically, so we abort rather than applying anything out of
-                  // order.
-                  throw new IllegalStateException(
-                      String.format(
-                          "Missing transaction: last txn id = %s, next available txn = %s",
-                          nextTxnId - 1, txnEntity.getId()));
-                } else if (nextTxnId > txnEntity.getId()) {
+
+                // Skip missing transactions.  Missed transactions can happen normally.  If a
+                // transaction gets rolled back, the sequence counter doesn't.
+                while (nextTxnId < txnEntity.getId()) {
+                  logger.atWarning().log(
+                      "Ignoring transaction %s, which does not exist.", nextTxnId);
+                  ++nextTxnId;
+                }
+
+                if (nextTxnId > txnEntity.getId()) {
                   // We've already replayed this transaction.  This shouldn't happen, as GAE cron
                   // is supposed to avoid overruns and this action shouldn't be executed from any
                   // other context, but it's not harmful as we can just ignore the transaction.  Log

core/src/main/java/google/registry/module/backend/BackendComponent.java

@@ -43,7 +43,7 @@ import google.registry.rde.JSchModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.URLFetchServiceModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
@@ -80,7 +80,7 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       SheetsServiceModule.class,
       StackdriverModule.class,
-      URLFetchServiceModule.class,
+      UrlConnectionServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
       VoidDnsWriterModule.class,

core/src/main/java/google/registry/module/frontend/FrontendComponent.java

@@ -34,7 +34,6 @@ import google.registry.monitoring.whitebox.StackdriverModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.ui.ConsoleDebug.ConsoleConfigModule;
@@ -65,7 +64,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -34,7 +34,6 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -62,7 +61,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/tools/ToolsComponent.java

@@ -36,7 +36,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -66,7 +65,6 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       StackdriverModule.class,
       ToolsRequestComponentModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java

@@ -14,6 +14,7 @@
 
 package google.registry.persistence.transaction;
 
+import com.google.common.annotations.VisibleForTesting;
 import google.registry.model.ImmutableObject;
 import google.registry.model.replay.SqlOnlyEntity;
 import javax.persistence.Entity;
@@ -39,7 +40,8 @@ public class TransactionEntity extends ImmutableObject implements SqlOnlyEntity
 
   TransactionEntity() {}
 
-  TransactionEntity(byte[] contents) {
+  @VisibleForTesting
+  public TransactionEntity(byte[] contents) {
     this.contents = contents;
   }
 

core/src/main/java/google/registry/rde/RdeReporter.java

@@ -14,26 +14,24 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
-import static com.google.common.io.BaseEncoding.base64;
-import static com.google.common.net.HttpHeaders.AUTHORIZATION;
-import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
+import static google.registry.request.UrlConnectionUtils.setPayload;
 import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
+import com.google.api.client.http.HttpMethods;
 import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.request.UrlConnectionService;
 import google.registry.util.Retrier;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
 import google.registry.xjc.iirdea.XjcIirdeaResult;
@@ -41,6 +39,7 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.SocketTimeoutException;
 import java.net.URL;
@@ -55,12 +54,15 @@ public class RdeReporter {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  /** @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
-   *     ICANN Registry Interfaces - Interface details</a>*/
-  private static final String REPORT_MIME = "text/xml";
+  /**
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
+   *     ICANN Registry Interfaces - Interface details</a>
+   */
+  private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
 
   @Inject Retrier retrier;
-  @Inject URLFetchService urlFetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
   @Inject @Key("icannReportingPassword") String password;
   @Inject RdeReporter() {}
@@ -74,29 +76,24 @@ public class RdeReporter {
     // Send a PUT request to ICANN's HTTPS server.
     URL url = makeReportUrl(header.getTld(), report.getId());
     String username = header.getTld() + "_ry";
-    String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
-    final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
-    req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
-    req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    req.setPayload(reportBytes);
     logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
-    HTTPResponse rsp =
+    byte[] responseBytes =
         retrier.callWithRetry(
             () -> {
-              HTTPResponse rsp1 = urlFetchService.fetch(req);
-              switch (rsp1.getResponseCode()) {
-                case SC_OK:
-                case SC_BAD_REQUEST:
-                  break;
-                default:
-                  throw new UrlFetchException("PUT failed", req, rsp1);
+              HttpURLConnection connection = urlConnectionService.createConnection(url);
+              connection.setRequestMethod(HttpMethods.PUT);
+              setBasicAuth(connection, username, password);
+              setPayload(connection, reportBytes, MEDIA_TYPE.toString());
+              int responseCode = connection.getResponseCode();
+              if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) {
+                return getResponseBytes(connection);
               }
-              return rsp1;
+              throw new UrlConnectionException("PUT failed", connection);
             },
             SocketTimeoutException.class);
 
     // Ensure the XML response is valid.
-    XjcIirdeaResult result = parseResult(rsp);
+    XjcIirdeaResult result = parseResult(responseBytes);
     if (result.getCode().getValue() != 1000) {
       logger.atWarning().log(
           "PUT rejected: %d %s\n%s",
@@ -108,11 +105,11 @@ public class RdeReporter {
   /**
    * Unmarshals IIRDEA XML result object from {@link HTTPResponse} payload.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
+   * @see <a
+   *     href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
    *     ICANN Registry Interfaces - IIRDEA Result Object</a>
    */
-  private XjcIirdeaResult parseResult(HTTPResponse rsp) throws XmlException {
-    byte[] responseBytes = rsp.getContent();
+  private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException {
     logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8));
     XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal(
         XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes));

core/src/main/java/google/registry/request/Modules.java

@@ -23,12 +23,11 @@ import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.appengine.api.datastore.DatastoreService;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
 import com.google.appengine.api.users.UserService;
 import com.google.appengine.api.users.UserServiceFactory;
 import dagger.Module;
 import dagger.Provides;
+import java.net.HttpURLConnection;
 import javax.inject.Singleton;
 
 /** Dagger modules for App Engine services and other vendor classes. */
@@ -45,14 +44,12 @@ public final class Modules {
     }
   }
 
-  /** Dagger module for {@link URLFetchService}. */
+  /** Dagger module for {@link UrlConnectionService}. */
   @Module
-  public static final class URLFetchServiceModule {
-    private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService();
-
+  public static final class UrlConnectionServiceModule {
     @Provides
-    static URLFetchService provideURLFetchService() {
-      return fetchService;
+    static UrlConnectionService provideUrlConnectionService() {
+      return url -> (HttpURLConnection) url.openConnection();
     }
   }
 

core/src/main/java/google/registry/request/UrlConnectionService.java

@@ -0,0 +1,25 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.request;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/** Functional interface for opening a connection from a URL, injectable for testing. */
+public interface UrlConnectionService {
+
+  HttpURLConnection createConnection(URL url) throws IOException;
+}

core/src/main/java/google/registry/request/UrlConnectionUtils.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.io.BaseEncoding.base64;
@@ -22,36 +22,41 @@ import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.common.base.Ascii;
 import com.google.common.base.Strings;
+import com.google.common.io.ByteStreams;
 import com.google.common.net.MediaType;
-import java.util.Optional;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.URLConnection;
 import java.util.Random;
 
-/** Helper methods for the App Engine URL fetch service. */
-public final class UrlFetchUtils {
+/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */
+public class UrlConnectionUtils {
 
-  /** Returns value of first header matching {@code name}. */
-  public static Optional<String> getHeaderFirst(HTTPResponse rsp, String name) {
-    return getHeaderFirstInternal(rsp.getHeadersUncombined(), name);
+  /** Retrieves the response from the given connection as a byte array. */
+  public static byte[] getResponseBytes(URLConnection connection) throws IOException {
+    return ByteStreams.toByteArray(connection.getInputStream());
   }
 
-  /** Returns value of first header matching {@code name}. */
-  public static Optional<String> getHeaderFirst(HTTPRequest req, String name) {
-    return getHeaderFirstInternal(req.getHeaders(), name);
+  /** Sets auth on the given connection with the given username/password. */
+  public static void setBasicAuth(URLConnection connection, String username, String password) {
+    setBasicAuth(connection, String.format("%s:%s", username, password));
   }
 
-  private static Optional<String> getHeaderFirstInternal(Iterable<HTTPHeader> hdrs, String name) {
-    name = Ascii.toLowerCase(name);
-    for (HTTPHeader header : hdrs) {
-      if (Ascii.toLowerCase(header.getName()).equals(name)) {
-        return Optional.of(header.getValue());
-      }
+  /** Sets auth on the given connection with the given string, formatted "username:password". */
+  public static void setBasicAuth(URLConnection connection, String usernameAndPassword) {
+    String token = base64().encode(usernameAndPassword.getBytes(UTF_8));
+    connection.setRequestProperty(AUTHORIZATION, "Basic " + token);
+  }
+
+  /** Sets the given byte[] payload on the given connection with a particular content type. */
+  public static void setPayload(URLConnection connection, byte[] bytes, String contentType)
+      throws IOException {
+    connection.setRequestProperty(CONTENT_TYPE, contentType);
+    connection.setDoOutput(true);
+    try (DataOutputStream dataStream = new DataOutputStream(connection.getOutputStream())) {
+      dataStream.write(bytes);
     }
-    return Optional.empty();
   }
 
   /**
@@ -62,16 +67,16 @@ public final class UrlFetchUtils {
    * @see <a href="http://www.ietf.org/rfc/rfc2388.txt">RFC2388 - Returning Values from Forms</a>
    */
   public static void setPayloadMultipart(
-      HTTPRequest request,
+      URLConnection connection,
       String name,
       String filename,
       MediaType contentType,
       String data,
-      Random random) {
+      Random random)
+      throws IOException {
     String boundary = createMultipartBoundary(random);
     checkState(
-        !data.contains(boundary),
-        "Multipart data contains autogenerated boundary: %s", boundary);
+        !data.contains(boundary), "Multipart data contains autogenerated boundary: %s", boundary);
     String multipart =
         String.format("--%s\r\n", boundary)
             + String.format(
@@ -83,11 +88,9 @@ public final class UrlFetchUtils {
             + "\r\n"
             + String.format("--%s--\r\n", boundary);
     byte[] payload = multipart.getBytes(UTF_8);
-    request.addHeader(
-        new HTTPHeader(
-            CONTENT_TYPE, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary)));
-    request.addHeader(new HTTPHeader(CONTENT_LENGTH, Integer.toString(payload.length)));
-    request.setPayload(payload);
+    connection.setRequestProperty(CONTENT_LENGTH, Integer.toString(payload.length));
+    setPayload(
+        connection, payload, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary));
   }
 
   private static String createMultipartBoundary(Random random) {
@@ -98,12 +101,4 @@ public final class UrlFetchUtils {
     // See https://tools.ietf.org/html/rfc2046#section-5.1.1
     return Strings.repeat("-", 30) + base64().encode(rand);
   }
-
-  /** Sets the HTTP Basic Authentication header on an {@link HTTPRequest}. */
-  public static void setAuthorizationHeader(HTTPRequest req, Optional<String> login) {
-    if (login.isPresent()) {
-      String token = base64().encode(login.get().getBytes(UTF_8));
-      req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    }
-  }
 }

core/src/main/java/google/registry/tmch/LordnRequestInitializer.java

@@ -15,12 +15,12 @@
 package google.registry.tmch;
 
 import static com.google.common.base.Verify.verifyNotNull;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
 import com.google.common.flogger.FluentLogger;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.tld.Registry;
+import google.registry.request.UrlConnectionUtils;
+import java.net.HttpURLConnection;
 import java.util.Optional;
 import javax.inject.Inject;
 
@@ -37,8 +37,9 @@ final class LordnRequestInitializer {
   }
 
   /** Initializes a URL fetch request for talking to the MarksDB server. */
-  void initialize(HTTPRequest request, String tld) {
-    setAuthorizationHeader(request, getMarksDbLordnCredentials(tld));
+  void initialize(HttpURLConnection connection, String tld) {
+    getMarksDbLordnCredentials(tld)
+        .ifPresent(login -> UrlConnectionUtils.setBasicAuth(connection, login));
   }
 
   /** Returns the username and password for the current TLD to login to the MarksDB server. */

core/src/main/java/google/registry/tmch/Marksdb.java

@@ -14,25 +14,23 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
 import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
 import static google.registry.util.HexDumper.dumpHex;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
-import google.registry.util.UrlFetchException;
+import google.registry.request.UrlConnectionService;
+import google.registry.util.UrlConnectionException;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.Security;
 import java.security.SignatureException;
@@ -57,7 +55,8 @@ public final class Marksdb {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final int MAX_DNL_LOGGING_LENGTH = 500;
 
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Key("marksdbPublicKey") PGPPublicKey marksdbPublicKey;
   @Inject Marksdb() {}
@@ -112,19 +111,16 @@ public final class Marksdb {
   }
 
   byte[] fetch(URL url, Optional<String> loginAndPassword) throws IOException {
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
-    setAuthorizationHeader(req, loginAndPassword);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
+    loginAndPassword.ifPresent(auth -> setBasicAuth(connection, auth));
     try {
-      rsp = fetchService.fetch(req);
+      if (connection.getResponseCode() != SC_OK) {
+        throw new UrlConnectionException("Failed to fetch from MarksDB", connection);
+      }
+      return getResponseBytes(connection);
     } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    if (rsp.getResponseCode() != SC_OK) {
-      throw new UrlFetchException("Failed to fetch from MarksDB", req, rsp);
-    }
-    return rsp.getContent();
   }
 
   List<String> fetchSignedCsv(Optional<String> loginAndPassword, String csvPath, String sigPath)

core/src/main/java/google/registry/tmch/NordnUploadAction.java

@@ -16,28 +16,23 @@ package google.registry.tmch;
 
 import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.POST;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.CSV_UTF_8;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_CLAIMS;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 
+import com.google.api.client.http.HttpMethods;
 import com.google.appengine.api.taskqueue.LeaseOptions;
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
@@ -49,16 +44,18 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
+import google.registry.request.UrlConnectionService;
+import google.registry.request.UrlConnectionUtils;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
-import java.util.Optional;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
@@ -97,7 +94,8 @@ public final class NordnUploadAction implements Runnable {
   @Inject Retrier retrier;
   @Inject SecureRandom random;
   @Inject LordnRequestInitializer lordnRequestInitializer;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Parameter(LORDN_PHASE_PARAM) String phase;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -193,47 +191,48 @@ public final class NordnUploadAction implements Runnable {
    * <p>Idempotency: If the exact same LORDN report is uploaded twice, the MarksDB server will
    * return the same confirmation number.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">
-   *     TMCH functional specifications - LORDN File</a>
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">TMCH
+   *     functional specifications - LORDN File</a>
    */
   private void uploadCsvToLordn(String urlPath, String csvData) throws IOException {
     String url = tmchMarksdbUrl + urlPath;
     logger.atInfo().log(
         "LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData);
-    HTTPRequest req = new HTTPRequest(new URL(url), POST, validateCertificate().setDeadline(60d));
-    lordnRequestInitializer.initialize(req, tld);
-    setPayloadMultipart(req, "file", "claims.csv", CSV_UTF_8, csvData, random);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(new URL(url));
+    connection.setRequestMethod(HttpMethods.POST);
+    lordnRequestInitializer.initialize(connection, tld);
+    UrlConnectionUtils.setPayloadMultipart(
+        connection, "file", "claims.csv", CSV_UTF_8, csvData, random);
     try {
-      rsp = fetchService.fetch(req);
+      int responseCode = connection.getResponseCode();
+      if (logger.atInfo().isEnabled()) {
+        String responseContent = new String(getResponseBytes(connection), US_ASCII);
+        if (responseContent.isEmpty()) {
+          responseContent = "(null)";
+        }
+        logger.atInfo().log(
+            "LORDN upload task %s response: HTTP response code %d, response data: %s",
+            actionLogId, responseCode, responseContent);
+      }
+      if (responseCode != SC_ACCEPTED) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB",
+                actionLogId),
+            connection);
+      }
+      String location = connection.getHeaderField(LOCATION);
+      if (location == null) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN upload task %s error: MarksDB failed to provide a Location header",
+                actionLogId),
+            connection);
+      }
+      getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location)));
     } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    if (logger.atInfo().isEnabled()) {
-      String response =
-          (rsp.getContent() == null) ? "(null)" : new String(rsp.getContent(), US_ASCII);
-      logger.atInfo().log(
-          "LORDN upload task %s response: HTTP response code %d, response data: %s",
-          actionLogId, rsp.getResponseCode(), response);
-    }
-    if (rsp.getResponseCode() != SC_ACCEPTED) {
-      throw new UrlFetchException(
-          String.format(
-              "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", actionLogId),
-          req,
-          rsp);
-    }
-    Optional<String> location = getHeaderFirst(rsp, LOCATION);
-    if (!location.isPresent()) {
-      throw new UrlFetchException(
-          String.format(
-              "LORDN upload task %s error: MarksDB failed to provide a Location header",
-              actionLogId),
-          req,
-          rsp);
-    }
-    getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location.get())));
   }
 
   private TaskOptions makeVerifyTask(URL url) {

core/src/main/java/google/registry/tmch/NordnVerifyAction.java

@@ -14,15 +14,11 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
@@ -32,9 +28,11 @@ import google.registry.request.HttpException.ConflictException;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
+import google.registry.request.UrlConnectionService;
 import google.registry.request.auth.Auth;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Map.Entry;
 import javax.inject.Inject;
@@ -68,7 +66,8 @@ public final class NordnVerifyAction implements Runnable {
 
   @Inject LordnRequestInitializer lordnRequestInitializer;
   @Inject Response response;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Header(URL_HEADER) URL url;
   @Inject @Header(HEADER_ACTION_LOG_ID) String actionLogId;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -96,51 +95,49 @@ public final class NordnVerifyAction implements Runnable {
   @VisibleForTesting
   LordnLog verify() throws IOException {
     logger.atInfo().log("LORDN verify task %s: Sending request to URL %s", actionLogId, url);
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
-    lordnRequestInitializer.initialize(req, tld);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
+    lordnRequestInitializer.initialize(connection, tld);
     try {
-      rsp = fetchService.fetch(req);
-    } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
-    }
-    logger.atInfo().log(
-        "LORDN verify task %s response: HTTP response code %d, response data: %s",
-        actionLogId, rsp.getResponseCode(), rsp.getContent());
-    if (rsp.getResponseCode() == SC_NO_CONTENT) {
-      // Send a 400+ status code so App Engine will retry the task.
-      throw new ConflictException("Not ready");
-    }
-    if (rsp.getResponseCode() != SC_OK) {
-      throw new UrlFetchException(
-          String.format("LORDN verify task %s: Failed to verify LORDN upload to MarksDB.",
-              actionLogId),
-          req, rsp);
-    }
-    LordnLog log =
-        LordnLog.parse(ByteSource.wrap(rsp.getContent()).asCharSource(UTF_8).readLines());
-    if (log.getStatus() == LordnLog.Status.ACCEPTED) {
-      logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
-    } else {
-      logger.atSevere().log(
-          "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
-    }
-    for (Entry<String, LordnLog.Result> result : log) {
-      switch (result.getValue().getOutcome()) {
-        case OK:
-          break;
-        case WARNING:
-          // fall through
-        case ERROR:
-          logger.atWarning().log(result.toString());
-          break;
-        default:
-          logger.atWarning().log(
-              "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
-          break;
+      int responseCode = connection.getResponseCode();
+      logger.atInfo().log(
+          "LORDN verify task %s response: HTTP response code %d", actionLogId, responseCode);
+      if (responseCode == SC_NO_CONTENT) {
+        // Send a 400+ status code so App Engine will retry the task.
+        throw new ConflictException("Not ready");
       }
+      if (responseCode != SC_OK) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN verify task %s: Failed to verify LORDN upload to MarksDB.", actionLogId),
+            connection);
+      }
+      LordnLog log =
+          LordnLog.parse(
+              ByteSource.wrap(getResponseBytes(connection)).asCharSource(UTF_8).readLines());
+      if (log.getStatus() == LordnLog.Status.ACCEPTED) {
+        logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
+      } else {
+        logger.atSevere().log(
+            "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
+      }
+      for (Entry<String, LordnLog.Result> result : log) {
+        switch (result.getValue().getOutcome()) {
+          case OK:
+            break;
+          case WARNING:
+            // fall through
+          case ERROR:
+            logger.atWarning().log(result.toString());
+            break;
+          default:
+            logger.atWarning().log(
+                "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
+            break;
+        }
+      }
+      return log;
+    } catch (IOException e) {
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    return log;
   }
 }

core/src/main/java/google/registry/tools/DigestType.java

@@ -26,19 +26,21 @@ import java.util.Optional;
  * https://www.iana.org/assignments/ds-rr-types/ds-rr-types.xhtml
  */
 public enum DigestType {
-  SHA1(1),
-  SHA256(2),
+  SHA1(1, 20),
+  SHA256(2, 32),
   // Algorithm number 3 is GOST R 34.11-94 and is deliberately NOT SUPPORTED.
   // This algorithm was reviewed by ise-crypto and deemed academically broken (b/207029800).
   // In addition, RFC 8624 specifies that this algorithm MUST NOT be used for DNSSEC delegations.
   // TODO(sarhabot@): Add note in Cloud DNS code to notify the Registry of any new changes to
   // supported digest types.
-  SHA384(4);
+  SHA384(4, 48);
 
   private final int wireValue;
+  private final int bytes;
 
-  DigestType(int wireValue) {
+  DigestType(int wireValue, int bytes) {
     this.wireValue = wireValue;
+    this.bytes = bytes;
   }
 
   /** Fetches a DigestType enumeration constant by its IANA assigned value. */
@@ -55,4 +57,9 @@ public enum DigestType {
   public int getWireValue() {
     return wireValue;
   }
+
+  /** Returns the expected length in bytes of the signature. */
+  public int getBytes() {
+    return bytes;
+  }
 }

core/src/main/java/google/registry/tools/DsRecord.java

@@ -51,6 +51,11 @@ abstract class DsRecord {
     checkArgumentPresent(
         DigestType.fromWireValue(digestType),
         String.format("DS record uses an unrecognized digest type: %d", digestType));
+    if (DigestType.fromWireValue(digestType).get().getBytes()
+        != BaseEncoding.base16().decode(digest).length) {
+      throw new IllegalArgumentException(
+          String.format("DS record has an invalid digest length: %s", digest));
+    }
 
     if (!DomainFlowUtils.validateAlgorithm(alg)) {
       throw new IllegalArgumentException(

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -38,8 +38,7 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
-import google.registry.request.Modules.URLFetchServiceModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
@@ -79,8 +78,7 @@ import javax.inject.Singleton;
       RegistryToolDataflowModule.class,
       RequestFactoryModule.class,
       SecretManagerModule.class,
-      URLFetchServiceModule.class,
-      UrlFetchTransportModule.class,
+      UrlConnectionServiceModule.class,
       UserServiceModule.class,
       UtilsModule.class,
       VoidDnsWriterModule.class,

core/src/main/java/google/registry/whois/NameserverLookupByIpCommand.java

@@ -64,11 +64,17 @@ final class NameserverLookupByIpCommand implements WhoisCommand {
           jpaTm()
               .transact(
                   () ->
-                      // We cannot query @Convert-ed fields in HQL so we must use native Postgres
+                      // We cannot query @Convert-ed fields in HQL so we must use native Postgres.
                       jpaTm()
                           .getEntityManager()
+                          /**
+                           * Using array_operator <@ (contained-by) with gin index on inet_address.
+                           * Without gin index, this is slightly slower than the alternative form of
+                           * ':address = ANY(inet_address)'.
+                           */
                           .createNativeQuery(
-                              "SELECT * From \"Host\" WHERE :address = ANY(inet_addresses) AND "
+                              "SELECT * From \"Host\" WHERE "
+                                  + "ARRAY[ CAST(:address AS TEXT) ] <@ inet_addresses AND "
                                   + "deletion_time > CAST(:now AS timestamptz)",
                               HostResource.class)
                           .setParameter("address", InetAddresses.toAddrString(ipAddress))

core/src/test/java/google/registry/batch/AsyncTaskEnqueuerTest.java

@@ -22,23 +22,20 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestLogHandlerUtils.assertLogMessage;
-import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardHours;
 import static org.joda.time.Duration.standardSeconds;
-import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSortedSet;
 import google.registry.model.contact.ContactResource;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.CapturingLogHandler;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.JdkLoggerConfig;
 import google.registry.util.Retrier;
 import java.util.logging.Level;
@@ -48,7 +45,6 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
@@ -63,27 +59,25 @@ public class AsyncTaskEnqueuerTest {
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  @Mock private AppEngineServiceUtils appEngineServiceUtils;
-
   private AsyncTaskEnqueuer asyncTaskEnqueuer;
   private final CapturingLogHandler logHandler = new CapturingLogHandler();
   private final FakeClock clock = new FakeClock(DateTime.parse("2015-05-18T12:34:56Z"));
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @BeforeEach
   void beforeEach() {
     JdkLoggerConfig.getConfig(AsyncTaskEnqueuer.class).addHandler(logHandler);
-    when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
-    asyncTaskEnqueuer = createForTesting(appEngineServiceUtils, clock, standardSeconds(90));
+    asyncTaskEnqueuer =
+        createForTesting(cloudTasksHelper.getTestCloudTasksUtils(), clock, standardSeconds(90));
   }
 
   public static AsyncTaskEnqueuer createForTesting(
-      AppEngineServiceUtils appEngineServiceUtils, FakeClock clock, Duration asyncDeleteDelay) {
+      CloudTasksUtils cloudTasksUtils, FakeClock clock, Duration asyncDeleteDelay) {
     return new AsyncTaskEnqueuer(
-        getQueue(QUEUE_ASYNC_ACTIONS),
         getQueue(QUEUE_ASYNC_DELETE),
         getQueue(QUEUE_ASYNC_HOST_RENAME),
         asyncDeleteDelay,
-        appEngineServiceUtils,
+        cloudTasksUtils,
         new Retrier(new FakeSleeper(clock), 1));
   }
 
@@ -92,18 +86,16 @@ public class AsyncTaskEnqueuerTest {
     ContactResource contact = persistActiveContact("jd23456");
     asyncTaskEnqueuer.enqueueAsyncResave(
         contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(5));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
-        new TaskMatcher()
+        new CloudTasksHelper.TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
-            .etaDelta(
-                standardDays(5).minus(standardSeconds(30)),
-                standardDays(5).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(Duration.standardDays(5))));
   }
 
   @Test
@@ -114,19 +106,17 @@ public class AsyncTaskEnqueuerTest {
         contact.createVKey(),
         now,
         ImmutableSortedSet.of(now.plusHours(24), now.plusHours(50), now.plusHours(75)));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, now.toString())
             .param(PARAM_RESAVE_TIMES, "2015-05-20T14:34:56.000Z,2015-05-21T15:34:56.000Z")
-            .etaDelta(
-                standardHours(24).minus(standardSeconds(30)),
-                standardHours(24).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(Duration.standardHours(24))));
   }
 
   @MockitoSettings(strictness = Strictness.LENIENT)
@@ -135,7 +125,7 @@ public class AsyncTaskEnqueuerTest {
     ContactResource contact = persistActiveContact("jd23456");
     asyncTaskEnqueuer.enqueueAsyncResave(
         contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(31));
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
     assertLogMessage(logHandler, Level.INFO, "Ignoring async re-save");
   }
 }

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -91,13 +91,13 @@ import google.registry.model.transfer.ContactTransferData;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.mapreduce.MapreduceTestCase;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.Retrier;
 import google.registry.util.Sleeper;
@@ -148,7 +148,7 @@ public class DeleteContactsAndHostsActionTest
     inject.setStaticField(Ofy.class, "clock", clock);
     enqueuer =
         AsyncTaskEnqueuerTest.createForTesting(
-            mock(AppEngineServiceUtils.class), clock, Duration.ZERO);
+            new CloudTasksHelper(clock).getTestCloudTasksUtils(), clock, Duration.ZERO);
     AsyncTaskMetrics asyncTaskMetricsMock = mock(AsyncTaskMetrics.class);
     action = new DeleteContactsAndHostsAction();
     action.asyncTaskMetrics = asyncTaskMetricsMock;

core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java

@@ -39,7 +39,6 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.persistence.transaction.QueryComposer.Comparator;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DualDatabaseTest;
@@ -78,8 +77,7 @@ class DeleteExpiredDomainsActionTest {
     createTld("tld");
     EppController eppController =
         DaggerEppTestComponent.builder()
-            .fakesAndMocksModule(
-                FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock)))
+            .fakesAndMocksModule(FakesAndMocksModule.create(clock))
             .build()
             .startRequest()
             .eppController();

core/src/test/java/google/registry/batch/RefreshDnsOnHostRenameActionTest.java

@@ -49,6 +49,7 @@ import google.registry.batch.RefreshDnsOnHostRenameAction.RefreshDnsOnHostRename
 import google.registry.dns.DnsQueue;
 import google.registry.model.host.HostResource;
 import google.registry.model.server.Lock;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
@@ -59,7 +60,6 @@ import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import google.registry.testing.TestSqlOnly;
 import google.registry.testing.mapreduce.MapreduceTestCase;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.Retrier;
 import google.registry.util.Sleeper;
@@ -89,7 +89,7 @@ public class RefreshDnsOnHostRenameActionTest
     createTld("tld");
     enqueuer =
         AsyncTaskEnqueuerTest.createForTesting(
-            mock(AppEngineServiceUtils.class), clock, Duration.ZERO);
+            new CloudTasksHelper(clock).getTestCloudTasksUtils(), clock, Duration.ZERO);
     AsyncTaskMetrics asyncTaskMetricsMock = mock(AsyncTaskMetrics.class);
     action = new RefreshDnsOnHostRenameAction();
     action.asyncTaskMetrics = asyncTaskMetricsMock;

core/src/test/java/google/registry/batch/ResaveEntityActionTest.java

@@ -25,12 +25,10 @@ import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static google.registry.testing.DatabaseHelper.persistDomainWithDependentResources;
 import static google.registry.testing.DatabaseHelper.persistDomainWithPendingTransfer;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardSeconds;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import google.registry.model.domain.DomainBase;
@@ -40,10 +38,11 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.Ofy;
 import google.registry.request.Response;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.AppEngineServiceUtils;
 import org.joda.time.DateTime;
@@ -71,13 +70,14 @@ public class ResaveEntityActionTest {
   @Mock private Response response;
   private final FakeClock clock = new FakeClock(DateTime.parse("2016-02-11T10:00:00Z"));
   private AsyncTaskEnqueuer asyncTaskEnqueuer;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @BeforeEach
   void beforeEach() {
     inject.setStaticField(Ofy.class, "clock", clock);
-    when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
     asyncTaskEnqueuer =
-        AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, Duration.ZERO);
+        AsyncTaskEnqueuerTest.createForTesting(
+            cloudTasksHelper.getTestCloudTasksUtils(), clock, Duration.ZERO);
     createTld("tld");
   }
 
@@ -143,17 +143,15 @@ public class ResaveEntityActionTest {
     DomainBase resavedDomain = loadByEntity(domain);
     assertThat(resavedDomain.getGracePeriods()).isEmpty();
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, resavedDomain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, requestedTime.toString())
-            .etaDelta(
-                standardDays(5).minus(standardSeconds(30)),
-                standardDays(5).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(standardDays(5))));
   }
 }

core/src/test/java/google/registry/flows/EppPointInTimeTest.java

@@ -72,7 +72,7 @@ class EppPointInTimeTest {
     SessionMetadata sessionMetadata = new HttpSessionMetadata(new FakeHttpSession());
     sessionMetadata.setRegistrarId("TheRegistrar");
     DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock)))
+        .fakesAndMocksModule(FakesAndMocksModule.create(clock))
         .build()
         .startRequest()
         .flowComponentBuilder()

core/src/test/java/google/registry/flows/EppTestCase.java

@@ -224,12 +224,14 @@ public class EppTestCase {
     EppRequestHandler handler = new EppRequestHandler();
     FakeResponse response = new FakeResponse();
     handler.response = response;
-    eppMetricBuilder = EppMetric.builderForRequest(clock);
-    handler.eppController = DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, eppMetricBuilder))
-        .build()
-        .startRequest()
-        .eppController();
+    FakesAndMocksModule fakesAndMocksModule = FakesAndMocksModule.create(clock);
+    eppMetricBuilder = fakesAndMocksModule.getMetricBuilder();
+    handler.eppController =
+        DaggerEppTestComponent.builder()
+            .fakesAndMocksModule(fakesAndMocksModule)
+            .build()
+            .startRequest()
+            .eppController();
     handler.executeEpp(
         sessionMetadata,
         credentials,

core/src/test/java/google/registry/flows/EppTestComponent.java

@@ -15,8 +15,6 @@
 package google.registry.flows;
 
 import static org.joda.time.Duration.standardSeconds;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import dagger.Component;
 import dagger.Module;
@@ -33,12 +31,12 @@ import google.registry.flows.domain.DomainFlowTmchUtils;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.request.RequestScope;
 import google.registry.request.lock.LockHandler;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeLockHandler;
 import google.registry.testing.FakeSleeper;
 import google.registry.tmch.TmchCertificateAuthority;
 import google.registry.tmch.TmchXmlSignature;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.Clock;
 import google.registry.util.Sleeper;
 import javax.inject.Singleton;
@@ -60,35 +58,32 @@ public interface EppTestComponent {
     private EppMetric.Builder metricBuilder;
     private FakeClock clock;
     private FakeLockHandler lockHandler;
-    private AppEngineServiceUtils appEngineServiceUtils;
     private Sleeper sleeper;
+    private CloudTasksHelper cloudTasksHelper;
 
-    public static FakesAndMocksModule create() {
-      FakeClock clock = new FakeClock();
-      return create(clock, EppMetric.builderForRequest(clock));
+    public CloudTasksHelper getCloudTasksHelper() {
+      return cloudTasksHelper;
     }
 
-    public static FakesAndMocksModule create(FakeClock clock, EppMetric.Builder metricBuilder) {
-      return create(
-          clock,
-          metricBuilder,
-          new TmchXmlSignature(new TmchCertificateAuthority(TmchCaMode.PILOT, clock)));
+    public EppMetric.Builder getMetricBuilder() {
+      return metricBuilder;
     }
 
-    public static FakesAndMocksModule create(
-        FakeClock clock, EppMetric.Builder eppMetricBuilder, TmchXmlSignature tmchXmlSignature) {
+    public static FakesAndMocksModule create(FakeClock clock) {
       FakesAndMocksModule instance = new FakesAndMocksModule();
-      AppEngineServiceUtils appEngineServiceUtils = mock(AppEngineServiceUtils.class);
-      when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
+      CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
       instance.asyncTaskEnqueuer =
-          AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, standardSeconds(90));
+          AsyncTaskEnqueuerTest.createForTesting(
+              cloudTasksHelper.getTestCloudTasksUtils(), clock, standardSeconds(90));
       instance.clock = clock;
-      instance.domainFlowTmchUtils = new DomainFlowTmchUtils(tmchXmlSignature);
-      instance.sleeper = new FakeSleeper(clock);
+      instance.domainFlowTmchUtils =
+          new DomainFlowTmchUtils(
+              new TmchXmlSignature(new TmchCertificateAuthority(TmchCaMode.PILOT, clock)));
+      instance.sleeper = new FakeSleeper(instance.clock);
       instance.dnsQueue = DnsQueue.create();
-      instance.metricBuilder = eppMetricBuilder;
-      instance.appEngineServiceUtils = appEngineServiceUtils;
+      instance.metricBuilder = EppMetric.builderForRequest(clock);
       instance.lockHandler = new FakeLockHandler(true);
+      instance.cloudTasksHelper = cloudTasksHelper;
       return instance;
     }
 
@@ -127,11 +122,6 @@ public interface EppTestComponent {
       return metricBuilder;
     }
 
-    @Provides
-    AppEngineServiceUtils provideAppEngineServiceUtils() {
-      return appEngineServiceUtils;
-    }
-
     @Provides
     Sleeper provideSleeper() {
       return sleeper;

core/src/test/java/google/registry/flows/FlowTestCase.java

@@ -33,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.collect.ObjectArrays;
-import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
 import google.registry.flows.EppTestComponent.FakesAndMocksModule;
 import google.registry.flows.picker.FlowPicker;
 import google.registry.model.billing.BillingEvent;
@@ -45,14 +44,13 @@ import google.registry.model.ofy.Ofy;
 import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.EppLoader;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeHttpSession;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TestDataHelper;
-import google.registry.tmch.TmchCertificateAuthority;
-import google.registry.tmch.TmchXmlSignature;
 import google.registry.util.TypeUtils.TypeInstantiator;
 import google.registry.xml.ValidationMode;
 import java.util.Arrays;
@@ -88,7 +86,7 @@ public abstract class FlowTestCase<F extends Flow> {
   protected FakeClock clock = new FakeClock(DateTime.now(UTC));
   protected TransportCredentials credentials = new PasswordOnlyTransportCredentials();
   protected EppRequestSource eppRequestSource = EppRequestSource.UNIT_TEST;
-  private TmchXmlSignature testTmchXmlSignature = null;
+  protected CloudTasksHelper cloudTasksHelper;
 
   private EppMetric.Builder eppMetricBuilder;
 
@@ -229,13 +227,12 @@ public abstract class FlowTestCase<F extends Flow> {
     // Assert that the xml triggers the flow we expect.
     assertThat(FlowPicker.getFlowClass(eppLoader.getEpp()))
         .isEqualTo(new TypeInstantiator<F>(getClass()){}.getExactType());
+
+    FakesAndMocksModule fakesAndMocksModule = FakesAndMocksModule.create(clock);
+    cloudTasksHelper = fakesAndMocksModule.getCloudTasksHelper();
     // Run the flow.
-    TmchXmlSignature tmchXmlSignature =
-        testTmchXmlSignature != null
-            ? testTmchXmlSignature
-            : new TmchXmlSignature(new TmchCertificateAuthority(tmchCaMode, clock));
     return DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, eppMetricBuilder, tmchXmlSignature))
+        .fakesAndMocksModule(fakesAndMocksModule)
         .build()
         .startRequest()
         .flowComponentBuilder()
@@ -300,8 +297,6 @@ public abstract class FlowTestCase<F extends Flow> {
     return output;
   }
 
-  private TmchCaMode tmchCaMode = TmchCaMode.PILOT;
-
   public EppOutput dryRunFlowAssertResponse(String xml, String... ignoredPaths) throws Exception {
     List<Object> beforeEntities = DatabaseHelper.loadAllEntities();
     EppOutput output =

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -856,7 +856,8 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     assertAboutDomains()
         .that(domain)
         .hasExactlyDsData(
-            DelegationSignerData.create(12345, 3, 1, base16().decode("49FD46E6C4B45C55D4AC"))
+            DelegationSignerData.create(
+                    12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))
                 .cloneWithDomainRepoId(domain.getRepoId()));
   }
 

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -54,14 +54,13 @@ import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
 import static google.registry.testing.TaskQueueHelper.assertDnsTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.money.CurrencyUnit.USD;
 import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -102,10 +101,10 @@ import google.registry.model.tld.Registry.TldType;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import java.util.Map;
@@ -314,17 +313,17 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
     clock.advanceOneMilli();
     runFlowAssertResponse(loadFile("domain_delete_response_pending.xml"));
     Duration when = standardDays(3);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .param(PARAM_RESAVE_TIMES, clock.nowUtc().plusDays(5).toString())
-            .etaDelta(when.minus(standardSeconds(30)), when.plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(when)));
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -43,12 +43,11 @@ import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
 import static google.registry.testing.HostResourceSubject.assertAboutHosts;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.money.CurrencyUnit.USD;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -106,10 +105,10 @@ import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import java.util.Map;
@@ -514,18 +513,16 @@ class DomainTransferRequestFlowTest
     assertPollMessagesEmitted(expectedExpirationTime, implicitTransferTime);
     assertAboutDomainAfterAutomaticTransfer(
         expectedExpirationTime, implicitTransferTime, Period.create(1, Unit.YEARS));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
-            .etaDelta(
-                registry.getAutomaticTransferLength().minus(standardSeconds(30)),
-                registry.getAutomaticTransferLength().plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(registry.getAutomaticTransferLength())));
   }
 
   private void doSuccessfulTest(

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -123,13 +123,17 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, DomainBase> {
 
   private static final DelegationSignerData SOME_DSDATA =
-      DelegationSignerData.create(1, 2, 2, base16().decode("0123"));
+      DelegationSignerData.create(
+          1,
+          2,
+          2,
+          base16().decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   private static final ImmutableMap<String, String> OTHER_DSDATA_TEMPLATE_MAP =
       ImmutableMap.of(
           "KEY_TAG", "12346",
           "ALG", "3",
           "DIGEST_TYPE", "1",
-          "DIGEST", "38EC35D5B3A34B44C39B");
+          "DIGEST", "A94A8FE5CCB19BA61C4C0873D391E987982FBBD3");
 
   private ContactResource sh8013Contact;
   private ContactResource mak21Contact;
@@ -523,9 +527,17 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         null,
         ImmutableSet.of(
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableMap.of(
-            "KEY_TAG", "12346", "ALG", "3", "DIGEST_TYPE", "1", "DIGEST", "38EC35D5B3A34B44C39B"));
+            "KEY_TAG",
+            "12346",
+            "ALG",
+            "3",
+            "DIGEST_TYPE",
+            "1",
+            "DIGEST",
+            "A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"));
   }
 
   @TestOfyAndSql
@@ -535,9 +547,17 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableMap.of(
-            "KEY_TAG", "12346", "ALG", "3", "DIGEST_TYPE", "1", "DIGEST", "38EC35D5B3A34B44C39B"));
+            "KEY_TAG",
+            "12346",
+            "ALG",
+            "3",
+            "DIGEST_TYPE",
+            "1",
+            "DIGEST",
+            "A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"));
   }
 
   @TestOfyAndSql
@@ -546,7 +566,15 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "0123"));
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
@@ -555,8 +583,23 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(
-            SOME_DSDATA, DelegationSignerData.create(1, 8, 4, base16().decode("4567"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "8", "DIGEST_TYPE", "4", "DIGEST", "4567"));
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                8,
+                4,
+                base16()
+                    .decode(
+                        "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "8",
+            "DIGEST_TYPE",
+            "4",
+            "DIGEST",
+            "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"));
   }
 
   // Changing any of the four fields in DelegationSignerData should result in a new object
@@ -566,8 +609,22 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(
-            SOME_DSDATA, DelegationSignerData.create(12346, 2, 2, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "12346", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "0123"));
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                12346,
+                2,
+                2,
+                base16()
+                    .decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "12346",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
@@ -575,8 +632,23 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 8, 2, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "8", "DIGEST_TYPE", "2", "DIGEST", "0123"));
+        ImmutableSet.of(
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                8,
+                2,
+                base16()
+                    .decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "8",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
@@ -584,8 +656,24 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 2, 4, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "4", "DIGEST", "0123"));
+        ImmutableSet.of(
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                2,
+                4,
+                base16()
+                    .decode(
+                        "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "4",
+            "DIGEST",
+            "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"));
   }
 
   @TestOfyAndSql
@@ -593,15 +681,35 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 2, 2, base16().decode("4567"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "4567"));
+        ImmutableSet.of(
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                2,
+                2,
+                base16()
+                    .decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
   void testSuccess_secDnsAddToMaxRecords() throws Exception {
     ImmutableSet.Builder<DelegationSignerData> builder = new ImmutableSet.Builder<>();
     for (int i = 0; i < 7; ++i) {
-      builder.add(DelegationSignerData.create(i, 2, 2, new byte[] {0, 1, 2}));
+      builder.add(
+          DelegationSignerData.create(
+              i,
+              2,
+              2,
+              base16().decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08")));
     }
     ImmutableSet<DelegationSignerData> commonDsData = builder.build();
 
@@ -613,7 +721,10 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
                 commonDsData,
                 ImmutableSet.of(
                     DelegationSignerData.create(
-                        12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))))));
+                        12346,
+                        3,
+                        1,
+                        base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))))));
   }
 
   @TestOfyAndSql
@@ -622,7 +733,8 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_rem.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of(SOME_DSDATA));
   }
 
@@ -633,7 +745,8 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_rem_all.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of());
   }
 
@@ -643,17 +756,24 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add_rem.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))),
+            DelegationSignerData.create(
+                12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))));
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))));
   }
 
   @TestOfyAndSql
   void testSuccess_secDnsAddRemoveToMaxRecords() throws Exception {
     ImmutableSet.Builder<DelegationSignerData> builder = new ImmutableSet.Builder<>();
     for (int i = 0; i < 7; ++i) {
-      builder.add(DelegationSignerData.create(i, 2, 2, new byte[] {0, 1, 2}));
+      builder.add(
+          DelegationSignerData.create(
+              i,
+              2,
+              2,
+              base16().decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08")));
     }
     ImmutableSet<DelegationSignerData> commonDsData = builder.build();
 
@@ -664,13 +784,19 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
                 commonDsData,
                 ImmutableSet.of(
                     DelegationSignerData.create(
-                        12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))))),
+                        12345,
+                        3,
+                        1,
+                        base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))))),
         ImmutableSet.copyOf(
             union(
                 commonDsData,
                 ImmutableSet.of(
                     DelegationSignerData.create(
-                        12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))))));
+                        12346,
+                        3,
+                        1,
+                        base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))))));
   }
 
   @TestOfyAndSql
@@ -680,10 +806,12 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add_rem_same.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))),
+            DelegationSignerData.create(
+                12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))));
+            DelegationSignerData.create(
+                12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))));
   }
 
   @TestOfyAndSql
@@ -852,6 +980,41 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
+  @TestOfyAndSql
+  void testFailure_secDnsInvalidDigestLength() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(ImmutableSet.of(DelegationSignerData.create(1, 2, 1, new byte[] {0, 1, 2})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains("Domain contains DS record(s) with an invalid digest length");
+  }
+
+  @TestOfyAndSql
+  void testFailure_secDnsMultipleInvalidDigestLengths() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(
+                ImmutableSet.of(
+                    DelegationSignerData.create(1, 2, 1, new byte[] {0, 1, 2, 3, 4}),
+                    DelegationSignerData.create(2, 2, 2, new byte[] {5, 6, 7})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertThat(thrown).hasMessageThat().contains("0, 1, 2, 3, 4");
+    assertThat(thrown).hasMessageThat().contains("5, 6, 7");
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains("Domain contains DS record(s) with an invalid digest length");
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
   @TestOfyAndSql
   void testFailure_secDnsInvalidAlgorithm() throws Exception {
     setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -21,16 +21,15 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.testing.DatabaseHelper.insertInDb;
 import static google.registry.testing.LogsSubject.assertAboutLogs;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.junit.Assert.assertThrows;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.testing.TestLogHandler;
 import com.google.common.truth.Truth8;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
@@ -63,6 +62,7 @@ import org.junitpioneer.jupiter.RetryingTest;
 public class ReplicateToDatastoreActionTest {
 
   private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ"));
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   @RegisterExtension
   public final AppEngineExtension appEngine =
@@ -203,41 +203,42 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @RetryingTest(4)
-  void testMissingTransactions() {
-    assumeTrue(ReplayExtension.replayTestsEnabled());
-
-    // Write a transaction (should have a transaction id of 1).
+  void testNoTransactionIdUpdate() {
+    // Create an object.
     TestObject foo = TestObject.create("foo");
     insertInDb(foo);
 
-    // Force the last transaction id back to -1 so that we look for transaction 0.
-    ofyTm().transact(() -> ofyTm().insert(new LastSqlTransaction(-1)));
+    // Fail two transactions.
+    for (int i = 0; i < 2; ++i) {
+      try {
+        jpaTm()
+            .transact(
+                () -> {
+                  jpaTm().delete(foo.key());
+                  // Explicitly save the transaction entity to force the id update.
+                  jpaTm().insert(new TransactionEntity(new byte[] {1, 2, 3}));
+                  throw new RuntimeException("fail!!!");
+                });
+      } catch (Exception e) {
+        logger.atInfo().log("Got expected exception.");
+      }
+    }
 
+    TestObject bar = TestObject.create("bar");
+    insertInDb(bar);
+
+    // Make sure we have only the expected transaction ids.
     List<TransactionEntity> txns = action.getTransactionBatchAtSnapshot();
-    assertThat(txns).hasSize(1);
-    assertThat(assertThrows(IllegalStateException.class, () -> applyTransaction(txns.get(0))))
-        .hasMessageThat()
-        .isEqualTo("Missing transaction: last txn id = -1, next available txn = 1");
-  }
+    assertThat(txns).hasSize(2);
+    for (TransactionEntity txn : txns) {
+      assertThat(txn.getId()).isNotEqualTo(2);
+      assertThat(txn.getId()).isNotEqualTo(3);
+      applyTransaction(txn);
+    }
 
-  @Test
-  void testMissingTransactions_fullTask() {
-    assumeTrue(ReplayExtension.replayTestsEnabled());
-
-    // Write a transaction (should have a transaction id of 1).
-    TestObject foo = TestObject.create("foo");
-    insertInDb(foo);
-
-    // Force the last transaction id back to -1 so that we look for transaction 0.
-    ofyTm().transact(() -> ofyTm().insert(new LastSqlTransaction(-1)));
-    action.run();
-    assertAboutLogs()
-        .that(logHandler)
-        .hasSevereLogWithCause(
-            new IllegalStateException(
-                "Missing transaction: last txn id = -1, next available txn = 1"));
-    assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
-    assertThat(response.getPayload()).isEqualTo("Errored out replaying files.");
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(foo.key()))).isEqualTo(foo);
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(bar.key()))).isEqualTo(bar);
+    assertThat(ofyTm().transact(() -> LastSqlTransaction.load()).getTransactionId()).isEqualTo(4);
   }
 
   @Test

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -14,7 +14,6 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@@ -29,20 +28,13 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
-import com.google.common.base.Ascii;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSource;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.common.Cursor;
@@ -57,20 +49,21 @@ import google.registry.testing.FakeClock;
 import google.registry.testing.FakeKeyringModule;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.Retrier;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
-import java.util.Map;
 import java.util.Optional;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
 
 /** Unit tests for {@link RdeReportAction}. */
 @DualDatabaseTest
@@ -89,20 +82,21 @@ public class RdeReportActionTest {
 
   private final FakeResponse response = new FakeResponse();
   private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
-  private final URLFetchService urlFetchService = mock(URLFetchService.class);
-  private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
-  private final HTTPResponse httpResponse = mock(HTTPResponse.class);
   private final PGPPublicKey encryptKey =
       new FakeKeyringModule().get().getRdeStagingEncryptionKey();
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
   private final BlobId reportFile =
       BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
 
   private RdeReportAction createAction() {
     RdeReporter reporter = new RdeReporter();
     reporter.reportUrlPrefix = "https://rde-report.example";
-    reporter.urlFetchService = urlFetchService;
     reporter.password = "foo";
+    reporter.urlConnectionService = urlConnectionService;
     reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
     RdeReportAction action = new RdeReportAction();
     action.gcsUtils = gcsUtils;
@@ -127,6 +121,7 @@ public class RdeReportActionTest {
         Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
     gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
   }
 
   @TestOfyAndSql
@@ -142,24 +137,22 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
-    assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
+    verify(httpUrlConnection).setRequestMethod("PUT");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -167,9 +160,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_withPrefix() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     RdeReportAction action = createAction();
     action.prefix = Optional.of("job-name/");
     gcsUtils.delete(reportFile);
@@ -182,16 +174,14 @@ public class RdeReportActionTest {
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
-    assertThat(headers)
-        .containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
+    verify(httpUrlConnection).setRequestMethod("PUT");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -204,9 +194,8 @@ public class RdeReportActionTest {
     PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
     gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
   }
@@ -239,9 +228,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream());
     InternalServerErrorException thrown =
         assertThrows(
             InternalServerErrorException.class,
@@ -252,18 +240,17 @@ public class RdeReportActionTest {
   @TestOfyAndSql
   void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
     class ExpectedThrownException extends RuntimeException {}
-    when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
+    when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException());
     assertThrows(
         ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
   }
 
   @TestOfyAndSql
   void testRunWithLock_socketTimeout_doesRetry() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture()))
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
+    when(httpUrlConnection.getResponseCode())
         .thenThrow(new SocketTimeoutException())
-        .thenReturn(httpResponse);
+        .thenReturn(SC_OK);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@@ -274,14 +261,6 @@ public class RdeReportActionTest {
     return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime();
   }
 
-  private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
-    ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
-    for (HTTPHeader header : headers) {
-      builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
-    }
-    return builder.build();
-  }
-
   private static XjcRdeReportReport parseReport(byte[] data) {
     try {
       return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));

core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.MediaType.CSV_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
+import static google.registry.request.UrlConnectionUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
@@ -27,22 +27,19 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import google.registry.testing.AppEngineExtension;
+import java.io.ByteArrayOutputStream;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
+import javax.net.ssl.HttpsURLConnection;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockito.ArgumentCaptor;
 
-/** Unit tests for {@link UrlFetchUtils}. */
-class UrlFetchUtilsTest {
-
-  @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().build();
+/** Tests for {@link UrlConnectionUtils}. */
+public class UrlConnectionUtilsTest {
 
   private final Random random = mock(Random.class);
 
@@ -58,25 +55,28 @@ class UrlFetchUtilsTest {
   }
 
   @Test
-  void testSetPayloadMultipart() {
-    HTTPRequest request = mock(HTTPRequest.class);
+  void testSetPayloadMultipart() throws Exception {
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
+    ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+    when(connection.getOutputStream()).thenReturn(connectionOutputStream);
     setPayloadMultipart(
-        request,
+        connection,
         "lol",
         "cat",
         CSV_UTF_8,
         "The nice people at the store say hello. ヘ(◕。◕ヘ)",
         random);
-    ArgumentCaptor<HTTPHeader> headerCaptor = ArgumentCaptor.forClass(HTTPHeader.class);
-    verify(request, times(2)).addHeader(headerCaptor.capture());
-    List<HTTPHeader> addedHeaders = headerCaptor.getAllValues();
-    assertThat(addedHeaders.get(0).getName()).isEqualTo(CONTENT_TYPE);
-    assertThat(addedHeaders.get(0).getValue())
-        .isEqualTo(
-            "multipart/form-data; "
-                + "boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"");
-    assertThat(addedHeaders.get(1).getName()).isEqualTo(CONTENT_LENGTH);
-    assertThat(addedHeaders.get(1).getValue()).isEqualTo("294");
+    ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
+    ArgumentCaptor<String> valueCaptor = ArgumentCaptor.forClass(String.class);
+    verify(connection, times(2)).setRequestProperty(keyCaptor.capture(), valueCaptor.capture());
+    List<String> addedKeys = keyCaptor.getAllValues();
+    assertThat(addedKeys).containsExactly(CONTENT_TYPE, CONTENT_LENGTH);
+    List<String> addedValues = valueCaptor.getAllValues();
+    assertThat(addedValues)
+        .containsExactly(
+            "multipart/form-data;"
+                + " boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"",
+            "294");
     String payload =
         "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n"
             + "Content-Disposition: form-data; name=\"lol\"; filename=\"cat\"\r\n"
@@ -84,18 +84,20 @@ class UrlFetchUtilsTest {
             + "\r\n"
             + "The nice people at the store say hello. ヘ(◕。◕ヘ)\r\n"
             + "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r\n";
-    verify(request).setPayload(payload.getBytes(UTF_8));
-    verifyNoMoreInteractions(request);
+    verify(connection).setDoOutput(true);
+    verify(connection).getOutputStream();
+    assertThat(connectionOutputStream.toByteArray()).isEqualTo(payload.getBytes(UTF_8));
+    verifyNoMoreInteractions(connection);
   }
 
   @Test
   void testSetPayloadMultipart_boundaryInPayload() {
-    HTTPRequest request = mock(HTTPRequest.class);
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
     String payload = "I screamed------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHHH";
     IllegalStateException thrown =
         assertThrows(
             IllegalStateException.class,
-            () -> setPayloadMultipart(request, "lol", "cat", CSV_UTF_8, payload, random));
+            () -> setPayloadMultipart(connection, "lol", "cat", CSV_UTF_8, payload, random));
     assertThat(thrown)
         .hasMessageThat()
         .contains(

core/src/test/java/google/registry/testing/FakeURLFetchService.java

@@ -1,46 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.collect.ImmutableList;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * A fake {@link URLFetchService} that serves constructed {@link HTTPResponse} objects from
- * a simple {@link Map} ({@link URL} to {@link HTTPResponse}) lookup.
- */
-public class FakeURLFetchService extends ForwardingURLFetchService {
-
-  private Map<URL, HTTPResponse> backingMap;
-
-  public FakeURLFetchService(Map<URL, HTTPResponse> backingMap) {
-    this.backingMap = backingMap;
-  }
-
-  @Override
-  public HTTPResponse fetch(HTTPRequest request) {
-    URL requestURL = request.getURL();
-    if (backingMap.containsKey(requestURL)) {
-      return backingMap.get(requestURL);
-    } else {
-      return new HTTPResponse(HttpURLConnection.HTTP_NOT_FOUND, null, null, ImmutableList.of());
-    }
-  }
-}

core/src/test/java/google/registry/testing/FakeUrlConnectionService.java

@@ -0,0 +1,46 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.testing;
+
+import static org.mockito.Mockito.when;
+
+import google.registry.request.UrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+/** A fake {@link UrlConnectionService} with a mocked HTTP connection for testing. */
+public class FakeUrlConnectionService implements UrlConnectionService {
+
+  private final HttpURLConnection mockConnection;
+  private final List<URL> connectedUrls;
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection) {
+    this(mockConnection, new ArrayList<>());
+  }
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection, List<URL> connectedUrls) {
+    this.mockConnection = mockConnection;
+    this.connectedUrls = connectedUrls;
+  }
+
+  @Override
+  public HttpURLConnection createConnection(URL url) {
+    connectedUrls.add(url);
+    when(mockConnection.getURL()).thenReturn(url);
+    return mockConnection;
+  }
+}

core/src/test/java/google/registry/testing/ForwardingURLFetchService.java

@@ -1,49 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.util.concurrent.Futures;
-import java.io.IOException;
-import java.net.URL;
-import java.util.concurrent.Future;
-
-/**
- * An implementation of the {@link URLFetchService} interface that forwards all requests through
- * a synchronous fetch call.
- */
-public abstract class ForwardingURLFetchService implements URLFetchService {
-
-  @Override
-  public HTTPResponse fetch(URL url) throws IOException {
-    return fetch(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(URL url) {
-    return fetchAsync(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(HTTPRequest request) {
-    try {
-      return Futures.immediateFuture(fetch(request));
-    } catch (Exception e) {
-      return Futures.immediateFailedFuture(e);
-    }
-  }
-}

core/src/test/java/google/registry/tmch/NordnUploadActionTest.java

@@ -19,21 +19,22 @@ import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.FORM_DATA;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistDomainAndEnqueueLordn;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.startsWith;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -43,10 +44,6 @@ import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.VerifyException;
 import com.google.common.collect.ImmutableList;
@@ -57,11 +54,15 @@ import google.registry.model.tld.Registry;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
@@ -69,17 +70,9 @@ import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.mockito.junit.jupiter.MockitoSettings;
-import org.mockito.quality.Strictness;
 
 /** Unit tests for {@link NordnUploadAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnUploadActionTest {
 
   private static final String CLAIMS_CSV =
@@ -101,29 +94,30 @@ class NordnUploadActionTest {
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
-
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
+  
   private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z"));
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnUploadAction action = new NordnUploadAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
     inject.setStaticField(Ofy.class, "clock", clock);
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
-    when(httpResponse.getContent()).thenReturn("Success".getBytes(US_ASCII));
-    when(httpResponse.getResponseCode()).thenReturn(SC_ACCEPTED);
-    when(httpResponse.getHeadersUncombined())
-        .thenReturn(ImmutableList.of(new HTTPHeader(LOCATION, "http://trololol")));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream("Success".getBytes(UTF_8)));
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_ACCEPTED);
+    when(httpUrlConnection.getHeaderField(LOCATION)).thenReturn("http://trololol");
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
     persistResource(loadRegistrar("TheRegistrar").asBuilder().setIanaIdentifier(99999L).build());
     createTld("tld");
     persistResource(Registry.get("tld").asBuilder().setLordnUsername("lolcat").build());
     action.clock = clock;
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.phase = "claims";
     action.taskQueueUtils = new TaskQueueUtils(new Retrier(new FakeSleeper(clock), 3));
@@ -133,7 +127,6 @@ class NordnUploadActionTest {
     action.retrier = new Retrier(new FakeSleeper(clock), 3);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv() {
     List<TaskHandle> tasks =
@@ -145,7 +138,6 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_dedupesDuplicates() {
     List<TaskHandle> tasks =
@@ -158,14 +150,12 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_doesntFailOnEmptyTasks() {
     assertThat(NordnUploadAction.convertTasksToCsv(ImmutableList.of(), clock.nowUtc(), "col1,col2"))
         .isEqualTo("1,2010-05-01T10:11:12.000Z,0\ncol1,col2\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_throwsNpeOnNullTasks() {
     assertThrows(
@@ -173,7 +163,6 @@ class NordnUploadActionTest {
         () -> NordnUploadAction.convertTasksToCsv(null, clock.nowUtc(), "header"));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @SuppressWarnings("unchecked")
   @Test
   void test_loadAllTasks_retryLogic_thirdTrysTheCharm() {
@@ -186,7 +175,6 @@ class NordnUploadActionTest {
     assertThat(action.loadAllTasks(queue, "tld")).containsExactly(task);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_loadAllTasks_retryLogic_allFailures() {
     Queue queue = mock(Queue.class);
@@ -201,47 +189,47 @@ class NordnUploadActionTest {
   void testRun_claimsMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
   }
 
   @Test
   void testRun_sunriseMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistSunriseModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
   }
 
   @Test
   void testRun_usesMultipartContentType() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), CONTENT_TYPE).get())
-        .startsWith("multipart/form-data; boundary=");
+    verify(httpUrlConnection)
+        .setRequestProperty(eq(CONTENT_TYPE), startsWith("multipart/form-data; boundary="));
+    verify(httpUrlConnection).setRequestMethod("POST");
   }
 
   @Test
-  void testRun_hasPassword_setsAuthorizationHeader() throws Exception {
+  void testRun_hasPassword_setsAuthorizationHeader() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+    verify(httpUrlConnection)
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testRun_noPassword_doesntSendAuthorizationHeader() throws Exception {
+  void testRun_noPassword_doesntSendAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
-  void testRun_claimsMode_payloadMatchesClaimsCsv() throws Exception {
+  void testRun_claimsMode_payloadMatchesClaimsCsv() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(CLAIMS_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(CLAIMS_CSV);
   }
 
   @Test
@@ -257,19 +245,19 @@ class NordnUploadActionTest {
   }
 
   @Test
-  void testRun_sunriseMode_payloadMatchesSunriseCsv() throws Exception {
+  void testRun_sunriseMode_payloadMatchesSunriseCsv() {
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
   void test_noResponseContent_stillWorksNormally() throws Exception {
     // Returning null only affects logging.
-    when(httpResponse.getContent()).thenReturn(null);
+    when(httpUrlConnection.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[] {}));
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
@@ -284,7 +272,6 @@ class NordnUploadActionTest {
             .header(CONTENT_TYPE, FORM_DATA.toString()));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void testFailure_nullRegistryUser() {
     persistClaimsModeDomain();
@@ -293,17 +280,11 @@ class NordnUploadActionTest {
     assertThat(thrown).hasMessageThat().contains("lordnUsername is not set for tld.");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
-  void testFetchFailure() {
+  void testFetchFailure() throws Exception {
     persistClaimsModeDomain();
-    when(httpResponse.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
-    assertThrows(UrlFetchException.class, action::run);
-  }
-
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
+    assertThrows(UrlConnectionException.class, action::run);
   }
 
   private void persistClaimsModeDomain() {

core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java

@@ -16,39 +16,34 @@ package google.registry.tmch;
 
 import static com.google.common.net.HttpHeaders.AUTHORIZATION;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.model.tld.Registry;
 import google.registry.request.HttpException.ConflictException;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeUrlConnectionService;
+import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Unit tests for {@link NordnVerifyAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnVerifyActionTest {
 
   private static final String LOG_ACCEPTED =
@@ -81,84 +76,83 @@ class NordnVerifyActionTest {
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
   private final FakeResponse response = new FakeResponse();
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnVerifyAction action = new NordnVerifyAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(LOG_ACCEPTED.getBytes(UTF_8));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
     createTld("gtld");
     persistResource(Registry.get("gtld").asBuilder().setLordnUsername("lolcat").build());
     action.tld = "gtld";
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ACCEPTED.getBytes(UTF_8)));
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.response = response;
     action.url = new URL("http://127.0.0.1/blobio");
   }
 
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
-  }
-
   @Test
   void testSuccess_sendHttpRequest_urlIsCorrect() throws Exception {
     action.run();
-    assertThat(getCapturedHttpRequest().getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
   }
 
   @Test
-  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() throws Exception {
+  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() {
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+    verify(httpUrlConnection)
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() throws Exception {
+  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
   void successVerifyRejected() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_REJECTED.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_REJECTED.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.getStatus()).isEqualTo(LordnLog.Status.REJECTED);
   }
 
   @Test
   void successVerifyWarnings() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_WARNINGS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_WARNINGS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
   void successVerifyErrors() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_ERRORS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ERRORS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
-  void failureVerifyUnauthorized() {
-    when(httpResponse.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
+  void failureVerifyUnauthorized() throws Exception {
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
     assertThrows(Exception.class, action::run);
   }
 
   @Test
-  void failureVerifyNotReady() {
-    when(httpResponse.getResponseCode()).thenReturn(SC_NO_CONTENT);
+  void failureVerifyNotReady() throws Exception {
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_NO_CONTENT);
     ConflictException thrown = assertThrows(ConflictException.class, action::run);
     assertThat(thrown).hasMessageThat().contains("Not ready");
   }

core/src/test/java/google/registry/tmch/TmchActionTestCase.java

@@ -15,21 +15,19 @@
 package google.registry.tmch;
 
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.FakeClock;
+import google.registry.testing.FakeUrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Common code for unit tests of classes that extend {@link Marksdb}. */
@@ -46,19 +44,19 @@ abstract class TmchActionTestCase {
   @RegisterExtension
   public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension();
 
-  @Mock URLFetchService fetchService;
-  @Mock HTTPResponse httpResponse;
-  @Captor ArgumentCaptor<HTTPRequest> httpRequest;
-
   final FakeClock clock = new FakeClock();
   final Marksdb marksdb = new Marksdb();
 
+  protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  protected final ArrayList<URL> connectedUrls = new ArrayList<>();
+  protected FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection, connectedUrls);
+
   @BeforeEach
   public void beforeEachTmchActionTestCase() throws Exception {
-    marksdb.fetchService = fetchService;
     marksdb.tmchMarksdbUrl = MARKSDB_URL;
     marksdb.marksdbPublicKey = TmchData.loadPublicKey(TmchTestData.loadBytes("pubkey"));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    marksdb.urlConnectionService = urlConnectionService;
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
   }
 }

core/src/test/java/google/registry/tmch/TmchCrlActionTest.java

@@ -22,6 +22,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
+import java.io.ByteArrayInputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.security.SignatureException;
@@ -44,19 +45,22 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testSuccess() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent()).thenReturn(
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     newTmchCrlAction(TmchCaMode.PRODUCTION).run();
-    verify(httpResponse).getContent();
-    verify(fetchService).fetch(httpRequest.capture());
-    assertThat(httpRequest.getValue().getURL().toString()).isEqualTo("http://sloth.lol/tmch.crl");
+    verify(httpUrlConnection).getInputStream();
+    assertThat(connectedUrls).containsExactly(new URL("http://sloth.lol/tmch.crl"));
   }
 
   @Test
   void testFailure_crlTooOld() throws Exception {
     clock.setTo(DateTime.parse("2020-01-01TZ"));
-    when(httpResponse.getContent())
-        .thenReturn(loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read()));
     TmchCrlAction action = newTmchCrlAction(TmchCaMode.PILOT);
     Exception e = assertThrows(Exception.class, action::run);
     assertThat(e).hasCauseThat().isInstanceOf(CRLException.class);
@@ -69,8 +73,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotSignedByRoot() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent())
-        .thenReturn(readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(SignatureException.class);
     assertThat(e).hasCauseThat().hasMessageThat().isEqualTo("Signature does not match.");
@@ -79,8 +85,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotYetValid() throws Exception {
     clock.setTo(DateTime.parse("1984-01-01TZ"));
-    when(httpResponse.getContent()).thenReturn(
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(CertificateNotYetValidException.class);
   }

core/src/test/java/google/registry/tmch/TmchDnlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static org.mockito.Mockito.times;
@@ -22,6 +23,8 @@ import static org.mockito.Mockito.when;
 
 import google.registry.model.tmch.ClaimsList;
 import google.registry.model.tmch.ClaimsListDao;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -39,15 +42,13 @@ class TmchDnlActionTest extends TmchActionTestCase {
   @Test
   void testDnl() throws Exception {
     assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
-    when(httpResponse.getContent())
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.csv").read()))
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.sig").read()));
     newTmchDnlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.csv");
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
+    verify(httpUrlConnection, times(2)).getInputStream();
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
+        .containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig");
 
     // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
     ClaimsList claimsList = ClaimsListDao.get();

core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.tmch.TmchTestData.loadBytes;
 import static org.mockito.Mockito.times;
@@ -21,6 +22,8 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.model.smd.SignedMarkRevocationList;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -42,15 +45,14 @@ class TmchSmdrlActionTest extends TmchActionTestCase {
     SignedMarkRevocationList smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isFalse();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();
-    when(httpResponse.getContent())
-        .thenReturn(loadBytes("smdrl-latest.csv").read())
-        .thenReturn(loadBytes("smdrl-latest.sig").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.csv").read()))
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.sig").read()));
     newTmchSmdrlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.csv");
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.sig");
+    verify(httpUrlConnection, times(2)).getInputStream();
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
+        .containsExactly(
+            MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig");
     smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isTrue();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();

core/src/test/java/google/registry/tools/CreateDomainCommandTest.java

@@ -50,7 +50,8 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
         "--admins=crr-admin",
         "--techs=crr-tech",
         "--password=2fooBAR",
-        "--ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--ds_records=60485 5  2  D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
         "example.tld");
     eppVerifier.verifySent("domain_create_complete.xml");
@@ -66,7 +67,8 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
         "--admins=crr-admin",
         "--techs=crr-tech",
         "--password=2fooBAR",
-        "--ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--ds_records=60485 5  2  D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
         "example.tld");
     eppVerifier.verifySent("domain_create_complete.xml");
@@ -330,6 +332,22 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized digest type: 3");
   }
 
+  @Test
+  void testFailure_invalidDigestLength() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar",
+                    "--registrant=crr-admin",
+                    "--admins=crr-admin",
+                    "--techs=crr-tech",
+                    "--ds_records=1 2 1 abcd",
+                    "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record has an invalid digest length: ABCD");
+  }
+
   @Test
   void testFailure_invalidAlgorithm() {
     IllegalArgumentException thrown =
@@ -341,7 +359,9 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
                     "--registrant=crr-admin",
                     "--admins=crr-admin",
                     "--techs=crr-tech",
-                    "--ds_records=1 999 4 abcd",
+                    "--ds_records=1 999 4"
+                        + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1C"
+                        + "CB126255D196047DFEDF17A0A9",
                     "example.tld"));
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized algorithm: 999");
   }

core/src/test/java/google/registry/tools/UniformRapidSuspensionCommandTest.java

@@ -78,7 +78,7 @@ class UniformRapidSuspensionCommandTest
     runCommandForced(
         "--domain_name=evil.tld",
         "--hosts=urs1.example.com,urs2.example.com",
-        "--dsdata=1 1 1 abcd",
+        "--dsdata=1 1 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--renew_one_year=false");
     eppVerifier
         .expectRegistrarId("CharlestonRoad")
@@ -149,7 +149,7 @@ class UniformRapidSuspensionCommandTest
     runCommandForced(
         "--domain_name=evil.tld",
         "--hosts=urs1.example.com,urs2.example.com",
-        "--dsdata=1 1 1 abcd",
+        "--dsdata=1 1 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--renew_one_year=false");
     eppVerifier
         .expectRegistrarId("CharlestonRoad")

core/src/test/java/google/registry/tools/UpdateDomainCommandTest.java

@@ -86,12 +86,14 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--remove_nameservers=ns3.zdns.google,ns4.zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld");
@@ -106,12 +108,14 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--remove_nameservers=ns[3-4].zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld");
@@ -128,12 +132,14 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--remove_nameservers=ns[3-4].zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld",
@@ -186,7 +192,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "example.tld");
     eppVerifier.verifySent("domain_update_add.xml");
   }
@@ -199,7 +206,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "example.tld");
     eppVerifier.verifySent("domain_update_remove.xml");
   }
@@ -277,7 +285,11 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
 
   @TestOfyAndSql
   void testSuccess_setDsRecords() throws Exception {
-    runCommandForced("--client=NewRegistrar", "--ds_records=1 2 2 abcd,4 5 1 EF01", "example.tld");
+    runCommandForced(
+        "--client=NewRegistrar",
+        "--ds_records=1 2 2 D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
+        "example.tld");
     eppVerifier.verifySent("domain_update_set_ds_records.xml");
   }
 
@@ -285,7 +297,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
   void testSuccess_setDsRecords_withUnneededClear() throws Exception {
     runCommandForced(
         "--client=NewRegistrar",
-        "--ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--ds_records=1 2 2 D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--clear_ds_records",
         "example.tld");
     eppVerifier.verifySent("domain_update_set_ds_records.xml");
@@ -636,7 +649,10 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             IllegalArgumentException.class,
             () ->
                 runCommandForced(
-                    "--client=NewRegistrar", "--add_ds_records=1 299 2 abcd", "example.tld"));
+                    "--client=NewRegistrar",
+                    "--add_ds_records=1 299 2"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
+                    "example.tld"));
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized algorithm: 299");
   }
 
@@ -647,10 +663,29 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             IllegalArgumentException.class,
             () ->
                 runCommandForced(
-                    "--client=NewRegistrar", "--add_ds_records=1 2 3 abcd", "example.tld"));
+                    "--client=NewRegistrar",
+                    "--add_ds_records=1 2 3"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
+                    "example.tld"));
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized digest type: 3");
   }
 
+  @TestOfyAndSql
+  void testFailure_invalidDigestLength() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar",
+                    "--registrant=crr-admin",
+                    "--admins=crr-admin",
+                    "--techs=crr-tech",
+                    "--ds_records=1 2 1 abcd",
+                    "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record has an invalid digest length: ABCD");
+  }
+
   @TestOfyAndSql
   void testFailure_provideDsRecordsAndAddDsRecords() {
     IllegalArgumentException thrown =
@@ -659,8 +694,9 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--add_ds_records=1 2 2 abcd",
-                    "--ds_records=4 5 1 EF01",
+                    "--add_ds_records=1 2 2"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
+                    "--ds_records=4 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
                     "example.tld"));
     assertThat(thrown)
         .hasMessageThat()
@@ -677,8 +713,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--remove_ds_records=7 8 1 12ab",
-                    "--ds_records=4 5 1 EF01",
+                    "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
+                    "--ds_records=4 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
                     "example.tld"));
     assertThat(thrown)
         .hasMessageThat()
@@ -695,7 +731,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--add_ds_records=1 2 2 abcd",
+                    "--add_ds_records=1 2 2"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
                     "--clear_ds_records",
                     "example.tld"));
     assertThat(thrown)
@@ -713,7 +750,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--remove_ds_records=7 8 1 12ab",
+                    "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
                     "--clear_ds_records",
                     "example.tld"));
     assertThat(thrown)

core/src/test/java/google/registry/webdriver/TestServerExtension.java

@@ -33,6 +33,7 @@ import google.registry.testing.UserInfo;
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
@@ -53,6 +54,8 @@ import org.junit.jupiter.api.extension.ExtensionContext;
  */
 public final class TestServerExtension implements BeforeEachCallback, AfterEachCallback {
 
+  private static final Duration SERVER_STATUS_POLLING_INTERVAL = Duration.ofSeconds(5);
+
   private final ImmutableList<Fixture> fixtures;
   private final AppEngineExtension appEngineExtension;
   private final BlockingQueue<FutureTask<?>> jobs = new LinkedBlockingDeque<>();
@@ -107,8 +110,11 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
     serverThread = new Thread(server);
     synchronized (this) {
       serverThread.start();
-      while (!server.isRunning) {
-        this.wait();
+      while (server.serverStatus.equals(ServerStatus.NOT_STARTED)) {
+        this.wait(SERVER_STATUS_POLLING_INTERVAL.toMillis());
+      }
+      if (server.serverStatus.equals(ServerStatus.FAILED)) {
+        throw new RuntimeException("TestServer failed to start. See log for details.");
       }
     }
   }
@@ -163,6 +169,12 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
     return job.get();
   }
 
+  enum ServerStatus {
+    NOT_STARTED,
+    RUNNING,
+    FAILED
+  }
+
   private final class Server implements Runnable {
 
     private ExtensionContext context;
@@ -171,7 +183,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
       this.context = context;
     }
 
-    private volatile boolean isRunning = false;
+    private volatile ServerStatus serverStatus = ServerStatus.NOT_STARTED;
 
     @Override
     public void run() {
@@ -185,6 +197,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
           appEngineExtension.afterEach(context);
         }
       } catch (Throwable e) {
+        serverStatus = ServerStatus.FAILED;
         throw new RuntimeException(e);
       }
     }
@@ -196,7 +209,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
       testServer.start();
       System.out.printf("TestServerExtension is listening on: %s\n", testServer.getUrl("/"));
       synchronized (TestServerExtension.this) {
-        isRunning = true;
+        serverStatus = ServerStatus.RUNNING;
         TestServerExtension.this.notify();
       }
       try {

core/src/test/resources/google/registry/flows/domain/domain_create_dsdata_8_records.xml

@@ -26,49 +26,49 @@
           <secDNS:keyTag>12345</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12346</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12347</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12348</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12349</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12350</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12351</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12352</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
       </secDNS:create>
     </extension>

core/src/test/resources/google/registry/flows/domain/domain_create_dsdata_no_maxsiglife.xml

@@ -26,7 +26,7 @@
           <secDNS:keyTag>12345</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
       </secDNS:create>
     </extension>

core/src/test/resources/google/registry/flows/domain/domain_update_dsdata_add_rem.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12345</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B33C99B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -24,7 +24,7 @@
             <secDNS:keyTag>12346</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B44C39B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/flows/domain/domain_update_dsdata_add_rem_same.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12345</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B33C99B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -24,7 +24,7 @@
             <secDNS:keyTag>12345</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B33C99B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/flows/domain/domain_update_dsdata_rem.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12346</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B44C39B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
       </secDNS:update>

core/src/test/resources/google/registry/flows/domain_update_dsdata_add.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12346</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B44C39B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_create_complete.xml

@@ -26,13 +26,13 @@
           <secDNS:keyTag>1</secDNS:keyTag>
           <secDNS:alg>2</secDNS:alg>
           <secDNS:digestType>2</secDNS:digestType>
-          <secDNS:digest>ABCD</secDNS:digest>
+          <secDNS:digest>9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>4</secDNS:keyTag>
           <secDNS:alg>5</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>EF01</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>60485</secDNS:keyTag>

core/src/test/resources/google/registry/tools/server/domain_update_add.xml

@@ -23,13 +23,13 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
             <secDNS:digestType>2</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>EF01</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_update_complete.xml

@@ -38,13 +38,13 @@
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>12AB</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>6</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>4</secDNS:digestType>
-            <secDNS:digest>34CD</secDNS:digest>
+            <secDNS:digest>768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -52,13 +52,13 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
             <secDNS:digestType>2</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>EF01</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_update_complete_abc.xml

@@ -38,13 +38,13 @@
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>12AB</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>6</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>4</secDNS:digestType>
-            <secDNS:digest>34CD</secDNS:digest>
+            <secDNS:digest>768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -52,13 +52,13 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
             <secDNS:digestType>2</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>EF01</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_update_remove.xml

@@ -22,13 +22,13 @@
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>12AB</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>6</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>4</secDNS:digestType>
-            <secDNS:digest>34CD</secDNS:digest>
+            <secDNS:digest>768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_update_set_ds_records.xml

@@ -17,13 +17,13 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
             <secDNS:digestType>2</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>EF01</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/uniform_rapid_suspension.xml

@@ -31,7 +31,7 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>1</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/uniform_rapid_suspension_with_client_hold.xml

@@ -32,7 +32,7 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>1</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

db/README.md

@@ -55,6 +55,12 @@ following steps:
     one. The generated SQL file from the previous step should help. New create
     table statements can be used as is, whereas alter table statements should be
     written to change any existing tables.
+    
+    Note that each incremental file MUST be limited to changes to a single
+    table (otherwise it may hit deadlock when applying on sandbox/production
+    where it'll be competing against live traffic that may also be locking said
+    tables but in a different order). It's OK to include these separate Flyway
+    scripts in a single PR.
 
     This script should be stored in a new file in the
     `db/src/main/resources/sql/flyway` folder using the naming pattern

db/src/main/resources/sql/er_diagram/full_er_diagram.html

@@ -261,11 +261,11 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-     <td class="property_value">2022-02-03 18:23:54.985476</td> 
+     <td class="property_value">2022-03-09 22:28:56.561494</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V105__add_index_on_host_name_in_host_table.sql</td>
+     <td id="lastFlywayFile" class="property_value">V112__add_billingrecurrence_missing_indexes.sql</td>
     </tr>
    </tbody>
   </table> 
@@ -284,7 +284,7 @@ td.section {
      generated on
     </text> 
     <text text-anchor="start" x="4755.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2022-02-03 18:23:54.985476
+     2022-03-09 22:28:56.561494
     </text> 
     <polygon fill="none" stroke="#888888" points="4668.02,-4 4668.02,-44 4933.02,-44 4933.02,-4 4668.02,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 
@@ -6899,6 +6899,42 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idx4ytbe5f3b39trsd4okx5ijhs4</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">billing_event_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxku0fopwyvd57ebo8bf0jg9xo2</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">billing_recurrence_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxl8vobbecsd32k4ksavdfx8st6</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">domain_repo_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">idxeokttmxtpq2hohcioe5t2242b</td> 
      <td class="description right">[non-unique index]</td> 
@@ -7170,6 +7206,18 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idx6ebt3nwk5ocvnremnhnlkl6ff</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">cancellation_matching_billing_recurrence_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">idxhmv411mdqo5ibn4vy7ykxpmlv</td> 
      <td class="description right">[non-unique index]</td> 
@@ -7182,6 +7230,18 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxbgfmveqa7e5hn689koikwn70r</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">domain_repo_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">idx73l103vc5900ig3p4odf0cngt</td> 
      <td class="description right">[non-unique index]</td> 
@@ -7418,6 +7478,18 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxoqttafcywwdn41um6kwlt0n8b</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">domain_repo_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">idxjny8wuot75b5e6p38r47wdawu</td> 
      <td class="description right">[non-unique index]</td> 
@@ -9490,6 +9562,42 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxsfci08jgsymxy6ovh4k7r358c</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">billing_recurrence_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
+    <tr> 
+     <td colspan="2" class="name">idx3y3k7m2bkgahm9sixiohgyrga</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">transfer_billing_event_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxcju58vqascbpve1t7fem53ctl</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">transfer_billing_recurrence_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">idxnb02m43jcx24r64n8rbg22u4q</td> 
      <td class="description right">[non-unique index]</td> 
@@ -10512,6 +10620,18 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxjw3rwtfrexyq53x9vu7qghrdt</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">host_repo_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">ukat9erbh52e4lg3jw6ai9wkjj9</td> 
      <td class="description right">[unique index]</td> 
@@ -10804,6 +10924,30 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxbgssjudpm428mrv0xfpvgifps</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">billing_event_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
+    <tr> 
+     <td colspan="2" class="name">idx5u5m6clpk3nktrvtyy5umacb6</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">billing_recurrence_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">GracePeriod_pkey</td> 
      <td class="description right">[unique index]</td> 
@@ -11167,6 +11311,18 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxrc77s1ndiemi2vwwudchye214</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">inet_addresses</td> 
+     <td class="minwidth">unknown</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">idxkpkh68n6dy5v51047yr6b0e9l</td> 
      <td class="description right">[non-unique index]</td> 

db/src/main/resources/sql/flyway.txt

@@ -103,3 +103,10 @@ V102__add_indexes_to_domain_history_sub_tables.sql
 V103__creation_time_not_null.sql
 V104__add_transfer_response_host_id_to_poll_message.sql
 V105__add_index_on_host_name_in_host_table.sql
+V106__add_domain_missing_indexes_from_query_analyzer.sql
+V107__add_billingevent_missing_indexes.sql
+V108__add_host_indexes_for_whois.sql
+V109__add_domain_host_missing_indexes_from_query_analyzer.sql
+V110__add_graceperiod_missing_indexes_from_query_analyzer.sql
+V111__add_billingcancellation_missing_indexes.sql
+V112__add_billingrecurrence_missing_indexes.sql

db/src/main/resources/sql/flyway/V106__add_domain_missing_indexes_from_query_analyzer.sql

@@ -0,0 +1,20 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS IDX3y3k7m2bkgahm9sixiohgyrga
+  ON "Domain" (transfer_billing_event_id);
+CREATE INDEX IF NOT EXISTS IDXsfci08jgsymxy6ovh4k7r358c
+  ON "Domain" (billing_recurrence_id);
+CREATE INDEX IF NOT EXISTS IDXcju58vqascbpve1t7fem53ctl
+  ON "Domain" (transfer_billing_recurrence_id);

db/src/main/resources/sql/flyway/V107__add_billingevent_missing_indexes.sql

@@ -0,0 +1,18 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed ON an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS IDX6ebt3nwk5ocvnremnhnlkl6ff
+  ON "BillingEvent" (cancellation_matching_billing_recurrence_id);
+CREATE INDEX IF NOT EXISTS IDXbgfmveqa7e5hn689koikwn70r
+  ON "BillingEvent" (domain_repo_id);

db/src/main/resources/sql/flyway/V108__add_host_indexes_for_whois.sql

@@ -0,0 +1,16 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS IDXrc77s1ndiemi2vwwudchye214
+  ON "Host" USING GIN (inet_addresses);

db/src/main/resources/sql/flyway/V109__add_domain_host_missing_indexes_from_query_analyzer.sql

@@ -0,0 +1,16 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS IDXjw3rwtfrexyq53x9vu7qghrdt
+  ON "DomainHost" (host_repo_id);

db/src/main/resources/sql/flyway/V110__add_graceperiod_missing_indexes_from_query_analyzer.sql

@@ -0,0 +1,18 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS IDXbgssjudpm428mrv0xfpvgifps
+  ON "GracePeriod" (billing_event_id);
+CREATE INDEX IF NOT EXISTS IDX5u5m6clpk3nktrvtyy5umacb6
+  ON "GracePeriod" (billing_recurrence_id);

db/src/main/resources/sql/flyway/V111__add_billingcancellation_missing_indexes.sql

@@ -0,0 +1,20 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS IDX4ytbe5f3b39trsd4okx5ijhs4
+  ON "BillingCancellation" (billing_event_id);
+CREATE INDEX IF NOT EXISTS IDXku0fopwyvd57ebo8bf0jg9xo2
+  ON "BillingCancellation" (billing_recurrence_id);
+CREATE INDEX IF NOT EXISTS IDXl8vobbecsd32k4ksavdfx8st6
+  ON "BillingCancellation" (domain_repo_id);

db/src/main/resources/sql/flyway/V112__add_billingrecurrence_missing_indexes.sql

@@ -0,0 +1,16 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS IDXoqttafcywwdn41um6kwlt0n8b
+  ON "BillingRecurrence" (domain_repo_id);

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -1532,6 +1532,13 @@ CREATE INDEX idx1rcgkdd777bpvj0r94sltwd5y ON public."Domain" USING btree (domain
 CREATE INDEX idx2exdfbx6oiiwnhr8j6gjpqt2j ON public."BillingCancellation" USING btree (event_time);
 
 
+--
+-- Name: idx3y3k7m2bkgahm9sixiohgyrga; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx3y3k7m2bkgahm9sixiohgyrga ON public."Domain" USING btree (transfer_billing_event_id);
+
+
 --
 -- Name: idx3y752kr9uh4kh6uig54vemx0l; Type: INDEX; Schema: public; Owner: -
 --
@@ -1539,6 +1546,13 @@ CREATE INDEX idx2exdfbx6oiiwnhr8j6gjpqt2j ON public."BillingCancellation" USING
 CREATE INDEX idx3y752kr9uh4kh6uig54vemx0l ON public."Contact" USING btree (creation_time);
 
 
+--
+-- Name: idx4ytbe5f3b39trsd4okx5ijhs4; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx4ytbe5f3b39trsd4okx5ijhs4 ON public."BillingCancellation" USING btree (billing_event_id);
+
+
 --
 -- Name: idx5mnf0wn20tno4b9do88j61klr; Type: INDEX; Schema: public; Owner: -
 --
@@ -1546,6 +1560,13 @@ CREATE INDEX idx3y752kr9uh4kh6uig54vemx0l ON public."Contact" USING btree (creat
 CREATE INDEX idx5mnf0wn20tno4b9do88j61klr ON public."Domain" USING btree (deletion_time);
 
 
+--
+-- Name: idx5u5m6clpk3nktrvtyy5umacb6; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx5u5m6clpk3nktrvtyy5umacb6 ON public."GracePeriod" USING btree (billing_recurrence_id);
+
+
 --
 -- Name: idx5yfbr88439pxw0v3j86c74fp8; Type: INDEX; Schema: public; Owner: -
 --
@@ -1560,6 +1581,13 @@ CREATE INDEX idx5yfbr88439pxw0v3j86c74fp8 ON public."BillingEvent" USING btree (
 CREATE INDEX idx67qwkjtlq5q8dv6egtrtnhqi7 ON public."HostHistory" USING btree (history_modification_time);
 
 
+--
+-- Name: idx6ebt3nwk5ocvnremnhnlkl6ff; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx6ebt3nwk5ocvnremnhnlkl6ff ON public."BillingEvent" USING btree (cancellation_matching_billing_recurrence_id);
+
+
 --
 -- Name: idx6py6ocrab0ivr76srcd2okpnq; Type: INDEX; Schema: public; Owner: -
 --
@@ -1637,6 +1665,20 @@ CREATE INDEX idxaro1omfuaxjwmotk3vo00trwm ON public."DomainHistory" USING btree
 CREATE INDEX idxaydgox62uno9qx8cjlj5lauye ON public."PollMessage" USING btree (event_time);
 
 
+--
+-- Name: idxbgfmveqa7e5hn689koikwn70r; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxbgfmveqa7e5hn689koikwn70r ON public."BillingEvent" USING btree (domain_repo_id);
+
+
+--
+-- Name: idxbgssjudpm428mrv0xfpvgifps; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxbgssjudpm428mrv0xfpvgifps ON public."GracePeriod" USING btree (billing_event_id);
+
+
 --
 -- Name: idxbn8t4wp85fgxjl8q4ctlscx55; Type: INDEX; Schema: public; Owner: -
 --
@@ -1644,6 +1686,13 @@ CREATE INDEX idxaydgox62uno9qx8cjlj5lauye ON public."PollMessage" USING btree (e
 CREATE INDEX idxbn8t4wp85fgxjl8q4ctlscx55 ON public."Contact" USING btree (current_sponsor_registrar_id);
 
 
+--
+-- Name: idxcju58vqascbpve1t7fem53ctl; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxcju58vqascbpve1t7fem53ctl ON public."Domain" USING btree (transfer_billing_recurrence_id);
+
+
 --
 -- Name: idxd01j17vrpjxaerxdmn8bwxs7s; Type: INDEX; Schema: public; Owner: -
 --
@@ -1714,6 +1763,13 @@ CREATE INDEX idxj77pfwhui9f0i7wjq6lmibovj ON public."HostHistory" USING btree (h
 CREATE INDEX idxjny8wuot75b5e6p38r47wdawu ON public."BillingRecurrence" USING btree (recurrence_time_of_year);
 
 
+--
+-- Name: idxjw3rwtfrexyq53x9vu7qghrdt; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxjw3rwtfrexyq53x9vu7qghrdt ON public."DomainHost" USING btree (host_repo_id);
+
+
 --
 -- Name: idxkjt9yaq92876dstimd93hwckh; Type: INDEX; Schema: public; Owner: -
 --
@@ -1735,6 +1791,20 @@ CREATE INDEX idxknk8gmj7s47q56cwpa6rmpt5l ON public."HostHistory" USING btree (h
 CREATE INDEX idxkpkh68n6dy5v51047yr6b0e9l ON public."Host" USING btree (host_name);
 
 
+--
+-- Name: idxku0fopwyvd57ebo8bf0jg9xo2; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxku0fopwyvd57ebo8bf0jg9xo2 ON public."BillingCancellation" USING btree (billing_recurrence_id);
+
+
+--
+-- Name: idxl8vobbecsd32k4ksavdfx8st6; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxl8vobbecsd32k4ksavdfx8st6 ON public."BillingCancellation" USING btree (domain_repo_id);
+
+
 --
 -- Name: idxlrq7v63pc21uoh3auq6eybyhl; Type: INDEX; Schema: public; Owner: -
 --
@@ -1777,6 +1847,13 @@ CREATE INDEX idxo1xdtpij2yryh0skxe9v91sep ON public."ContactHistory" USING btree
 CREATE INDEX idxoqd7n4hbx86hvlgkilq75olas ON public."Contact" USING btree (contact_id);
 
 
+--
+-- Name: idxoqttafcywwdn41um6kwlt0n8b; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxoqttafcywwdn41um6kwlt0n8b ON public."BillingRecurrence" USING btree (domain_repo_id);
+
+
 --
 -- Name: idxp3usbtvk0v1m14i5tdp4xnxgc; Type: INDEX; Schema: public; Owner: -
 --
@@ -1812,6 +1889,13 @@ CREATE INDEX idxqa3g92jc17e8dtiaviy4fet4x ON public."BillingCancellation" USING
 CREATE INDEX idxr22ciyccwi9rrqmt1ro0s59qf ON public."Domain" USING btree (tech_contact);
 
 
+--
+-- Name: idxrc77s1ndiemi2vwwudchye214; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxrc77s1ndiemi2vwwudchye214 ON public."Host" USING gin (inet_addresses);
+
+
 --
 -- Name: idxrh4xmrot9bd63o382ow9ltfig; Type: INDEX; Schema: public; Owner: -
 --
@@ -1826,6 +1910,13 @@ CREATE INDEX idxrh4xmrot9bd63o382ow9ltfig ON public."DomainHistory" USING btree
 CREATE INDEX idxrwl38wwkli1j7gkvtywi9jokq ON public."Domain" USING btree (tld);
 
 
+--
+-- Name: idxsfci08jgsymxy6ovh4k7r358c; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxsfci08jgsymxy6ovh4k7r358c ON public."Domain" USING btree (billing_recurrence_id);
+
+
 --
 -- Name: idxsu1nam10cjes9keobapn5jvxj; Type: INDEX; Schema: public; Owner: -
 --

util/src/main/java/google/registry/util/UrlConnectionException.java

@@ -0,0 +1,71 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+
+/**
+ * Used when HTTP requests return a bad response, with troubleshooting info.
+ *
+ * <p>This class displays lots of helpful troubleshooting information.
+ */
+public class UrlConnectionException extends RuntimeException {
+
+  private final HttpURLConnection connection;
+
+  public UrlConnectionException(String message, HttpURLConnection connection) {
+    super(message);
+    this.connection = connection;
+  }
+
+  @Override
+  public String getMessage() {
+    byte[] resultContent;
+    int responseCode;
+    try {
+      resultContent = ByteStreams.toByteArray(connection.getInputStream());
+      responseCode = connection.getResponseCode();
+    } catch (IOException e) {
+      resultContent = new byte[] {};
+      responseCode = 0;
+    }
+    StringBuilder result =
+        new StringBuilder(2048 + resultContent.length)
+            .append(
+                String.format(
+                    "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\n",
+                    getClass().getSimpleName(),
+                    super.getMessage(),
+                    responseCode,
+                    connection.getURL().toString()));
+    connection
+        .getRequestProperties()
+        .forEach(
+            (key, value) -> {
+              result.append(key);
+              result.append(": ");
+              result.append(value);
+              result.append('\n');
+            });
+    result.append(">>>\n");
+    result.append(new String(resultContent, UTF_8));
+    result.append("\n<<<");
+    return result.toString();
+  }
+}

util/src/main/java/google/registry/util/UrlFetchException.java

@@ -1,63 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.util;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-
-/**
- * Exception for when App Engine HTTP requests return a bad response.
- *
- * <p>This class displays lots of helpful troubleshooting information.
- */
-public class UrlFetchException extends RuntimeException {
-
-  private final HTTPRequest req;
-  private final HTTPResponse rsp;
-
-  public UrlFetchException(String message, HTTPRequest req, HTTPResponse rsp) {
-    super(message);
-    this.req = checkNotNull(req, "req");
-    this.rsp = checkNotNull(rsp, "rsp");
-  }
-
-  @Override
-  public String getMessage() {
-    StringBuilder res =
-        new StringBuilder(2048 + rsp.getContent().length)
-            .append(
-                String.format(
-                    "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\nX-Final-URL: %s\n",
-                    getClass().getSimpleName(),
-                    super.getMessage(),
-                    rsp.getResponseCode(),
-                    req.getURL().toString(),
-                    rsp.getFinalUrl()));
-    for (HTTPHeader header : rsp.getHeadersUncombined()) {
-      res.append(header.getName());
-      res.append(": ");
-      res.append(header.getValue());
-      res.append('\n');
-    }
-    res.append(">>>\n");
-    res.append(new String(rsp.getContent(), UTF_8));
-    res.append("\n<<<");
-    return res.toString();
-  }
-}