Remove static methods in back up actions (#1481)

* Remove static methods in back up actions

* Remove BigqueryPollJob helper class

* Add schedule time in task comparison

* Change payload type from byte[] to ByteString

build.gradle

@@ -55,7 +55,7 @@ plugins {
 
 node {
   download = true
-  version = "14.15.5"
+  version = "16.14.0"
   npmVersion = "6.14.11"
 }
 

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -28,11 +28,11 @@ import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
-import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 
 /**
  * Action that saves commit log checkpoints to Datastore and kicks off a diff export task.
@@ -57,7 +57,22 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   private static final String QUEUE_NAME = "export-commits";
 
-  @Inject Clock clock;
+  /**
+   * The amount of time enqueueing should be delayed.
+   *
+   * <p>The {@link ExportCommitLogDiffAction} is enqueued in {@link CommitLogCheckpointAction},
+   * which is inside a Datastore transaction that persists the checkpoint to be exported. After the
+   * switch to CloudTasks API, the task may be invoked before the Datastore transaction commits.
+   * When this happens, the checkpoint is not found which leads to {@link
+   * com.google.common.base.VerifyException}.
+   *
+   * <p>In order to invoke the task after the transaction commits, a reasonable delay should be
+   * added to each task. The latency of the request is mostly in the range of 4-6 seconds; Choosing
+   * a value 30% greater than the upper bound should solve the issue invoking a task before the
+   * transaction commits.
+   */
+  static final Duration ENQUEUE_DELAY_SECONDS = Duration.standardSeconds(8);
+
   @Inject CommitLogCheckpointStrategy strategy;
   @Inject CloudTasksUtils cloudTasksUtils;
 
@@ -96,14 +111,15 @@ public final class CommitLogCheckpointAction implements Runnable {
                   // Enqueue a diff task between previous and current checkpoints.
                   cloudTasksUtils.enqueue(
                       QUEUE_NAME,
-                      cloudTasksUtils.createPostTask(
+                      cloudTasksUtils.createPostTaskWithDelay(
                           ExportCommitLogDiffAction.PATH,
                           Service.BACKEND.toString(),
                           ImmutableMultimap.of(
                               LOWER_CHECKPOINT_TIME_PARAM,
                               lastWrittenTime.toString(),
                               UPPER_CHECKPOINT_TIME_PARAM,
-                              checkpoint.getCheckpointTime().toString())));
+                              checkpoint.getCheckpointTime().toString()),
+                          ENQUEUE_DELAY_SECONDS));
                   return true;
                 });
     return isCheckPointPersisted ? Optional.of(checkpoint) : Optional.empty();

core/src/main/java/google/registry/backup/SyncDatastoreToSqlSnapshotAction.java

@@ -29,6 +29,8 @@ import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Sleeper;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -114,9 +116,22 @@ public class SyncDatastoreToSqlSnapshotAction implements Runnable {
       response.setPayload(
           String.format(SUCCESS_RESPONSE_TEMPLATE, sqlSnapshotId, checkpoint.getCheckpointTime()));
       return;
-    } catch (Exception e) {
+    } catch (Throwable e) {
+      logger.atSevere().withCause(e).log("Failed to sync Datastore to SQL.");
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setPayload(e.getMessage());
+      response.setPayload(getStackTrace(e));
+    }
+  }
+
+  private static String getStackTrace(Throwable e) {
+    try {
+      ByteArrayOutputStream bis = new ByteArrayOutputStream();
+      PrintStream printStream = new PrintStream(bis);
+      e.printStackTrace(printStream);
+      printStream.close();
+      return bis.toString();
+    } catch (RuntimeException re) {
+      return re.getMessage();
     }
   }
 

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -22,14 +22,17 @@ import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Multimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
-import google.registry.util.AppEngineServiceUtils;
+import google.registry.request.Action.Service;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.Retrier;
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -58,25 +61,23 @@ public final class AsyncTaskEnqueuer {
   private static final Duration MAX_ASYNC_ETA = Duration.standardDays(30);
 
   private final Duration asyncDeleteDelay;
-  private final Queue asyncActionsPushQueue;
   private final Queue asyncDeletePullQueue;
   private final Queue asyncDnsRefreshPullQueue;
-  private final AppEngineServiceUtils appEngineServiceUtils;
   private final Retrier retrier;
 
+  private CloudTasksUtils cloudTasksUtils;
+
   @Inject
   public AsyncTaskEnqueuer(
-      @Named(QUEUE_ASYNC_ACTIONS) Queue asyncActionsPushQueue,
       @Named(QUEUE_ASYNC_DELETE) Queue asyncDeletePullQueue,
       @Named(QUEUE_ASYNC_HOST_RENAME) Queue asyncDnsRefreshPullQueue,
       @Config("asyncDeleteFlowMapreduceDelay") Duration asyncDeleteDelay,
-      AppEngineServiceUtils appEngineServiceUtils,
+      CloudTasksUtils cloudTasksUtils,
       Retrier retrier) {
-    this.asyncActionsPushQueue = asyncActionsPushQueue;
     this.asyncDeletePullQueue = asyncDeletePullQueue;
     this.asyncDnsRefreshPullQueue = asyncDnsRefreshPullQueue;
     this.asyncDeleteDelay = asyncDeleteDelay;
-    this.appEngineServiceUtils = appEngineServiceUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
     this.retrier = retrier;
   }
 
@@ -102,19 +103,17 @@ public final class AsyncTaskEnqueuer {
           entityKey, firstResave, MAX_ASYNC_ETA);
       return;
     }
-    logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
-    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
-    TaskOptions task =
-        TaskOptions.Builder.withUrl(ResaveEntityAction.PATH)
-            .method(Method.POST)
-            .header("Host", backendHostname)
-            .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.stringify())
-            .param(PARAM_REQUESTED_TIME, now.toString());
+    Multimap<String, String> params = ArrayListMultimap.create();
+    params.put(PARAM_RESOURCE_KEY, entityKey.stringify());
+    params.put(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
-      task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
+      params.put(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
     }
-    addTaskToQueueWithRetry(asyncActionsPushQueue, task);
+    logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
+    cloudTasksUtils.enqueue(
+        QUEUE_ASYNC_ACTIONS,
+        cloudTasksUtils.createPostTaskWithDelay(
+            ResaveEntityAction.PATH, Service.BACKEND.toString(), params, etaDuration));
   }
 
   /** Enqueues a task to asynchronously delete a contact or host, by key. */

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -68,6 +68,7 @@ import org.apache.beam.sdk.values.PDone;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 
 public class RdeIO {
 
@@ -261,6 +262,7 @@ public class RdeIO {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
     private static final long serialVersionUID = 5822176227753327224L;
+    private static final Duration ENQUEUE_DELAY = Duration.standardMinutes(1);
 
     private final CloudTasksUtils cloudTasksUtils;
 
@@ -271,6 +273,7 @@ public class RdeIO {
     @ProcessElement
     public void processElement(
         @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
+
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -296,26 +299,30 @@ public class RdeIO {
                 tm().put(Cursor.create(key.cursor(), newPosition, registry));
                 logger.atInfo().log(
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
-                RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), input.getValue());
                 // Enqueueing a task is a side effect that is not undone if the transaction rolls
                 // back. So this may result in multiple copies of the same task being processed.
                 // This is fine because the RdeUploadAction is guarded by a lock and tracks progress
-                // by cursor. The BrdaCopyAction writes a file to GCS, which is an atomic action.
+                // by cursor. The BrdaCopyAction writes a file to GCS, which is an atomic action. It
+                // is also guarded by a cursor to not run before the cursor is updated. We also
+                // include a delay to minimize the chance that the enqueued job executes before the
+                // transaction is committed, which triggers a retry.
                 if (key.mode() == RdeMode.FULL) {
                   cloudTasksUtils.enqueue(
                       RDE_UPLOAD_QUEUE,
-                      cloudTasksUtils.createPostTask(
+                      cloudTasksUtils.createPostTaskWithDelay(
                           RdeUploadAction.PATH,
                           Service.BACKEND.getServiceId(),
                           ImmutableMultimap.of(
                               RequestParameters.PARAM_TLD,
                               key.tld(),
                               RdeModule.PARAM_PREFIX,
-                              options.getJobName() + '/')));
+                              options.getJobName() + '/'),
+                          ENQUEUE_DELAY));
                 } else {
                   cloudTasksUtils.enqueue(
                       BRDA_QUEUE,
-                      cloudTasksUtils.createPostTask(
+                      cloudTasksUtils.createPostTaskWithDelay(
                           BrdaCopyAction.PATH,
                           Service.BACKEND.getServiceId(),
                           ImmutableMultimap.of(
@@ -324,7 +331,8 @@ public class RdeIO {
                               RdeModule.PARAM_WATERMARK,
                               key.watermark().toString(),
                               RdeModule.PARAM_PREFIX,
-                              options.getJobName() + '/')));
+                              options.getJobName() + '/'),
+                          ENQUEUE_DELAY));
                 }
               });
     }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -486,9 +486,9 @@ sslCertificateValidation:
   # The minimum number of days between two successive expiring notification emails.
   expirationWarningIntervalDays: 15
   # Text for expiring certificate notification email subject.
-  expirationWarningEmailSubjectText: "[Important] Expiring SSL certificate for Google Registry EPP connection"
-  # Text for expiring certificate notification email body that accepts 3 parameters:
-  # registrar name, certificate type, and expiration date, respectively.
+  expirationWarningEmailSubjectText: "Expiring SSL certificate for Example Registry EPP connection"
+  # Text for expiring certificate notification email body that accepts 4 parameters:
+  # registrar name, certificate type, expiration date and registrar id, respectively.
   expirationWarningEmailBodyText: |
     Dear %1$s,
 
@@ -496,25 +496,15 @@ sslCertificateValidation:
 
     Kindly update your production account certificate within the support console using the following steps:
 
-      1. Navigate to support.registry.google and login using your %4$s@registry.google credentials.
-          * If this is your first time logging in, you will be prompted to reset your password, so please keep your new password safe.
-          * If you are already logged in with some other Google account(s) but not your %4$s@registry.google account, you need to click on
-          “Add Account” and login using your %4$s@registry.google credentials.
+      1. Navigate to support.registry.example and login using your %4$s@registry.example credentials.
       2. Select “Settings > Security” from the left navigation bar.
       3. Click “Edit” on the top left corner.
-      4. Enter your full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----) in the box.
-      5. Click “Save”. If there are validation issues with the form, you will be prompted to fix them and click “Save” again.
+      4. Enter your full certificate string in the box.
+      5. Click “Save”.
 
-    A failover SSL certificate can also be added in order to prevent connection issues once your main certificate expires. Connecting with either of the certificates will work with our production EPP server.
-
-    Further information about our EPP connection requirements can be found in section 9.2 in the updated Technical Guide in your Google Drive folder.
-
-    Note that account certificate changes take a few minutes to become effective and that the existing connections will remain unaffected by the change.
-
-    If you also would like to update your OT&E account certificate, please send an email from your primary or technical contact to registry-support@google.com and include the full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).
 
     Regards,
-    Google Registry
+    Example Registry
 
   # The minimum number of bits an RSA key must contain.
   minimumRsaKeyLength: 2048

core/src/main/java/google/registry/export/BackupDatastoreAction.java

@@ -14,18 +14,22 @@
 
 package google.registry.export;
 
-import static google.registry.export.CheckBackupAction.enqueuePollTask;
 import static google.registry.request.Action.Method.POST;
 
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig;
 import google.registry.export.datastore.DatastoreAdmin;
 import google.registry.export.datastore.Operation;
 import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 
 /**
@@ -59,6 +63,8 @@ public class BackupDatastoreAction implements Runnable {
 
   @Inject DatastoreAdmin datastoreAdmin;
   @Inject Response response;
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
   @Inject
   BackupDatastoreAction() {}
@@ -73,8 +79,19 @@ public class BackupDatastoreAction implements Runnable {
               .execute();
 
       String backupName = backup.getName();
-      // Enqueue a poll task to monitor the backup and load REPORTING-related kinds into bigquery.
-      enqueuePollTask(backupName, AnnotatedEntities.getReportingKinds());
+      // Enqueue a poll task to monitor the backup for completion and load reporting-related kinds
+      // into bigquery.
+      cloudTasksUtils.enqueue(
+          CheckBackupAction.QUEUE,
+          cloudTasksUtils.createPostTaskWithDelay(
+              CheckBackupAction.PATH,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(
+                  CheckBackupAction.CHECK_BACKUP_NAME_PARAM,
+                  backupName,
+                  CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM,
+                  Joiner.on(',').join(AnnotatedEntities.getReportingKinds())),
+              CheckBackupAction.POLL_COUNTDOWN));
       String message =
           String.format(
               "Datastore backup started with name: %s\nSaving to %s",

core/src/main/java/google/registry/export/BigqueryPollJobAction.java

@@ -14,18 +14,14 @@
 
 package google.registry.export;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.bigquery.BigqueryUtils.toJobReferenceString;
 
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.model.Job;
 import com.google.api.services.bigquery.model.JobReference;
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
+import com.google.cloud.tasks.v2.Task;
 import com.google.common.flogger.FluentLogger;
+import com.google.protobuf.ByteString;
 import dagger.Lazy;
 import google.registry.request.Action;
 import google.registry.request.Header;
@@ -33,12 +29,10 @@ import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.HttpException.NotModifiedException;
 import google.registry.request.Payload;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
 import javax.inject.Inject;
 import org.joda.time.Duration;
 
@@ -67,11 +61,14 @@ public class BigqueryPollJobAction implements Runnable {
   static final Duration POLL_COUNTDOWN = Duration.standardSeconds(20);
 
   @Inject Bigquery bigquery;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject @Header(CHAINED_TASK_QUEUE_HEADER) Lazy<String> chainedQueueName;
   @Inject @Header(PROJECT_ID_HEADER) String projectId;
   @Inject @Header(JOB_ID_HEADER) String jobId;
-  @Inject @Payload byte[] payload;
+
+  @Inject @Payload ByteString payload;
+
   @Inject BigqueryPollJobAction() {}
 
   @Override
@@ -79,20 +76,25 @@ public class BigqueryPollJobAction implements Runnable {
     boolean jobOutcome =
         checkJobOutcome(); // Throws a NotModifiedException if the job hasn't completed.
     // If the job failed, do not enqueue the next step.
-    if (!jobOutcome || payload == null || payload.length == 0) {
+    if (!jobOutcome || payload == null || payload.size() == 0) {
       return;
     }
     // If there is a payload, it's a chained task, so enqueue it.
-    TaskOptions task;
+    Task task;
     try {
-      task = (TaskOptions) new ObjectInputStream(new ByteArrayInputStream(payload)).readObject();
+      task =
+          (Task)
+              new ObjectInputStream(new ByteArrayInputStream(payload.toByteArray())).readObject();
     } catch (ClassNotFoundException | IOException e) {
       throw new BadRequestException("Cannot deserialize task from payload", e);
     }
-    String taskName = taskQueueUtils.enqueue(getQueue(chainedQueueName.get()), task).getName();
+    Task enqueuedTask = cloudTasksUtils.enqueue(chainedQueueName.get(), task);
     logger.atInfo().log(
         "Added chained task %s for %s to queue %s: %s",
-        taskName, task.getUrl(), chainedQueueName.get(), task);
+        enqueuedTask.getName(),
+        enqueuedTask.getAppEngineHttpRequest().getRelativeUri(),
+        chainedQueueName.get(),
+        enqueuedTask);
   }
 
   /**
@@ -124,51 +126,4 @@ public class BigqueryPollJobAction implements Runnable {
     logger.atInfo().log("Bigquery job succeeded - %s.", jobRefString);
     return true;
   }
-
-
-  /** Helper class to enqueue a bigquery poll job. */
-  public static class BigqueryPollJobEnqueuer {
-
-    private final TaskQueueUtils taskQueueUtils;
-
-    @Inject
-    BigqueryPollJobEnqueuer(TaskQueueUtils taskQueueUtils) {
-      this.taskQueueUtils = taskQueueUtils;
-    }
-
-    /** Enqueue a task to poll for the success or failure of the referenced BigQuery job. */
-    public TaskHandle enqueuePollTask(JobReference jobRef) {
-      return taskQueueUtils.enqueue(
-          getQueue(QUEUE), createCommonPollTask(jobRef).method(Method.GET));
-    }
-
-    /**
-     * Enqueue a task to poll for the success or failure of the referenced BigQuery job and to
-     * launch the provided task in the specified queue if the job succeeds.
-     */
-    public TaskHandle enqueuePollTask(
-        JobReference jobRef, TaskOptions chainedTask, Queue chainedTaskQueue) throws IOException {
-      // Serialize the chainedTask into a byte array to put in the task payload.
-      ByteArrayOutputStream taskBytes = new ByteArrayOutputStream();
-      new ObjectOutputStream(taskBytes).writeObject(chainedTask);
-      return taskQueueUtils.enqueue(
-          getQueue(QUEUE),
-          createCommonPollTask(jobRef)
-              .method(Method.POST)
-              .header(CHAINED_TASK_QUEUE_HEADER, chainedTaskQueue.getQueueName())
-              .payload(taskBytes.toByteArray()));
-    }
-
-    /**
-     * Enqueue a task to poll for the success or failure of the referenced BigQuery job and to
-     * launch the provided task in the specified queue if the job succeeds.
-     */
-    private static TaskOptions createCommonPollTask(JobReference jobRef) {
-      // Omit host header so that task will be run on the current backend/module.
-      return withUrl(PATH)
-          .countdownMillis(POLL_COUNTDOWN.getMillis())
-          .header(PROJECT_ID_HEADER, jobRef.getProjectId())
-          .header(JOB_ID_HEADER, jobRef.getJobId());
-    }
-  }
 }

core/src/main/java/google/registry/export/CheckBackupAction.java

@@ -16,18 +16,14 @@ package google.registry.export;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.Sets.intersection;
-import static google.registry.export.UploadDatastoreBackupAction.enqueueUploadBackupTask;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND;
 
 import com.google.api.client.googleapis.json.GoogleJsonResponseException;
-import com.google.appengine.api.taskqueue.QueueFactory;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import com.google.common.flogger.FluentLogger;
@@ -35,6 +31,7 @@ import google.registry.export.datastore.DatastoreAdmin;
 import google.registry.export.datastore.Operation;
 import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.HttpException;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.HttpException.InternalServerErrorException;
@@ -45,6 +42,7 @@ import google.registry.request.RequestMethod;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.io.IOException;
 import java.util.Set;
 import javax.inject.Inject;
@@ -83,6 +81,7 @@ public class CheckBackupAction implements Runnable {
   @Inject DatastoreAdmin datastoreAdmin;
   @Inject Clock clock;
   @Inject Response response;
+  @Inject CloudTasksUtils cloudTasksUtils;
   @Inject @RequestMethod Action.Method requestMethod;
 
   @Inject
@@ -175,21 +174,22 @@ public class CheckBackupAction implements Runnable {
     if (exportedKindsToLoad.isEmpty()) {
       message += "no kinds to load into BigQuery.";
     } else {
-      enqueueUploadBackupTask(backupId, backup.getExportFolderUrl(), exportedKindsToLoad);
+      /** Enqueue a task for starting a backup load. */
+      cloudTasksUtils.enqueue(
+          UploadDatastoreBackupAction.QUEUE,
+          cloudTasksUtils.createPostTask(
+              UploadDatastoreBackupAction.PATH,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(
+                  UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM,
+                  backupId,
+                  UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM,
+                  backup.getExportFolderUrl(),
+                  UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM,
+                  Joiner.on(',').join(exportedKindsToLoad))));
       message += "BigQuery load task enqueued.";
     }
     logger.atInfo().log(message);
     response.setPayload(message);
   }
-
-  /** Enqueue a poll task to monitor the named backup for completion. */
-  static TaskHandle enqueuePollTask(String backupId, ImmutableSet<String> kindsToLoad) {
-    return QueueFactory.getQueue(QUEUE)
-        .add(
-            TaskOptions.Builder.withUrl(PATH)
-                .method(Method.POST)
-                .countdownMillis(POLL_COUNTDOWN.getMillis())
-                .param(CHECK_BACKUP_NAME_PARAM, backupId)
-                .param(CHECK_BACKUP_KINDS_TO_LOAD_PARAM, Joiner.on(',').join(kindsToLoad)));
-  }
 }

core/src/main/java/google/registry/export/UpdateSnapshotViewAction.java

@@ -21,8 +21,6 @@ import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.model.Table;
 import com.google.api.services.bigquery.model.TableReference;
 import com.google.api.services.bigquery.model.ViewDefinition;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.common.flogger.FluentLogger;
 import google.registry.bigquery.CheckedBigquery;
 import google.registry.config.RegistryConfig.Config;
@@ -85,17 +83,6 @@ public class UpdateSnapshotViewAction implements Runnable {
   @Inject
   UpdateSnapshotViewAction() {}
 
-  /** Create a task for updating a snapshot view. */
-  static TaskOptions createViewUpdateTask(
-      String datasetId, String tableId, String kindName, String viewName) {
-    return TaskOptions.Builder.withUrl(PATH)
-        .method(Method.POST)
-        .param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, datasetId)
-        .param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, tableId)
-        .param(UPDATE_SNAPSHOT_KIND_PARAM, kindName)
-        .param(UPDATE_SNAPSHOT_VIEWNAME_PARAM, viewName);
-  }
-
   @Override
   public void run() {
     try {

core/src/main/java/google/registry/export/UploadDatastoreBackupAction.java

@@ -14,9 +14,7 @@
 
 package google.registry.export;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.common.base.MoreObjects.firstNonNull;
-import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
 import static google.registry.request.Action.Method.POST;
 
 import com.google.api.services.bigquery.Bigquery;
@@ -25,27 +23,33 @@ import com.google.api.services.bigquery.model.JobConfiguration;
 import com.google.api.services.bigquery.model.JobConfigurationLoad;
 import com.google.api.services.bigquery.model.JobReference;
 import com.google.api.services.bigquery.model.TableReference;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
+import com.google.cloud.tasks.v2.Task;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
+import com.google.common.net.HttpHeaders;
+import com.google.common.net.MediaType;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.util.Timestamps;
 import google.registry.bigquery.BigqueryUtils.SourceFormat;
 import google.registry.bigquery.BigqueryUtils.WriteDisposition;
 import google.registry.bigquery.CheckedBigquery;
 import google.registry.config.RegistryConfig.Config;
-import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
 import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.ObjectOutputStream;
 import javax.inject.Inject;
 
 /** Action to load a Datastore backup from Google Cloud Storage into BigQuery. */
@@ -75,7 +79,9 @@ public class UploadDatastoreBackupAction implements Runnable {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   @Inject CheckedBigquery checkedBigquery;
-  @Inject BigqueryPollJobEnqueuer bigqueryPollEnqueuer;
+  @Inject CloudTasksUtils cloudTasksUtils;
+  @Inject Clock clock;
+
   @Inject @Config("projectId") String projectId;
 
   @Inject
@@ -93,18 +99,6 @@ public class UploadDatastoreBackupAction implements Runnable {
   @Inject
   UploadDatastoreBackupAction() {}
 
-  /** Enqueue a task for starting a backup load. */
-  public static TaskHandle enqueueUploadBackupTask(
-      String backupId, String gcsFile, ImmutableSet<String> kinds) {
-    return getQueue(QUEUE)
-        .add(
-            TaskOptions.Builder.withUrl(PATH)
-                .method(Method.POST)
-                .param(UPLOAD_BACKUP_ID_PARAM, backupId)
-                .param(UPLOAD_BACKUP_FOLDER_PARAM, gcsFile)
-                .param(UPLOAD_BACKUP_KINDS_PARAM, Joiner.on(',').join(kinds)));
-  }
-
   @Override
   public void run() {
     try {
@@ -142,12 +136,46 @@ public class UploadDatastoreBackupAction implements Runnable {
       Job job = makeLoadJob(jobRef, sourceUri, tableId);
       bigquery.jobs().insert(projectId, job).execute();
 
-      // Enqueue a task to check on the load job's completion, and if it succeeds, to update a
-      // well-known view in BigQuery to point at the newly loaded backup table for this kind.
-      bigqueryPollEnqueuer.enqueuePollTask(
-          jobRef,
-          createViewUpdateTask(BACKUP_DATASET, tableId, kindName, LATEST_BACKUP_VIEW_NAME),
-          getQueue(UpdateSnapshotViewAction.QUEUE));
+      // Serialize the chainedTask into a byte array to put in the task payload.
+      ByteArrayOutputStream taskBytes = new ByteArrayOutputStream();
+      new ObjectOutputStream(taskBytes)
+          .writeObject(
+              cloudTasksUtils.createPostTask(
+                  UpdateSnapshotViewAction.PATH,
+                  Service.BACKEND.toString(),
+                  ImmutableMultimap.of(
+                      UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM,
+                      BACKUP_DATASET,
+                      UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM,
+                      tableId,
+                      UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM,
+                      kindName,
+                      UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM,
+                      LATEST_BACKUP_VIEW_NAME)));
+
+      // Enqueues a task to poll for the success or failure of the referenced BigQuery job and to
+      // launch the provided task in the specified queue if the job succeeds.
+      cloudTasksUtils.enqueue(
+          BigqueryPollJobAction.QUEUE,
+          Task.newBuilder()
+              .setAppEngineHttpRequest(
+                  cloudTasksUtils
+                      .createPostTask(BigqueryPollJobAction.PATH, Service.BACKEND.toString(), null)
+                      .getAppEngineHttpRequest()
+                      .toBuilder()
+                      .putHeaders(BigqueryPollJobAction.PROJECT_ID_HEADER, jobRef.getProjectId())
+                      .putHeaders(BigqueryPollJobAction.JOB_ID_HEADER, jobRef.getJobId())
+                      .putHeaders(
+                          BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER,
+                          UpdateSnapshotViewAction.QUEUE)
+                      // need to include CONTENT_TYPE in header when body is not empty
+                      .putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
+                      .setBody(ByteString.copyFrom(taskBytes.toByteArray()))
+                      .build())
+              .setScheduleTime(
+                  Timestamps.fromMillis(
+                      clock.nowUtc().plus(BigqueryPollJobAction.POLL_COUNTDOWN).getMillis()))
+              .build());
 
       builder.append(String.format(" - %s:%s\n", projectId, jobId));
       logger.atInfo().log("Submitted load job %s:%s.", projectId, jobId);

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -301,7 +301,6 @@ public class DomainFlowUtils {
             String.format(
                 "A maximum of %s DS records are allowed per domain.", MAX_DS_RECORDS_PER_DOMAIN));
       }
-      // TODO(sarahbot@): Add signature length verification
       ImmutableList<DelegationSignerData> invalidAlgorithms =
           dsData.stream()
               .filter(ds -> !validateAlgorithm(ds.getAlgorithm()))
@@ -322,6 +321,20 @@ public class DomainFlowUtils {
                 "Domain contains DS record(s) with an invalid digest type: %s",
                 invalidDigestTypes));
       }
+      ImmutableList<DelegationSignerData> digestsWithInvalidDigestLength =
+          dsData.stream()
+              .filter(
+                  ds ->
+                      DigestType.fromWireValue(ds.getDigestType()).isPresent()
+                          && (ds.getDigest().length
+                              != DigestType.fromWireValue(ds.getDigestType()).get().getBytes()))
+              .collect(toImmutableList());
+      if (!digestsWithInvalidDigestLength.isEmpty()) {
+        throw new InvalidDsRecordException(
+            String.format(
+                "Domain contains DS record(s) with an invalid digest length: %s",
+                digestsWithInvalidDigestLength));
+      }
     }
   }
 

core/src/main/java/google/registry/model/EntityClasses.java

@@ -40,6 +40,7 @@ import google.registry.model.rde.RdeRevision;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.replay.LastSqlTransaction;
+import google.registry.model.replay.ReplayGap;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.server.Lock;
 import google.registry.model.server.ServerSecret;
@@ -86,6 +87,7 @@ public final class EntityClasses {
           Registrar.class,
           RegistrarContact.class,
           Registry.class,
+          ReplayGap.class,
           ServerSecret.class);
 
   private EntityClasses() {}

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -308,7 +308,9 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "eventTime"),
         @javax.persistence.Index(columnList = "billingTime"),
         @javax.persistence.Index(columnList = "syntheticCreationTime"),
-        @javax.persistence.Index(columnList = "allocationToken")
+        @javax.persistence.Index(columnList = "domainRepoId"),
+        @javax.persistence.Index(columnList = "allocationToken"),
+        @javax.persistence.Index(columnList = "cancellation_matching_billing_recurrence_id")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
   @WithLongVKey(compositeKey = true)
@@ -518,6 +520,7 @@ public abstract class BillingEvent extends ImmutableObject
       indexes = {
         @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
+        @javax.persistence.Index(columnList = "domainRepoId"),
         @javax.persistence.Index(columnList = "recurrenceEndTime"),
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
@@ -614,7 +617,10 @@ public abstract class BillingEvent extends ImmutableObject
       indexes = {
         @javax.persistence.Index(columnList = "registrarId"),
         @javax.persistence.Index(columnList = "eventTime"),
-        @javax.persistence.Index(columnList = "billingTime")
+        @javax.persistence.Index(columnList = "domainRepoId"),
+        @javax.persistence.Index(columnList = "billingTime"),
+        @javax.persistence.Index(columnList = "billing_event_id"),
+        @javax.persistence.Index(columnList = "billing_recurrence_id")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
   @WithLongVKey(compositeKey = true)

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -69,7 +69,10 @@ import org.joda.time.DateTime;
       @Index(columnList = "techContact"),
       @Index(columnList = "tld"),
       @Index(columnList = "registrantContact"),
-      @Index(columnList = "dnsRefreshRequestTime")
+      @Index(columnList = "dnsRefreshRequestTime"),
+      @Index(columnList = "billing_recurrence_id"),
+      @Index(columnList = "transfer_billing_event_id"),
+      @Index(columnList = "transfer_billing_recurrence_id")
     })
 @WithStringVKey
 @ExternalMessagingName("domain")
@@ -89,7 +92,10 @@ public class DomainBase extends DomainContent
   @ElementCollection
   @JoinTable(
       name = "DomainHost",
-      indexes = {@Index(columnList = "domain_repo_id,host_repo_id", unique = true)})
+      indexes = {
+        @Index(columnList = "domain_repo_id,host_repo_id", unique = true),
+        @Index(columnList = "host_repo_id")
+      })
   @Access(AccessType.PROPERTY)
   @Column(name = "host_repo_id")
   public Set<VKey<HostResource>> getNsHosts() {

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -46,7 +46,12 @@ import org.joda.time.DateTime;
  */
 @Embed
 @Entity
-@Table(indexes = @Index(columnList = "domainRepoId"))
+@Table(
+    indexes = {
+      @Index(columnList = "domainRepoId"),
+      @Index(columnList = "billing_event_id"),
+      @Index(columnList = "billing_recurrence_id")
+    })
 public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntity {
 
   @Id

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -76,6 +76,8 @@ import org.joda.time.DateTime;
       @javax.persistence.Index(
           columnList = "domainName",
           name = "allocation_token_domain_name_idx"),
+      @javax.persistence.Index(columnList = "tokenType"),
+      @javax.persistence.Index(columnList = "redemption_domain_repo_id")
     })
 public class AllocationToken extends BackupGroupRoot implements Buildable, DatastoreAndSqlEntity {
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -36,7 +36,23 @@ import javax.persistence.AccessType;
 @javax.persistence.Entity(name = "Host")
 @javax.persistence.Table(
     name = "Host",
-    indexes = {@javax.persistence.Index(columnList = "hostName")})
+    /**
+     * A gin index defined on the inet_addresses field ({@link HostBase#inetAddresses} cannot be
+     * declared here because JPA/Hibernate does not support index type specification. As a result,
+     * the hibernate-generated schema (which is for reference only) does not have this index.
+     *
+     * <p>There are Hibernate-specific solutions for adding this index to Hibernate's domain model.
+     * We could either declare the index in hibernate.cfg.xml or add it to the {@link
+     * org.hibernate.cfg.Configuration} instance for {@link SessionFactory} instantiation (which
+     * would prevent us from using JPA standard bootstrapping). For now, there is no obvious benefit
+     * doing either.
+     */
+    indexes = {
+      @javax.persistence.Index(columnList = "hostName"),
+      @javax.persistence.Index(columnList = "creationTime"),
+      @javax.persistence.Index(columnList = "deletionTime"),
+      @javax.persistence.Index(columnList = "currentSponsorRegistrarId")
+    })
 @ExternalMessagingName("host")
 @WithStringVKey
 @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)

core/src/main/java/google/registry/model/replay/ReplayGap.java

@@ -0,0 +1,60 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.replay;
+
+import static google.registry.model.annotations.NotBackedUp.Reason.TRANSIENT;
+
+import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Id;
+import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.annotations.NotBackedUp;
+import org.joda.time.DateTime;
+
+/**
+ * Tracks gaps in transaction ids when replicating from SQL to datastore.
+ *
+ * <p>SQL -&gt; DS replication uses a Transaction table indexed by a SEQUENCE column, which normally
+ * increments monotonically for each committed transaction. Gaps in this sequence can occur when a
+ * transaction is rolled back or when a transaction has been initiated but not committed to the
+ * table at the time of a query. To protect us from the latter scenario, we need to keep track of
+ * these gaps and replay any of them that have been filled in since we processed their batch.
+ */
+@DeleteAfterMigration
+@NotBackedUp(reason = TRANSIENT)
+@Entity
+public class ReplayGap extends ImmutableObject implements DatastoreOnlyEntity {
+  @Id long transactionId;
+
+  // We can't use a CreateAutoTimestamp here because this ends up getting persisted in an ofy
+  // transaction that happens in JPA mode, so we don't end up getting an active transaction manager
+  // when the timestamp needs to be set.
+  DateTime timestamp;
+
+  ReplayGap() {}
+
+  ReplayGap(DateTime timestamp, long transactionId) {
+    this.timestamp = timestamp;
+    this.transactionId = transactionId;
+  }
+
+  long getTransactionId() {
+    return transactionId;
+  }
+
+  DateTime getTimestamp() {
+    return timestamp;
+  }
+}

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.replay;
 
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
@@ -40,6 +42,7 @@ import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.RequestStatusChecker;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -70,6 +73,20 @@ public class ReplicateToDatastoreAction implements Runnable {
    */
   public static final int BATCH_SIZE = 200;
 
+  /**
+   * The longest time that we'll keep trying to resolve a gap in the Transaction table in
+   * milliseconds, after which, the gap record will be deleted.
+   */
+  public static final long MAX_GAP_RETENTION_MILLIS = 300000;
+
+  /**
+   * The maximum number of entitities to be mutated per transaction. For our purposes, the entities
+   * that we're keeping track of are each in their own entity group. Per datastore documentation, we
+   * should be allowed to update up to 25 of them. In practice, we get an error if we go beyond 24
+   * (possibly due to something in our own infrastructure).
+   */
+  private static final int MAX_ENTITIES_PER_TXN = 24;
+
   public static final Duration REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH = standardHours(1);
 
   private final Clock clock;
@@ -89,6 +106,12 @@ public class ReplicateToDatastoreAction implements Runnable {
     return getTransactionBatchAtSnapshot(Optional.empty());
   }
 
+  /**
+   * Get the next batch of transactions, optionally from a specific SQL database snapshot.
+   *
+   * <p>Note that this method may also apply transactions from previous batches that had not yet
+   * been committed at the time the previous batch was retrieved.
+   */
   static List<TransactionEntity> getTransactionBatchAtSnapshot(Optional<String> snapshotId) {
     // Get the next batch of transactions that we haven't replicated.
     LastSqlTransaction lastSqlTxnBeforeBatch = ofyTm().transact(LastSqlTransaction::load);
@@ -97,6 +120,11 @@ public class ReplicateToDatastoreAction implements Runnable {
           .transactWithoutBackup(
               () -> {
                 snapshotId.ifPresent(jpaTm()::setDatabaseSnapshot);
+
+                // Fill in any gaps in the transaction log that have since become available before
+                // processing the next batch.
+                applyMissingTransactions();
+
                 return jpaTm()
                     .query(
                         "SELECT txn FROM TransactionEntity txn WHERE id >" + " :lastId ORDER BY id",
@@ -111,60 +139,175 @@ public class ReplicateToDatastoreAction implements Runnable {
   }
 
   /**
-   * Apply a transaction to Datastore, returns true if there was a fatal error and the batch should
-   * be aborted.
+   * Iterate over the recent gaps in the Transaction table and apply any that have been filled in.
    *
-   * <p>Throws an exception if a fatal error occurred and the batch should be aborted
+   * <p>Must be called from within a JPA transaction.
+   *
+   * <p>Gap rewriting is a complicated matter, and the algorithm is the product of some very deep
+   * consideration by mmuller and weiminyu. Basically, the constraints are:
+   *
+   * <ol>
+   *   <li>Replay has to work against a database snapshot (gap replay would break this, so we don't
+   *       call this method when replaying against a snapshot)
+   * </ol>
+   */
+  private static void applyMissingTransactions() {
+    long now = jpaTm().getTransactionTime().getMillis();
+    ImmutableList<ReplayGap> gaps = ofyTm().loadAllOf(ReplayGap.class);
+    jpaTm()
+        .query("SELECT txn from TransactionEntity txn WHERE id IN :gapIds", TransactionEntity.class)
+        .setParameter(
+            "gapIds", gaps.stream().map(gap -> gap.getTransactionId()).collect(toImmutableList()))
+        .getResultStream()
+        .forEach(
+            txn -> {
+              // Transcribe the transaction and delete the gap record in the same ofy transaction.
+              ofyTm()
+                  .transact(
+                      () -> {
+                        // Write the transaction to datastore.
+                        try {
+                          Transaction.deserialize(txn.getContents()).writeToDatastore();
+                        } catch (IOException e) {
+                          throw new RuntimeException("Error during transaction deserialization", e);
+                        }
+
+                        // Find and delete the gap record.
+                        ImmutableList<ReplayGap> filledGaps =
+                            gaps.stream()
+                                .filter(gap -> gap.getTransactionId() == txn.getId())
+                                .collect(toImmutableList());
+                        checkState(
+                            filledGaps.size() == 1,
+                            "Bad list of gaps for discovered id: %s",
+                            filledGaps);
+                        auditedOfy().deleteIgnoringReadOnlyWithoutBackup().entity(gaps.get(0));
+                      });
+              logger.atInfo().log("Applied missing transaction %s", txn.getId());
+            });
+
+    // Clean up any gaps that have expired (in batches because they're each in their own entity
+    // group).
+    ArrayList<ReplayGap> gapBatch = new ArrayList<>();
+    gaps.stream()
+        .forEach(
+            gap -> {
+              if (now - gap.getTimestamp().getMillis() > MAX_GAP_RETENTION_MILLIS) {
+                gapBatch.add(gap);
+              }
+              if (gapBatch.size() == MAX_ENTITIES_PER_TXN) {
+                deleteReplayGaps(gapBatch);
+                gapBatch.clear();
+              }
+            });
+    if (!gapBatch.isEmpty()) {
+      deleteReplayGaps(gapBatch);
+    }
+  }
+
+  private static void deleteReplayGaps(ArrayList<ReplayGap> gapsToDelete) {
+    logger.atInfo().log(
+        "deleting gap records for %s",
+        gapsToDelete.stream().map(g -> g.getTransactionId()).collect(toImmutableList()));
+    ofyTm()
+        .transact(() -> auditedOfy().deleteIgnoringReadOnlyWithoutBackup().entities(gapsToDelete));
+  }
+
+  /**
+   * Apply a transaction to Datastore.
+   *
+   * <p>Throws an exception if a fatal error occurred and the batch should be aborted.
    */
   @VisibleForTesting
   public static void applyTransaction(TransactionEntity txnEntity) {
     logger.atInfo().log("Applying a single transaction Cloud SQL -> Cloud Datastore.");
+    boolean done = false;
     try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
         UpdateAutoTimestamp.disableAutoUpdate()) {
-      ofyTm()
-          .transact(
-              () -> {
-                // Reload the last transaction id, which could possibly have changed.
-                LastSqlTransaction lastSqlTxn = LastSqlTransaction.load();
-                long nextTxnId = lastSqlTxn.getTransactionId() + 1;
-                if (nextTxnId < txnEntity.getId()) {
-                  // We're missing a transaction.  This is bad.  Transaction ids are supposed to
-                  // increase monotonically, so we abort rather than applying anything out of
-                  // order.
-                  throw new IllegalStateException(
-                      String.format(
-                          "Missing transaction: last txn id = %s, next available txn = %s",
-                          nextTxnId - 1, txnEntity.getId()));
-                } else if (nextTxnId > txnEntity.getId()) {
-                  // We've already replayed this transaction.  This shouldn't happen, as GAE cron
-                  // is supposed to avoid overruns and this action shouldn't be executed from any
-                  // other context, but it's not harmful as we can just ignore the transaction.  Log
-                  // it so that we know about it and move on.
-                  logger.atWarning().log(
-                      "Ignoring transaction %s, which appears to have already been applied.",
-                      txnEntity.getId());
-                  return;
-                }
 
-                logger.atInfo().log(
-                    "Applying transaction %s to Cloud Datastore.", txnEntity.getId());
+      // We put this in a do/while loop because we can potentially clean out some range of gaps
+      // first and we want to do those in their own transaction so as not to run up the entity group
+      // count.  (This is a highly pathological case: consecutive gaps are rare, but the fact that
+      // they can occur potentially increases the entity group count to beyond what we can
+      // accommodate in a single transaction.)
+      do {
+        done =
+            ofyTm()
+                .transact(
+                    () -> {
+                      // Reload the last transaction id, which could possibly have changed.
+                      LastSqlTransaction lastSqlTxn = LastSqlTransaction.load();
+                      long nextTxnId = lastSqlTxn.getTransactionId() + 1;
 
-                // At this point, we know txnEntity is the correct next transaction, so write it
-                // to Datastore.
-                try {
-                  Transaction.deserialize(txnEntity.getContents()).writeToDatastore();
-                } catch (IOException e) {
-                  throw new RuntimeException("Error during transaction deserialization", e);
-                }
+                      // Skip missing transactions.  Missed transactions can happen normally.  If a
+                      // transaction gets rolled back, the sequence counter doesn't.
+                      int gapCount = 0;
+                      while (nextTxnId < txnEntity.getId()) {
+                        logger.atWarning().log(
+                            "Ignoring transaction %s, which does not exist.", nextTxnId);
+                        auditedOfy()
+                            .saveIgnoringReadOnlyWithoutBackup()
+                            .entity(new ReplayGap(ofyTm().getTransactionTime(), nextTxnId));
+                        ++nextTxnId;
 
-                // Write the updated last transaction id to Datastore as part of this Datastore
-                // transaction.
-                auditedOfy()
-                    .saveIgnoringReadOnlyWithoutBackup()
-                    .entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
-                logger.atInfo().log(
-                    "Finished applying single transaction Cloud SQL -> Cloud Datastore.");
-              });
+                        // Don't exceed the entity group count trying to clean these up (we stop at
+                        // max
+                        // - 1 because we also want to save the lastSqlTransaction).
+                        if (++gapCount == MAX_ENTITIES_PER_TXN - 1) {
+                          break;
+                        }
+                      }
+
+                      // Don't write gap records in the same transaction as the SQL transaction that
+                      // we're replaying.  Return false to force us to go through and repeat in a
+                      // new
+                      // transaction.
+                      if (gapCount > 0) {
+                        // We haven't replayed the transaction, but we've determined that we can
+                        // ignore everything before it so update lastSqlTransaction accordingly.
+                        auditedOfy()
+                            .saveIgnoringReadOnlyWithoutBackup()
+                            .entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId - 1));
+                        return false;
+                      }
+
+                      if (nextTxnId > txnEntity.getId()) {
+                        // We've already replayed this transaction.  This shouldn't happen, as GAE
+                        // cron
+                        // is supposed to avoid overruns and this action shouldn't be executed from
+                        // any
+                        // other context, but it's not harmful as we can just ignore the
+                        // transaction.
+                        // Log it so that we know about it and move on.
+                        logger.atWarning().log(
+                            "Ignoring transaction %s, which appears to have already been applied.",
+                            txnEntity.getId());
+                        return true;
+                      }
+
+                      logger.atInfo().log(
+                          "Applying transaction %s to Cloud Datastore.", txnEntity.getId());
+
+                      // At this point, we know txnEntity is the correct next transaction, so write
+                      // it
+                      // to Datastore.
+                      try {
+                        Transaction.deserialize(txnEntity.getContents()).writeToDatastore();
+                      } catch (IOException e) {
+                        throw new RuntimeException("Error during transaction deserialization", e);
+                      }
+
+                      // Write the updated last transaction id to Datastore as part of this
+                      // Datastore
+                      // transaction.
+                      auditedOfy()
+                          .saveIgnoringReadOnlyWithoutBackup()
+                          .entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
+                      logger.atInfo().log(
+                          "Finished applying single transaction Cloud SQL -> Cloud Datastore.");
+                      return true;
+                    });
+      } while (!done);
     }
   }
 

core/src/main/java/google/registry/model/tld/Registries.java

@@ -24,6 +24,7 @@ import static com.google.common.collect.Maps.filterValues;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
@@ -38,6 +39,8 @@ import com.google.common.net.InternetDomainName;
 import com.googlecode.objectify.Key;
 import google.registry.model.tld.Registry.TldType;
 import java.util.Optional;
+import java.util.stream.Stream;
+import javax.persistence.EntityManager;
 
 /** Utilities for finding and listing {@link Registry} entities. */
 public final class Registries {
@@ -58,23 +61,31 @@ public final class Registries {
         () ->
             tm().doTransactionless(
                     () -> {
-                      ImmutableSet<String> tlds =
-                          tm().isOfy()
-                              ? auditedOfy()
-                                  .load()
-                                  .type(Registry.class)
-                                  .ancestor(getCrossTldKey())
-                                  .keys()
-                                  .list()
-                                  .stream()
-                                  .map(Key::getName)
-                                  .collect(toImmutableSet())
-                              : tm().loadAllOf(Registry.class).stream()
-                                  .map(Registry::getTldStr)
-                                  .collect(toImmutableSet());
-                      return Registry.getAll(tlds).stream()
-                          .map(e -> Maps.immutableEntry(e.getTldStr(), e.getTldType()))
-                          .collect(entriesToImmutableMap());
+                      if (tm().isOfy()) {
+                        ImmutableSet<String> tlds =
+                            auditedOfy()
+                                .load()
+                                .type(Registry.class)
+                                .ancestor(getCrossTldKey())
+                                .keys()
+                                .list()
+                                .stream()
+                                .map(Key::getName)
+                                .collect(toImmutableSet());
+                        return Registry.getAll(tlds).stream()
+                            .map(e -> Maps.immutableEntry(e.getTldStr(), e.getTldType()))
+                            .collect(entriesToImmutableMap());
+                      } else {
+                        EntityManager entityManager = jpaTm().getEntityManager();
+                        Stream<?> resultStream =
+                            entityManager
+                                .createQuery("SELECT tldStrId, tldType FROM Tld")
+                                .getResultStream();
+                        return resultStream
+                            .map(e -> ((Object[]) e))
+                            .map(e -> Maps.immutableEntry((String) e[0], ((TldType) e[1])))
+                            .collect(entriesToImmutableMap());
+                      }
                     }));
   }
 

core/src/main/java/google/registry/model/transfer/DomainTransferData.java

@@ -94,8 +94,6 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
    *
    * <p>This field should be null if there is not currently a pending transfer or if the object
    * being transferred is not a domain.
-   *
-   * <p>TODO(b/158230654) Remove unused columns for TransferData in Contact table.
    */
   @IgnoreSave(IfNull.class)
   @Column(name = "transfer_billing_event_id")

core/src/main/java/google/registry/module/backend/BackendComponent.java

@@ -43,7 +43,7 @@ import google.registry.rde.JSchModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.URLFetchServiceModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
@@ -80,7 +80,7 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       SheetsServiceModule.class,
       StackdriverModule.class,
-      URLFetchServiceModule.class,
+      UrlConnectionServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
       VoidDnsWriterModule.class,

core/src/main/java/google/registry/module/frontend/FrontendComponent.java

@@ -34,7 +34,6 @@ import google.registry.monitoring.whitebox.StackdriverModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.ui.ConsoleDebug.ConsoleConfigModule;
@@ -65,7 +64,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -34,7 +34,6 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -62,7 +61,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/tools/ToolsComponent.java

@@ -36,7 +36,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -66,7 +65,6 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       StackdriverModule.class,
       ToolsRequestComponentModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java

@@ -14,6 +14,7 @@
 
 package google.registry.persistence.transaction;
 
+import com.google.common.annotations.VisibleForTesting;
 import google.registry.model.ImmutableObject;
 import google.registry.model.replay.SqlOnlyEntity;
 import javax.persistence.Entity;
@@ -39,7 +40,8 @@ public class TransactionEntity extends ImmutableObject implements SqlOnlyEntity
 
   TransactionEntity() {}
 
-  TransactionEntity(byte[] contents) {
+  @VisibleForTesting
+  public TransactionEntity(byte[] contents) {
     this.contents = contents;
   }
 

core/src/main/java/google/registry/rde/BrdaCopyAction.java

@@ -14,8 +14,13 @@
 
 package google.registry.rde;
 
+import static google.registry.model.common.Cursor.CursorType.BRDA;
+import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.model.rde.RdeMode.THIN;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.request.Action.Method.POST;
+import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.google.cloud.storage.BlobId;
 import com.google.common.flogger.FluentLogger;
@@ -23,9 +28,11 @@ import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.KeyModule.Key;
+import google.registry.model.common.Cursor;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
 import google.registry.request.Action;
+import google.registry.request.HttpException.NoContentException;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.auth.Auth;
@@ -89,6 +96,16 @@ public final class BrdaCopyAction implements Runnable {
   private void copyAsRyde() throws IOException {
     // TODO(b/217772483): consider guarding this action with a lock and check if there is work.
     // Not urgent since file writes on GCS are atomic.
+    Optional<Cursor> cursor =
+        transactIfJpaTm(() -> tm().loadByKeyIfPresent(Cursor.createVKey(BRDA, tld)));
+    DateTime brdaCursorTime = getCursorTimeOrStartOfTime(cursor);
+    if (isBeforeOrAt(brdaCursorTime, watermark)) {
+      throw new NoContentException(
+          String.format(
+              "Waiting on RdeStagingAction for TLD %s to copy BRDA deposit for %s to GCS; "
+                  + "last BRDA staging completion was before %s",
+              tld, watermark, brdaCursorTime));
+    }
     int revision =
         RdeRevision.getCurrentRevision(tld, watermark, THIN)
             .orElseThrow(

core/src/main/java/google/registry/rde/RdeReporter.java

@@ -14,26 +14,24 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
-import static com.google.common.io.BaseEncoding.base64;
-import static com.google.common.net.HttpHeaders.AUTHORIZATION;
-import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
+import static google.registry.request.UrlConnectionUtils.setPayload;
 import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
+import com.google.api.client.http.HttpMethods;
 import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.request.UrlConnectionService;
 import google.registry.util.Retrier;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
 import google.registry.xjc.iirdea.XjcIirdeaResult;
@@ -41,6 +39,7 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.SocketTimeoutException;
 import java.net.URL;
@@ -55,12 +54,15 @@ public class RdeReporter {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  /** @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
-   *     ICANN Registry Interfaces - Interface details</a>*/
-  private static final String REPORT_MIME = "text/xml";
+  /**
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
+   *     ICANN Registry Interfaces - Interface details</a>
+   */
+  private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
 
   @Inject Retrier retrier;
-  @Inject URLFetchService urlFetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
   @Inject @Key("icannReportingPassword") String password;
   @Inject RdeReporter() {}
@@ -74,29 +76,24 @@ public class RdeReporter {
     // Send a PUT request to ICANN's HTTPS server.
     URL url = makeReportUrl(header.getTld(), report.getId());
     String username = header.getTld() + "_ry";
-    String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
-    final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
-    req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
-    req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    req.setPayload(reportBytes);
     logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
-    HTTPResponse rsp =
+    byte[] responseBytes =
         retrier.callWithRetry(
             () -> {
-              HTTPResponse rsp1 = urlFetchService.fetch(req);
-              switch (rsp1.getResponseCode()) {
-                case SC_OK:
-                case SC_BAD_REQUEST:
-                  break;
-                default:
-                  throw new UrlFetchException("PUT failed", req, rsp1);
+              HttpURLConnection connection = urlConnectionService.createConnection(url);
+              connection.setRequestMethod(HttpMethods.PUT);
+              setBasicAuth(connection, username, password);
+              setPayload(connection, reportBytes, MEDIA_TYPE.toString());
+              int responseCode = connection.getResponseCode();
+              if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) {
+                return getResponseBytes(connection);
               }
-              return rsp1;
+              throw new UrlConnectionException("PUT failed", connection);
             },
             SocketTimeoutException.class);
 
     // Ensure the XML response is valid.
-    XjcIirdeaResult result = parseResult(rsp);
+    XjcIirdeaResult result = parseResult(responseBytes);
     if (result.getCode().getValue() != 1000) {
       logger.atWarning().log(
           "PUT rejected: %d %s\n%s",
@@ -108,11 +105,11 @@ public class RdeReporter {
   /**
    * Unmarshals IIRDEA XML result object from {@link HTTPResponse} payload.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
+   * @see <a
+   *     href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
    *     ICANN Registry Interfaces - IIRDEA Result Object</a>
    */
-  private XjcIirdeaResult parseResult(HTTPResponse rsp) throws XmlException {
-    byte[] responseBytes = rsp.getContent();
+  private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException {
     logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8));
     XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal(
         XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes));

core/src/main/java/google/registry/rde/RdeStagingReducer.java

@@ -64,6 +64,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
   private static final long serialVersionUID = 60326234579091203L;
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final Duration ENQUEUE_DELAY = Duration.standardMinutes(1);
 
   private final CloudTasksUtils cloudTasksUtils;
   private final LockHandler lockHandler;
@@ -202,6 +203,14 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
       logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task.");
       return;
     }
+    // We need to save the revision in a separate transaction because the subsequent upload/copy
+    // action reads the most current revision from the database. If it is done in the same
+    // transaction with the enqueueing, the action might start running before the transaction is
+    // committed, due to Cloud Tasks not being transaction aware, unlike Task Queue. The downside
+    // is that if for some reason the second transaction is rolled back, the revision update is not
+    // undone. But this should be fine since the next run will just increment the revision and start
+    // over.
+    tm().transact(() -> RdeRevision.saveRevision(tld, watermark, mode, revision));
     tm().transact(
             () -> {
               Registry registry = Registry.get(tld);
@@ -225,29 +234,33 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
               tm().put(Cursor.create(key.cursor(), newPosition, registry));
               logger.atInfo().log(
                   "Rolled forward %s on %s cursor to %s.", key.cursor(), tld, newPosition);
-              RdeRevision.saveRevision(tld, watermark, mode, revision);
               // Enqueueing a task is a side effect that is not undone if the transaction rolls
-              // back. So this may result in multiple copies of the same task being processed. This
-              // is fine because the RdeUploadAction is guarded by a lock and tracks progress by
-              // cursor. The BrdaCopyAction writes a file to GCS, which is an atomic action.
+              // back. So this may result in multiple copies of the same task being processed.
+              // This is fine because the RdeUploadAction is guarded by a lock and tracks progress
+              // by cursor. The BrdaCopyAction writes a file to GCS, which is an atomic action. It
+              // is also guarded by a cursor to not run before the cursor is updated. We also
+              // include a delay to minimize the chance that the enqueued job executes before the
+              // transaction is committed, which triggers a retry.
               if (mode == RdeMode.FULL) {
                 cloudTasksUtils.enqueue(
                     "rde-upload",
-                    cloudTasksUtils.createPostTask(
+                    cloudTasksUtils.createPostTaskWithDelay(
                         RdeUploadAction.PATH,
                         Service.BACKEND.toString(),
-                        ImmutableMultimap.of(RequestParameters.PARAM_TLD, tld)));
+                        ImmutableMultimap.of(RequestParameters.PARAM_TLD, tld),
+                        ENQUEUE_DELAY));
               } else {
                 cloudTasksUtils.enqueue(
                     "brda",
-                    cloudTasksUtils.createPostTask(
+                    cloudTasksUtils.createPostTaskWithDelay(
                         BrdaCopyAction.PATH,
                         Service.BACKEND.toString(),
                         ImmutableMultimap.of(
                             RequestParameters.PARAM_TLD,
                             tld,
                             RdeModule.PARAM_WATERMARK,
-                            watermark.toString())));
+                            watermark.toString()),
+                        ENQUEUE_DELAY));
               }
             });
   }

core/src/main/java/google/registry/rde/RdeUploadAction.java

@@ -148,7 +148,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
       throw new NoContentException(
           String.format(
               "Waiting on RdeStagingAction for TLD %s to send %s upload; "
-                  + "last RDE staging completion was at %s",
+                  + "last RDE staging completion was before %s",
               tld, watermark, stagingCursorTime));
     }
     DateTime sftpCursorTime =

core/src/main/java/google/registry/request/Modules.java

@@ -23,12 +23,11 @@ import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.appengine.api.datastore.DatastoreService;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
 import com.google.appengine.api.users.UserService;
 import com.google.appengine.api.users.UserServiceFactory;
 import dagger.Module;
 import dagger.Provides;
+import java.net.HttpURLConnection;
 import javax.inject.Singleton;
 
 /** Dagger modules for App Engine services and other vendor classes. */
@@ -45,14 +44,12 @@ public final class Modules {
     }
   }
 
-  /** Dagger module for {@link URLFetchService}. */
+  /** Dagger module for {@link UrlConnectionService}. */
   @Module
-  public static final class URLFetchServiceModule {
-    private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService();
-
+  public static final class UrlConnectionServiceModule {
     @Provides
-    static URLFetchService provideURLFetchService() {
-      return fetchService;
+    static UrlConnectionService provideUrlConnectionService() {
+      return url -> (HttpURLConnection) url.openConnection();
     }
   }
 

core/src/main/java/google/registry/request/RequestModule.java

@@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.io.ByteStreams;
 import com.google.common.io.CharStreams;
 import com.google.common.net.MediaType;
+import com.google.protobuf.ByteString;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
@@ -184,6 +185,16 @@ public final class RequestModule {
     }
   }
 
+  @Provides
+  @Payload
+  static ByteString providePayloadAsByteString(HttpServletRequest req) {
+    try {
+      return ByteString.copyFrom(ByteStreams.toByteArray(req.getInputStream()));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   @Provides
   static LockHandler provideLockHandler(LockHandlerImpl lockHandler) {
     return lockHandler;

core/src/main/java/google/registry/request/UrlConnectionService.java

@@ -0,0 +1,25 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.request;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/** Functional interface for opening a connection from a URL, injectable for testing. */
+public interface UrlConnectionService {
+
+  HttpURLConnection createConnection(URL url) throws IOException;
+}

core/src/main/java/google/registry/request/UrlConnectionUtils.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.io.BaseEncoding.base64;
@@ -22,36 +22,41 @@ import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.common.base.Ascii;
 import com.google.common.base.Strings;
+import com.google.common.io.ByteStreams;
 import com.google.common.net.MediaType;
-import java.util.Optional;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.URLConnection;
 import java.util.Random;
 
-/** Helper methods for the App Engine URL fetch service. */
-public final class UrlFetchUtils {
+/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */
+public class UrlConnectionUtils {
 
-  /** Returns value of first header matching {@code name}. */
-  public static Optional<String> getHeaderFirst(HTTPResponse rsp, String name) {
-    return getHeaderFirstInternal(rsp.getHeadersUncombined(), name);
+  /** Retrieves the response from the given connection as a byte array. */
+  public static byte[] getResponseBytes(URLConnection connection) throws IOException {
+    return ByteStreams.toByteArray(connection.getInputStream());
   }
 
-  /** Returns value of first header matching {@code name}. */
-  public static Optional<String> getHeaderFirst(HTTPRequest req, String name) {
-    return getHeaderFirstInternal(req.getHeaders(), name);
+  /** Sets auth on the given connection with the given username/password. */
+  public static void setBasicAuth(URLConnection connection, String username, String password) {
+    setBasicAuth(connection, String.format("%s:%s", username, password));
   }
 
-  private static Optional<String> getHeaderFirstInternal(Iterable<HTTPHeader> hdrs, String name) {
-    name = Ascii.toLowerCase(name);
-    for (HTTPHeader header : hdrs) {
-      if (Ascii.toLowerCase(header.getName()).equals(name)) {
-        return Optional.of(header.getValue());
-      }
+  /** Sets auth on the given connection with the given string, formatted "username:password". */
+  public static void setBasicAuth(URLConnection connection, String usernameAndPassword) {
+    String token = base64().encode(usernameAndPassword.getBytes(UTF_8));
+    connection.setRequestProperty(AUTHORIZATION, "Basic " + token);
+  }
+
+  /** Sets the given byte[] payload on the given connection with a particular content type. */
+  public static void setPayload(URLConnection connection, byte[] bytes, String contentType)
+      throws IOException {
+    connection.setRequestProperty(CONTENT_TYPE, contentType);
+    connection.setDoOutput(true);
+    try (DataOutputStream dataStream = new DataOutputStream(connection.getOutputStream())) {
+      dataStream.write(bytes);
     }
-    return Optional.empty();
   }
 
   /**
@@ -62,16 +67,16 @@ public final class UrlFetchUtils {
    * @see <a href="http://www.ietf.org/rfc/rfc2388.txt">RFC2388 - Returning Values from Forms</a>
    */
   public static void setPayloadMultipart(
-      HTTPRequest request,
+      URLConnection connection,
       String name,
       String filename,
       MediaType contentType,
       String data,
-      Random random) {
+      Random random)
+      throws IOException {
     String boundary = createMultipartBoundary(random);
     checkState(
-        !data.contains(boundary),
-        "Multipart data contains autogenerated boundary: %s", boundary);
+        !data.contains(boundary), "Multipart data contains autogenerated boundary: %s", boundary);
     String multipart =
         String.format("--%s\r\n", boundary)
             + String.format(
@@ -83,11 +88,9 @@ public final class UrlFetchUtils {
             + "\r\n"
             + String.format("--%s--\r\n", boundary);
     byte[] payload = multipart.getBytes(UTF_8);
-    request.addHeader(
-        new HTTPHeader(
-            CONTENT_TYPE, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary)));
-    request.addHeader(new HTTPHeader(CONTENT_LENGTH, Integer.toString(payload.length)));
-    request.setPayload(payload);
+    connection.setRequestProperty(CONTENT_LENGTH, Integer.toString(payload.length));
+    setPayload(
+        connection, payload, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary));
   }
 
   private static String createMultipartBoundary(Random random) {
@@ -98,12 +101,4 @@ public final class UrlFetchUtils {
     // See https://tools.ietf.org/html/rfc2046#section-5.1.1
     return Strings.repeat("-", 30) + base64().encode(rand);
   }
-
-  /** Sets the HTTP Basic Authentication header on an {@link HTTPRequest}. */
-  public static void setAuthorizationHeader(HTTPRequest req, Optional<String> login) {
-    if (login.isPresent()) {
-      String token = base64().encode(login.get().getBytes(UTF_8));
-      req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    }
-  }
 }

core/src/main/java/google/registry/tmch/LordnRequestInitializer.java

@@ -15,12 +15,12 @@
 package google.registry.tmch;
 
 import static com.google.common.base.Verify.verifyNotNull;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
 import com.google.common.flogger.FluentLogger;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.tld.Registry;
+import google.registry.request.UrlConnectionUtils;
+import java.net.HttpURLConnection;
 import java.util.Optional;
 import javax.inject.Inject;
 
@@ -37,8 +37,9 @@ final class LordnRequestInitializer {
   }
 
   /** Initializes a URL fetch request for talking to the MarksDB server. */
-  void initialize(HTTPRequest request, String tld) {
-    setAuthorizationHeader(request, getMarksDbLordnCredentials(tld));
+  void initialize(HttpURLConnection connection, String tld) {
+    getMarksDbLordnCredentials(tld)
+        .ifPresent(login -> UrlConnectionUtils.setBasicAuth(connection, login));
   }
 
   /** Returns the username and password for the current TLD to login to the MarksDB server. */

core/src/main/java/google/registry/tmch/Marksdb.java

@@ -14,25 +14,23 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
 import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
 import static google.registry.util.HexDumper.dumpHex;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
-import google.registry.util.UrlFetchException;
+import google.registry.request.UrlConnectionService;
+import google.registry.util.UrlConnectionException;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.Security;
 import java.security.SignatureException;
@@ -57,7 +55,8 @@ public final class Marksdb {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final int MAX_DNL_LOGGING_LENGTH = 500;
 
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Key("marksdbPublicKey") PGPPublicKey marksdbPublicKey;
   @Inject Marksdb() {}
@@ -112,19 +111,16 @@ public final class Marksdb {
   }
 
   byte[] fetch(URL url, Optional<String> loginAndPassword) throws IOException {
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
-    setAuthorizationHeader(req, loginAndPassword);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
+    loginAndPassword.ifPresent(auth -> setBasicAuth(connection, auth));
     try {
-      rsp = fetchService.fetch(req);
+      if (connection.getResponseCode() != SC_OK) {
+        throw new UrlConnectionException("Failed to fetch from MarksDB", connection);
+      }
+      return getResponseBytes(connection);
     } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    if (rsp.getResponseCode() != SC_OK) {
-      throw new UrlFetchException("Failed to fetch from MarksDB", req, rsp);
-    }
-    return rsp.getContent();
   }
 
   List<String> fetchSignedCsv(Optional<String> loginAndPassword, String csvPath, String sigPath)

core/src/main/java/google/registry/tmch/NordnUploadAction.java

@@ -16,28 +16,23 @@ package google.registry.tmch;
 
 import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.POST;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.CSV_UTF_8;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_CLAIMS;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 
+import com.google.api.client.http.HttpMethods;
 import com.google.appengine.api.taskqueue.LeaseOptions;
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
@@ -49,16 +44,18 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
+import google.registry.request.UrlConnectionService;
+import google.registry.request.UrlConnectionUtils;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
-import java.util.Optional;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
@@ -97,7 +94,8 @@ public final class NordnUploadAction implements Runnable {
   @Inject Retrier retrier;
   @Inject SecureRandom random;
   @Inject LordnRequestInitializer lordnRequestInitializer;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Parameter(LORDN_PHASE_PARAM) String phase;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -193,47 +191,48 @@ public final class NordnUploadAction implements Runnable {
    * <p>Idempotency: If the exact same LORDN report is uploaded twice, the MarksDB server will
    * return the same confirmation number.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">
-   *     TMCH functional specifications - LORDN File</a>
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">TMCH
+   *     functional specifications - LORDN File</a>
    */
   private void uploadCsvToLordn(String urlPath, String csvData) throws IOException {
     String url = tmchMarksdbUrl + urlPath;
     logger.atInfo().log(
         "LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData);
-    HTTPRequest req = new HTTPRequest(new URL(url), POST, validateCertificate().setDeadline(60d));
-    lordnRequestInitializer.initialize(req, tld);
-    setPayloadMultipart(req, "file", "claims.csv", CSV_UTF_8, csvData, random);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(new URL(url));
+    connection.setRequestMethod(HttpMethods.POST);
+    lordnRequestInitializer.initialize(connection, tld);
+    UrlConnectionUtils.setPayloadMultipart(
+        connection, "file", "claims.csv", CSV_UTF_8, csvData, random);
     try {
-      rsp = fetchService.fetch(req);
+      int responseCode = connection.getResponseCode();
+      if (logger.atInfo().isEnabled()) {
+        String responseContent = new String(getResponseBytes(connection), US_ASCII);
+        if (responseContent.isEmpty()) {
+          responseContent = "(null)";
+        }
+        logger.atInfo().log(
+            "LORDN upload task %s response: HTTP response code %d, response data: %s",
+            actionLogId, responseCode, responseContent);
+      }
+      if (responseCode != SC_ACCEPTED) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB",
+                actionLogId),
+            connection);
+      }
+      String location = connection.getHeaderField(LOCATION);
+      if (location == null) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN upload task %s error: MarksDB failed to provide a Location header",
+                actionLogId),
+            connection);
+      }
+      getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location)));
     } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    if (logger.atInfo().isEnabled()) {
-      String response =
-          (rsp.getContent() == null) ? "(null)" : new String(rsp.getContent(), US_ASCII);
-      logger.atInfo().log(
-          "LORDN upload task %s response: HTTP response code %d, response data: %s",
-          actionLogId, rsp.getResponseCode(), response);
-    }
-    if (rsp.getResponseCode() != SC_ACCEPTED) {
-      throw new UrlFetchException(
-          String.format(
-              "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", actionLogId),
-          req,
-          rsp);
-    }
-    Optional<String> location = getHeaderFirst(rsp, LOCATION);
-    if (!location.isPresent()) {
-      throw new UrlFetchException(
-          String.format(
-              "LORDN upload task %s error: MarksDB failed to provide a Location header",
-              actionLogId),
-          req,
-          rsp);
-    }
-    getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location.get())));
   }
 
   private TaskOptions makeVerifyTask(URL url) {

core/src/main/java/google/registry/tmch/NordnVerifyAction.java

@@ -14,15 +14,11 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
@@ -32,9 +28,11 @@ import google.registry.request.HttpException.ConflictException;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
+import google.registry.request.UrlConnectionService;
 import google.registry.request.auth.Auth;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Map.Entry;
 import javax.inject.Inject;
@@ -68,7 +66,8 @@ public final class NordnVerifyAction implements Runnable {
 
   @Inject LordnRequestInitializer lordnRequestInitializer;
   @Inject Response response;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Header(URL_HEADER) URL url;
   @Inject @Header(HEADER_ACTION_LOG_ID) String actionLogId;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -96,51 +95,49 @@ public final class NordnVerifyAction implements Runnable {
   @VisibleForTesting
   LordnLog verify() throws IOException {
     logger.atInfo().log("LORDN verify task %s: Sending request to URL %s", actionLogId, url);
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
-    lordnRequestInitializer.initialize(req, tld);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
+    lordnRequestInitializer.initialize(connection, tld);
     try {
-      rsp = fetchService.fetch(req);
-    } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
-    }
-    logger.atInfo().log(
-        "LORDN verify task %s response: HTTP response code %d, response data: %s",
-        actionLogId, rsp.getResponseCode(), rsp.getContent());
-    if (rsp.getResponseCode() == SC_NO_CONTENT) {
-      // Send a 400+ status code so App Engine will retry the task.
-      throw new ConflictException("Not ready");
-    }
-    if (rsp.getResponseCode() != SC_OK) {
-      throw new UrlFetchException(
-          String.format("LORDN verify task %s: Failed to verify LORDN upload to MarksDB.",
-              actionLogId),
-          req, rsp);
-    }
-    LordnLog log =
-        LordnLog.parse(ByteSource.wrap(rsp.getContent()).asCharSource(UTF_8).readLines());
-    if (log.getStatus() == LordnLog.Status.ACCEPTED) {
-      logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
-    } else {
-      logger.atSevere().log(
-          "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
-    }
-    for (Entry<String, LordnLog.Result> result : log) {
-      switch (result.getValue().getOutcome()) {
-        case OK:
-          break;
-        case WARNING:
-          // fall through
-        case ERROR:
-          logger.atWarning().log(result.toString());
-          break;
-        default:
-          logger.atWarning().log(
-              "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
-          break;
+      int responseCode = connection.getResponseCode();
+      logger.atInfo().log(
+          "LORDN verify task %s response: HTTP response code %d", actionLogId, responseCode);
+      if (responseCode == SC_NO_CONTENT) {
+        // Send a 400+ status code so App Engine will retry the task.
+        throw new ConflictException("Not ready");
       }
+      if (responseCode != SC_OK) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN verify task %s: Failed to verify LORDN upload to MarksDB.", actionLogId),
+            connection);
+      }
+      LordnLog log =
+          LordnLog.parse(
+              ByteSource.wrap(getResponseBytes(connection)).asCharSource(UTF_8).readLines());
+      if (log.getStatus() == LordnLog.Status.ACCEPTED) {
+        logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
+      } else {
+        logger.atSevere().log(
+            "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
+      }
+      for (Entry<String, LordnLog.Result> result : log) {
+        switch (result.getValue().getOutcome()) {
+          case OK:
+            break;
+          case WARNING:
+            // fall through
+          case ERROR:
+            logger.atWarning().log(result.toString());
+            break;
+          default:
+            logger.atWarning().log(
+                "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
+            break;
+        }
+      }
+      return log;
+    } catch (IOException e) {
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    return log;
   }
 }

core/src/main/java/google/registry/tools/DigestType.java

@@ -14,6 +14,9 @@
 
 package google.registry.tools;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import java.util.Arrays;
 import java.util.Optional;
 
 /**
@@ -26,33 +29,38 @@ import java.util.Optional;
  * https://www.iana.org/assignments/ds-rr-types/ds-rr-types.xhtml
  */
 public enum DigestType {
-  SHA1(1),
-  SHA256(2),
+  SHA1(1, 20),
+  SHA256(2, 32),
   // Algorithm number 3 is GOST R 34.11-94 and is deliberately NOT SUPPORTED.
   // This algorithm was reviewed by ise-crypto and deemed academically broken (b/207029800).
   // In addition, RFC 8624 specifies that this algorithm MUST NOT be used for DNSSEC delegations.
   // TODO(sarhabot@): Add note in Cloud DNS code to notify the Registry of any new changes to
   // supported digest types.
-  SHA384(4);
+  SHA384(4, 48);
 
   private final int wireValue;
+  private final int bytes;
 
-  DigestType(int wireValue) {
+  DigestType(int wireValue, int bytes) {
     this.wireValue = wireValue;
+    this.bytes = bytes;
   }
 
+  private static final ImmutableMap<Integer, DigestType> WIRE_VALUE_TO_DIGEST_TYPE =
+      Maps.uniqueIndex(Arrays.stream(DigestType.values()).iterator(), DigestType::getWireValue);
+
   /** Fetches a DigestType enumeration constant by its IANA assigned value. */
   public static Optional<DigestType> fromWireValue(int wireValue) {
-    for (DigestType alg : DigestType.values()) {
-      if (alg.getWireValue() == wireValue) {
-        return Optional.of(alg);
-      }
-    }
-    return Optional.empty();
+    return Optional.ofNullable(WIRE_VALUE_TO_DIGEST_TYPE.get(wireValue));
   }
 
   /** Fetches a value in the range [0, 255] that encodes this DS digest type on the wire. */
   public int getWireValue() {
     return wireValue;
   }
+
+  /** Returns the expected length in bytes of the signature. */
+  public int getBytes() {
+    return bytes;
+  }
 }

core/src/main/java/google/registry/tools/DsRecord.java

@@ -51,6 +51,11 @@ abstract class DsRecord {
     checkArgumentPresent(
         DigestType.fromWireValue(digestType),
         String.format("DS record uses an unrecognized digest type: %d", digestType));
+    if (DigestType.fromWireValue(digestType).get().getBytes()
+        != BaseEncoding.base16().decode(digest).length) {
+      throw new IllegalArgumentException(
+          String.format("DS record has an invalid digest length: %s", digest));
+    }
 
     if (!DomainFlowUtils.validateAlgorithm(alg)) {
       throw new IllegalArgumentException(

core/src/main/java/google/registry/tools/GenerateEscrowDepositCommand.java

@@ -33,7 +33,6 @@ import google.registry.model.rde.RdeMode;
 import google.registry.rde.RdeStagingAction;
 import google.registry.request.Action.Service;
 import google.registry.tools.params.DateTimeParameter;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.CloudTasksUtils;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -90,8 +89,6 @@ final class GenerateEscrowDepositCommand implements CommandWithRemoteApi {
       required = true)
   private String outdir;
 
-  @Inject AppEngineServiceUtils appEngineServiceUtils;
-
   @Inject CloudTasksUtils cloudTasksUtils;
 
   @Override

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -38,8 +38,7 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
-import google.registry.request.Modules.URLFetchServiceModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
@@ -79,8 +78,7 @@ import javax.inject.Singleton;
       RegistryToolDataflowModule.class,
       RequestFactoryModule.class,
       SecretManagerModule.class,
-      URLFetchServiceModule.class,
-      UrlFetchTransportModule.class,
+      UrlConnectionServiceModule.class,
       UserServiceModule.class,
       UtilsModule.class,
       VoidDnsWriterModule.class,

core/src/main/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java

@@ -59,7 +59,6 @@ import google.registry.ui.forms.FormException;
 import google.registry.ui.forms.FormFieldException;
 import google.registry.ui.server.RegistrarFormFields;
 import google.registry.ui.server.SendEmailUtils;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.CollectionUtils;
 import google.registry.util.DiffUtils;
@@ -108,7 +107,6 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
   private static ThreadLocal<Boolean> isInTestDriver = ThreadLocal.withInitial(() -> false);
 
   @Inject JsonActionRunner jsonActionRunner;
-  @Inject AppEngineServiceUtils appEngineServiceUtils;
   @Inject RegistrarConsoleMetrics registrarConsoleMetrics;
   @Inject SendEmailUtils sendEmailUtils;
   @Inject AuthenticatedRegistrarAccessor registrarAccessor;

core/src/main/java/google/registry/whois/NameserverLookupByIpCommand.java

@@ -64,11 +64,17 @@ final class NameserverLookupByIpCommand implements WhoisCommand {
           jpaTm()
               .transact(
                   () ->
-                      // We cannot query @Convert-ed fields in HQL so we must use native Postgres
+                      // We cannot query @Convert-ed fields in HQL so we must use native Postgres.
                       jpaTm()
                           .getEntityManager()
+                          /**
+                           * Using array_operator <@ (contained-by) with gin index on inet_address.
+                           * Without gin index, this is slightly slower than the alternative form of
+                           * ':address = ANY(inet_address)'.
+                           */
                           .createNativeQuery(
-                              "SELECT * From \"Host\" WHERE :address = ANY(inet_addresses) AND "
+                              "SELECT * From \"Host\" WHERE "
+                                  + "ARRAY[ CAST(:address AS TEXT) ] <@ inet_addresses AND "
                                   + "deletion_time > CAST(:now AS timestamptz)",
                               HostResource.class)
                           .setParameter("address", InetAddresses.toAddrString(ipAddress))

core/src/test/java/google/registry/backup/CommitLogCheckpointActionTest.java

@@ -47,11 +47,10 @@ public class CommitLogCheckpointActionTest {
 
   private DateTime now = DateTime.now(UTC);
   private CommitLogCheckpointAction task = new CommitLogCheckpointAction();
-  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(new FakeClock(now));
 
   @BeforeEach
   void beforeEach() {
-    task.clock = new FakeClock(now);
     task.strategy = strategy;
     task.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
     when(strategy.computeCheckpoint())
@@ -68,7 +67,8 @@ public class CommitLogCheckpointActionTest {
         new TaskMatcher()
             .url(ExportCommitLogDiffAction.PATH)
             .param(ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM, START_OF_TIME.toString())
-            .param(ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM, now.toString()));
+            .param(ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM, now.toString())
+            .scheduleTime(now.plus(CommitLogCheckpointAction.ENQUEUE_DELAY_SECONDS)));
     assertThat(loadRoot().getLastWrittenTime()).isEqualTo(now);
   }
 
@@ -82,7 +82,8 @@ public class CommitLogCheckpointActionTest {
         new TaskMatcher()
             .url(ExportCommitLogDiffAction.PATH)
             .param(ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM, oneMinuteAgo.toString())
-            .param(ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM, now.toString()));
+            .param(ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM, now.toString())
+            .scheduleTime(now.plus(CommitLogCheckpointAction.ENQUEUE_DELAY_SECONDS)));
     assertThat(loadRoot().getLastWrittenTime()).isEqualTo(now);
   }
 

core/src/test/java/google/registry/batch/AsyncTaskEnqueuerTest.java

@@ -22,23 +22,20 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestLogHandlerUtils.assertLogMessage;
-import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardHours;
 import static org.joda.time.Duration.standardSeconds;
-import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSortedSet;
 import google.registry.model.contact.ContactResource;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.CapturingLogHandler;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.JdkLoggerConfig;
 import google.registry.util.Retrier;
 import java.util.logging.Level;
@@ -48,7 +45,6 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
@@ -63,27 +59,25 @@ public class AsyncTaskEnqueuerTest {
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  @Mock private AppEngineServiceUtils appEngineServiceUtils;
-
   private AsyncTaskEnqueuer asyncTaskEnqueuer;
   private final CapturingLogHandler logHandler = new CapturingLogHandler();
   private final FakeClock clock = new FakeClock(DateTime.parse("2015-05-18T12:34:56Z"));
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @BeforeEach
   void beforeEach() {
     JdkLoggerConfig.getConfig(AsyncTaskEnqueuer.class).addHandler(logHandler);
-    when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
-    asyncTaskEnqueuer = createForTesting(appEngineServiceUtils, clock, standardSeconds(90));
+    asyncTaskEnqueuer =
+        createForTesting(cloudTasksHelper.getTestCloudTasksUtils(), clock, standardSeconds(90));
   }
 
   public static AsyncTaskEnqueuer createForTesting(
-      AppEngineServiceUtils appEngineServiceUtils, FakeClock clock, Duration asyncDeleteDelay) {
+      CloudTasksUtils cloudTasksUtils, FakeClock clock, Duration asyncDeleteDelay) {
     return new AsyncTaskEnqueuer(
-        getQueue(QUEUE_ASYNC_ACTIONS),
         getQueue(QUEUE_ASYNC_DELETE),
         getQueue(QUEUE_ASYNC_HOST_RENAME),
         asyncDeleteDelay,
-        appEngineServiceUtils,
+        cloudTasksUtils,
         new Retrier(new FakeSleeper(clock), 1));
   }
 
@@ -92,18 +86,16 @@ public class AsyncTaskEnqueuerTest {
     ContactResource contact = persistActiveContact("jd23456");
     asyncTaskEnqueuer.enqueueAsyncResave(
         contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(5));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
-        new TaskMatcher()
+        new CloudTasksHelper.TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
-            .etaDelta(
-                standardDays(5).minus(standardSeconds(30)),
-                standardDays(5).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(Duration.standardDays(5))));
   }
 
   @Test
@@ -114,19 +106,17 @@ public class AsyncTaskEnqueuerTest {
         contact.createVKey(),
         now,
         ImmutableSortedSet.of(now.plusHours(24), now.plusHours(50), now.plusHours(75)));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, now.toString())
             .param(PARAM_RESAVE_TIMES, "2015-05-20T14:34:56.000Z,2015-05-21T15:34:56.000Z")
-            .etaDelta(
-                standardHours(24).minus(standardSeconds(30)),
-                standardHours(24).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(Duration.standardHours(24))));
   }
 
   @MockitoSettings(strictness = Strictness.LENIENT)
@@ -135,7 +125,7 @@ public class AsyncTaskEnqueuerTest {
     ContactResource contact = persistActiveContact("jd23456");
     asyncTaskEnqueuer.enqueueAsyncResave(
         contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(31));
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
     assertLogMessage(logHandler, Level.INFO, "Ignoring async re-save");
   }
 }

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -91,13 +91,13 @@ import google.registry.model.transfer.ContactTransferData;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.mapreduce.MapreduceTestCase;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.Retrier;
 import google.registry.util.Sleeper;
@@ -148,7 +148,7 @@ public class DeleteContactsAndHostsActionTest
     inject.setStaticField(Ofy.class, "clock", clock);
     enqueuer =
         AsyncTaskEnqueuerTest.createForTesting(
-            mock(AppEngineServiceUtils.class), clock, Duration.ZERO);
+            new CloudTasksHelper(clock).getTestCloudTasksUtils(), clock, Duration.ZERO);
     AsyncTaskMetrics asyncTaskMetricsMock = mock(AsyncTaskMetrics.class);
     action = new DeleteContactsAndHostsAction();
     action.asyncTaskMetrics = asyncTaskMetricsMock;

core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java

@@ -39,7 +39,6 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.persistence.transaction.QueryComposer.Comparator;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DualDatabaseTest;
@@ -78,8 +77,7 @@ class DeleteExpiredDomainsActionTest {
     createTld("tld");
     EppController eppController =
         DaggerEppTestComponent.builder()
-            .fakesAndMocksModule(
-                FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock)))
+            .fakesAndMocksModule(FakesAndMocksModule.create(clock))
             .build()
             .startRequest()
             .eppController();

core/src/test/java/google/registry/batch/RefreshDnsOnHostRenameActionTest.java

@@ -49,6 +49,7 @@ import google.registry.batch.RefreshDnsOnHostRenameAction.RefreshDnsOnHostRename
 import google.registry.dns.DnsQueue;
 import google.registry.model.host.HostResource;
 import google.registry.model.server.Lock;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
@@ -59,7 +60,6 @@ import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import google.registry.testing.TestSqlOnly;
 import google.registry.testing.mapreduce.MapreduceTestCase;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.Retrier;
 import google.registry.util.Sleeper;
@@ -89,7 +89,7 @@ public class RefreshDnsOnHostRenameActionTest
     createTld("tld");
     enqueuer =
         AsyncTaskEnqueuerTest.createForTesting(
-            mock(AppEngineServiceUtils.class), clock, Duration.ZERO);
+            new CloudTasksHelper(clock).getTestCloudTasksUtils(), clock, Duration.ZERO);
     AsyncTaskMetrics asyncTaskMetricsMock = mock(AsyncTaskMetrics.class);
     action = new RefreshDnsOnHostRenameAction();
     action.asyncTaskMetrics = asyncTaskMetricsMock;

core/src/test/java/google/registry/batch/RelockDomainActionTest.java

@@ -31,8 +31,6 @@ import static google.registry.testing.SqlHelper.saveRegistryLock;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.mockito.Mockito.lenient;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 
@@ -51,7 +49,6 @@ import google.registry.testing.FakeResponse;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.UserInfo;
 import google.registry.tools.DomainLockUtils;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import google.registry.util.StringGenerator.Alphabets;
@@ -111,11 +108,6 @@ public class RelockDomainActionTest {
             DOMAIN_NAME, CLIENT_ID, false, Optional.empty());
     assertThat(loadByEntity(domain).getStatusValues()).containsNoneIn(REGISTRY_LOCK_STATUSES);
 
-    AppEngineServiceUtils appEngineServiceUtils = mock(AppEngineServiceUtils.class);
-    lenient()
-        .when(appEngineServiceUtils.getServiceHostname("backend"))
-        .thenReturn("backend.hostname.fake");
-
     action = createAction(oldLock.getRevisionId());
   }
 

core/src/test/java/google/registry/batch/ResaveEntityActionTest.java

@@ -25,12 +25,10 @@ import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static google.registry.testing.DatabaseHelper.persistDomainWithDependentResources;
 import static google.registry.testing.DatabaseHelper.persistDomainWithPendingTransfer;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardSeconds;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import google.registry.model.domain.DomainBase;
@@ -40,12 +38,12 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.Ofy;
 import google.registry.request.Response;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
-import google.registry.util.AppEngineServiceUtils;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
@@ -67,17 +65,17 @@ public class ResaveEntityActionTest {
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  @Mock private AppEngineServiceUtils appEngineServiceUtils;
   @Mock private Response response;
   private final FakeClock clock = new FakeClock(DateTime.parse("2016-02-11T10:00:00Z"));
   private AsyncTaskEnqueuer asyncTaskEnqueuer;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @BeforeEach
   void beforeEach() {
     inject.setStaticField(Ofy.class, "clock", clock);
-    when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
     asyncTaskEnqueuer =
-        AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, Duration.ZERO);
+        AsyncTaskEnqueuerTest.createForTesting(
+            cloudTasksHelper.getTestCloudTasksUtils(), clock, Duration.ZERO);
     createTld("tld");
   }
 
@@ -143,17 +141,15 @@ public class ResaveEntityActionTest {
     DomainBase resavedDomain = loadByEntity(domain);
     assertThat(resavedDomain.getGracePeriods()).isEmpty();
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, resavedDomain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, requestedTime.toString())
-            .etaDelta(
-                standardDays(5).minus(standardSeconds(30)),
-                standardDays(5).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(standardDays(5))));
   }
 }

core/src/test/java/google/registry/batch/SendExpiringCertificateNotificationEmailActionTest.java

@@ -59,49 +59,18 @@ class SendExpiringCertificateNotificationEmailActionTest {
   private static final String EXPIRATION_WARNING_EMAIL_BODY_TEXT =
       " Dear %1$s,\n"
           + "\n"
-          + "    We would like to inform you that your %2$s SSL certificate will expire at\n"
-          + "    %3$s. Please take note that using expired certificates will prevent\n"
-          + "    successful Registry login.\n"
+          + "We would like to inform you that your %2$s certificate will expire at %3$s."
           + "\n"
-          + "    Kindly update your production account certificate within the support\n"
-          + "    console using the following steps:\n"
+          + " Kind update your account using the following steps: "
           + "\n"
-          + "      1. Navigate to support.registry.google and login using your\n"
-          + "    %4$s@registry.google credentials.\n"
-          + "          * If this is your first time logging in, you will be prompted to\n"
-          + "          reset your password, so please keep your new password safe.\n"
-          + "          * If you are already logged in with some other Google account(s) but\n"
-          + "          not your %4$s@registry.google account, you need to click on\n"
-          + "          “Add Account” and login using your %4$s@registry.google credentials.\n"
-          + "      2. Select “Settings > Security” from the left navigation bar.\n"
-          + "      3. Click “Edit” on the top left corner.\n"
-          + "      4. Enter your full certificate string\n"
-          + "         (including lines -----BEGIN CERTIFICATE----- and\n"
-          + "         -----END CERTIFICATE-----) in the box.\n"
-          + "      5. Click “Save”. If there are validation issues with the form, you will\n"
-          + "         be prompted to fix them and click “Save” again.\n"
-          + "\n"
-          + "    A failover SSL certificate can also be added in order to prevent connection\n"
-          + "    issues once your main certificate expires. Connecting with either of the\n"
-          + "    certificates will work with our production EPP server.\n"
-          + "\n"
-          + "    Further information about our EPP connection requirements can be found in\n"
-          + "    section 9.2 in the updated Technical Guide in your Google Drive folder.\n"
-          + "\n"
-          + "    Note that account certificate changes take a few minutes to become\n"
-          + "    effective and that the existing connections will remain unaffected by\n"
-          + "    the change.\n"
-          + "\n"
-          + "    If you also would like to update your OT&E account certificate, please send\n"
-          + "    an email from your primary or technical contact to\n"
-          + "    registry-support@google.com and include the full certificate string\n"
-          + "    (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).\n"
-          + "\n"
-          + "    Regards,\n"
-          + "    Google Registry\n";
+          + "  1. Navigate to support and login using your %4$s@registry.example credentials.\n"
+          + "  2. Click Settings -> Privacy on the top left corner.\n"
+          + "  3. Click Edit and enter certificate string."
+          + "  3. Click Save"
+          + "Regards,"
+          + "Example Registry";
 
-  private static final String EXPIRATION_WARNING_EMAIL_SUBJECT_TEXT =
-      "[Important] Expiring SSL certificate for Google " + "Registry EPP connection";
+  private static final String EXPIRATION_WARNING_EMAIL_SUBJECT_TEXT = "Expiration Warning Email";
 
   @RegisterExtension
   public final AppEngineExtension appEngine =
@@ -623,11 +592,11 @@ class SendExpiringCertificateNotificationEmailActionTest {
     assertThat(emailBody).contains(registrarName);
     assertThat(emailBody).contains(certificateType.getDisplayName());
     assertThat(emailBody).contains(certExpirationDateStr);
-    assertThat(emailBody).contains(registrarId + "@registry.google");
-    assertThat(emailBody).doesNotContain("%1$s@registry.google");
-    assertThat(emailBody).doesNotContain("%2$s@registry.google");
-    assertThat(emailBody).doesNotContain("%3$s@registry.google");
-    assertThat(emailBody).doesNotContain("%4$s@registry.google");
+    assertThat(emailBody).contains(registrarId + "@registry.example");
+    assertThat(emailBody).doesNotContain("%1$s");
+    assertThat(emailBody).doesNotContain("%2$s");
+    assertThat(emailBody).doesNotContain("%3$s");
+    assertThat(emailBody).doesNotContain("%4$s");
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/export/BackupDatastoreActionTest.java

@@ -17,16 +17,19 @@ package google.registry.export;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM;
 import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.base.Joiner;
+import com.google.protobuf.util.Timestamps;
 import google.registry.export.datastore.DatastoreAdmin;
 import google.registry.export.datastore.DatastoreAdmin.Export;
 import google.registry.export.datastore.Operation;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
+import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -46,13 +49,15 @@ public class BackupDatastoreActionTest {
   @Mock private Operation backupOperation;
 
   private final FakeResponse response = new FakeResponse();
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
   private final BackupDatastoreAction action = new BackupDatastoreAction();
 
   @BeforeEach
   void beforeEach() throws Exception {
     action.datastoreAdmin = datastoreAdmin;
     action.response = response;
-
+    action.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
+    action.clock = new FakeClock();
     when(datastoreAdmin.export(
             "gs://registry-project-id-datastore-backups", AnnotatedEntities.getBackupKinds()))
         .thenReturn(exportRequest);
@@ -66,7 +71,7 @@ public class BackupDatastoreActionTest {
   @Test
   void testBackup_enqueuesPollTask() {
     action.run();
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         CheckBackupAction.QUEUE,
         new TaskMatcher()
             .url(CheckBackupAction.PATH)
@@ -74,7 +79,10 @@ public class BackupDatastoreActionTest {
             .param(
                 CHECK_BACKUP_KINDS_TO_LOAD_PARAM,
                 Joiner.on(",").join(AnnotatedEntities.getReportingKinds()))
-            .method("POST"));
+            .method(HttpMethod.POST)
+            .scheduleTime(
+                Timestamps.fromMillis(
+                    action.clock.nowUtc().plus(CheckBackupAction.POLL_COUNTDOWN).getMillis())));
     assertThat(response.getPayload())
         .isEqualTo(
             "Datastore backup started with name: "

core/src/test/java/google/registry/export/BigqueryPollJobActionTest.java

@@ -14,11 +14,7 @@
 
 package google.registry.export;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.common.collect.Iterables.getOnlyElement;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestLogHandlerUtils.assertLogMessage;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.logging.Level.INFO;
@@ -30,27 +26,22 @@ import static org.mockito.Mockito.when;
 import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.model.ErrorProto;
 import com.google.api.services.bigquery.model.Job;
-import com.google.api.services.bigquery.model.JobReference;
 import com.google.api.services.bigquery.model.JobStatus;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
-import com.google.appengine.api.taskqueue.dev.QueueStateInfo.TaskStateInfo;
-import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
+import com.google.cloud.tasks.v2.AppEngineHttpRequest;
+import com.google.cloud.tasks.v2.HttpMethod;
+import com.google.cloud.tasks.v2.Task;
+import com.google.common.net.HttpHeaders;
+import com.google.common.net.MediaType;
+import com.google.protobuf.ByteString;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.HttpException.NotModifiedException;
 import google.registry.testing.AppEngineExtension;
-import google.registry.testing.FakeClock;
-import google.registry.testing.FakeSleeper;
-import google.registry.testing.TaskQueueHelper;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.util.CapturingLogHandler;
 import google.registry.util.JdkLoggerConfig;
-import google.registry.util.Retrier;
-import google.registry.util.TaskQueueUtils;
-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -66,8 +57,6 @@ public class BigqueryPollJobActionTest {
   private static final String PROJECT_ID = "project_id";
   private static final String JOB_ID = "job_id";
   private static final String CHAINED_QUEUE_NAME = UpdateSnapshotViewAction.QUEUE;
-  private static final TaskQueueUtils TASK_QUEUE_UTILS =
-      new TaskQueueUtils(new Retrier(new FakeSleeper(new FakeClock()), 1));
 
   private final Bigquery bigquery = mock(Bigquery.class);
   private final Bigquery.Jobs bigqueryJobs = mock(Bigquery.Jobs.class);
@@ -75,53 +64,20 @@ public class BigqueryPollJobActionTest {
 
   private final CapturingLogHandler logHandler = new CapturingLogHandler();
   private BigqueryPollJobAction action = new BigqueryPollJobAction();
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
 
   @BeforeEach
   void beforeEach() throws Exception {
     action.bigquery = bigquery;
     when(bigquery.jobs()).thenReturn(bigqueryJobs);
     when(bigqueryJobs.get(PROJECT_ID, JOB_ID)).thenReturn(bigqueryJobsGet);
-    action.taskQueueUtils = TASK_QUEUE_UTILS;
+    action.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
     action.projectId = PROJECT_ID;
     action.jobId = JOB_ID;
     action.chainedQueueName = () -> CHAINED_QUEUE_NAME;
     JdkLoggerConfig.getConfig(BigqueryPollJobAction.class).addHandler(logHandler);
   }
 
-  private static TaskMatcher newPollJobTaskMatcher(String method) {
-    return new TaskMatcher()
-        .method(method)
-        .url(BigqueryPollJobAction.PATH)
-        .header(BigqueryPollJobAction.PROJECT_ID_HEADER, PROJECT_ID)
-        .header(BigqueryPollJobAction.JOB_ID_HEADER, JOB_ID);
-  }
-
-  @Test
-  void testSuccess_enqueuePollTask() {
-    new BigqueryPollJobEnqueuer(TASK_QUEUE_UTILS).enqueuePollTask(
-        new JobReference().setProjectId(PROJECT_ID).setJobId(JOB_ID));
-    assertTasksEnqueued(BigqueryPollJobAction.QUEUE, newPollJobTaskMatcher("GET"));
-  }
-
-  @Test
-  void testSuccess_enqueuePollTask_withChainedTask() throws Exception {
-    TaskOptions chainedTask = TaskOptions.Builder
-        .withUrl("/_dr/something")
-        .method(Method.POST)
-        .header("X-Testing", "foo")
-        .param("testing", "bar");
-    new BigqueryPollJobEnqueuer(TASK_QUEUE_UTILS).enqueuePollTask(
-        new JobReference().setProjectId(PROJECT_ID).setJobId(JOB_ID),
-        chainedTask,
-        getQueue(CHAINED_QUEUE_NAME));
-    assertTasksEnqueued(BigqueryPollJobAction.QUEUE, newPollJobTaskMatcher("POST"));
-    TaskStateInfo taskInfo = getOnlyElement(
-        TaskQueueHelper.getQueueInfo(BigqueryPollJobAction.QUEUE).getTaskInfo());
-    ByteArrayInputStream taskBodyBytes = new ByteArrayInputStream(taskInfo.getBodyAsBytes());
-    TaskOptions taskOptions = (TaskOptions) new ObjectInputStream(taskBodyBytes).readObject();
-    assertThat(taskOptions).isEqualTo(chainedTask);
-  }
-
   @Test
   void testSuccess_jobCompletedSuccessfully() throws Exception {
     when(bigqueryJobsGet.execute()).thenReturn(
@@ -136,15 +92,20 @@ public class BigqueryPollJobActionTest {
     when(bigqueryJobsGet.execute()).thenReturn(
         new Job().setStatus(new JobStatus().setState("DONE")));
 
-    TaskOptions chainedTask =
-        TaskOptions.Builder.withUrl("/_dr/something")
-            .method(Method.POST)
-            .header("X-Testing", "foo")
-            .param("testing", "bar")
-            .taskName("my_task_name");
+    Task chainedTask =
+        Task.newBuilder()
+            .setName("my_task_name")
+            .setAppEngineHttpRequest(
+                AppEngineHttpRequest.newBuilder()
+                    .setHttpMethod(HttpMethod.POST)
+                    .setRelativeUri("/_dr/something")
+                    .putHeaders("X-Test", "foo")
+                    .putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
+                    .setBody(ByteString.copyFromUtf8("testing=bar")))
+            .build();
     ByteArrayOutputStream bytes = new ByteArrayOutputStream();
     new ObjectOutputStream(bytes).writeObject(chainedTask);
-    action.payload = bytes.toByteArray();
+    action.payload = ByteString.copyFrom(bytes.toByteArray());
 
     action.run();
     assertLogMessage(
@@ -153,14 +114,15 @@ public class BigqueryPollJobActionTest {
         logHandler,
         INFO,
         "Added chained task my_task_name for /_dr/something to queue " + CHAINED_QUEUE_NAME);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         CHAINED_QUEUE_NAME,
         new TaskMatcher()
             .url("/_dr/something")
-            .method("POST")
-            .header("X-Testing", "foo")
+            .header("X-Test", "foo")
+            .header(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
             .param("testing", "bar")
-            .taskName("my_task_name"));
+            .taskName("my_task_name")
+            .method(HttpMethod.POST));
   }
 
   @Test
@@ -172,7 +134,7 @@ public class BigqueryPollJobActionTest {
     action.run();
     assertLogMessage(
         logHandler, SEVERE, String.format("Bigquery job failed - %s:%s", PROJECT_ID, JOB_ID));
-    assertNoTasksEnqueued(CHAINED_QUEUE_NAME);
+    cloudTasksHelper.assertNoTasksEnqueued(CHAINED_QUEUE_NAME);
   }
 
   @Test
@@ -192,7 +154,7 @@ public class BigqueryPollJobActionTest {
   void testFailure_badChainedTaskPayload() throws Exception {
     when(bigqueryJobsGet.execute()).thenReturn(
         new Job().setStatus(new JobStatus().setState("DONE")));
-    action.payload = "payload".getBytes(UTF_8);
+    action.payload = ByteString.copyFrom("payload".getBytes(UTF_8));
     BadRequestException thrown = assertThrows(BadRequestException.class, action::run);
     assertThat(thrown).hasMessageThat().contains("Cannot deserialize task from payload");
   }

core/src/test/java/google/registry/export/CheckBackupActionTest.java

@@ -15,10 +15,6 @@
 package google.registry.export;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.export.CheckBackupAction.CHECK_BACKUP_KINDS_TO_LOAD_PARAM;
-import static google.registry.export.CheckBackupAction.CHECK_BACKUP_NAME_PARAM;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.when;
@@ -27,7 +23,7 @@ import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.client.http.HttpHeaders;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
-import com.google.common.collect.ImmutableSet;
+import com.google.cloud.tasks.v2.HttpMethod;
 import google.registry.export.datastore.DatastoreAdmin;
 import google.registry.export.datastore.DatastoreAdmin.Get;
 import google.registry.export.datastore.Operation;
@@ -36,9 +32,10 @@ import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.HttpException.NoContentException;
 import google.registry.request.HttpException.NotModifiedException;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestDataHelper;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
@@ -57,7 +54,6 @@ public class CheckBackupActionTest {
 
   private static final DateTime START_TIME = DateTime.parse("2014-08-01T01:02:03Z");
   private static final DateTime COMPLETE_TIME = START_TIME.plus(Duration.standardMinutes(30));
-
   private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
 
   @RegisterExtension
@@ -71,6 +67,7 @@ public class CheckBackupActionTest {
   private final FakeResponse response = new FakeResponse();
   private final FakeClock clock = new FakeClock(COMPLETE_TIME.plusMillis(1000));
   private final CheckBackupAction action = new CheckBackupAction();
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
 
   @BeforeEach
   void beforeEach() throws Exception {
@@ -80,6 +77,7 @@ public class CheckBackupActionTest {
     action.backupName = "some_backup";
     action.kindsToLoadParam = "one,two";
     action.response = response;
+    action.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
 
     when(datastoreAdmin.get(anyString())).thenReturn(getBackupProgressRequest);
     when(getBackupProgressRequest.execute()).thenAnswer(arg -> backupOperation);
@@ -110,30 +108,17 @@ public class CheckBackupActionTest {
                 null));
   }
 
-  private static void assertLoadTaskEnqueued(String id, String folder, String kinds) {
-    assertTasksEnqueued(
+  private void assertLoadTaskEnqueued(String id, String folder, String kinds) {
+    cloudTasksHelper.assertTasksEnqueued(
         "export-snapshot",
         new TaskMatcher()
             .url("/_dr/task/uploadDatastoreBackup")
-            .method("POST")
+            .method(HttpMethod.POST)
             .param("id", id)
             .param("folder", folder)
             .param("kinds", kinds));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
-  @Test
-  void testSuccess_enqueuePollTask() {
-    CheckBackupAction.enqueuePollTask("some_backup_name", ImmutableSet.of("one", "two", "three"));
-    assertTasksEnqueued(
-        CheckBackupAction.QUEUE,
-        new TaskMatcher()
-            .url(CheckBackupAction.PATH)
-            .param(CHECK_BACKUP_NAME_PARAM, "some_backup_name")
-            .param(CHECK_BACKUP_KINDS_TO_LOAD_PARAM, "one,two,three")
-            .method("POST"));
-  }
-
   @Test
   void testPost_forPendingBackup_returnsNotModified() throws Exception {
     setPendingBackup();
@@ -189,7 +174,7 @@ public class CheckBackupActionTest {
     action.kindsToLoadParam = "";
 
     action.run();
-    assertNoTasksEnqueued("export-snapshot");
+    cloudTasksHelper.assertNoTasksEnqueued("export-snapshot");
   }
 
   @MockitoSettings(strictness = Strictness.LENIENT)

core/src/test/java/google/registry/export/UpdateSnapshotViewActionTest.java

@@ -14,15 +14,7 @@
 
 package google.registry.export;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.export.UpdateSnapshotViewAction.QUEUE;
-import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
-import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
-import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
-import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM;
-import static google.registry.export.UpdateSnapshotViewAction.createViewUpdateTask;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
@@ -38,7 +30,6 @@ import com.google.common.collect.Iterables;
 import google.registry.bigquery.CheckedBigquery;
 import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.testing.AppEngineExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.io.IOException;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -81,23 +72,6 @@ public class UpdateSnapshotViewActionTest {
     action.tableId = "12345_fookind";
   }
 
-  @Test
-  void testSuccess_createViewUpdateTask() {
-    getQueue(QUEUE)
-        .add(
-            createViewUpdateTask(
-                "some_dataset", "12345_fookind", "fookind", "latest_datastore_export"));
-    assertTasksEnqueued(
-        QUEUE,
-        new TaskMatcher()
-            .url(UpdateSnapshotViewAction.PATH)
-            .method("POST")
-            .param(UPDATE_SNAPSHOT_DATASET_ID_PARAM, "some_dataset")
-            .param(UPDATE_SNAPSHOT_TABLE_ID_PARAM, "12345_fookind")
-            .param(UPDATE_SNAPSHOT_KIND_PARAM, "fookind")
-            .param(UPDATE_SNAPSHOT_VIEWNAME_PARAM, "latest_datastore_export"));
-  }
-
   @Test
   void testSuccess_doPost() throws Exception {
     action.run();

core/src/test/java/google/registry/export/UploadDatastoreBackupActionTest.java

@@ -16,16 +16,16 @@ package google.registry.export;
 
 import static com.google.common.collect.Iterables.transform;
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.export.BigqueryPollJobAction.CHAINED_TASK_QUEUE_HEADER;
+import static google.registry.export.BigqueryPollJobAction.JOB_ID_HEADER;
+import static google.registry.export.BigqueryPollJobAction.PROJECT_ID_HEADER;
+import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_DATASET_ID_PARAM;
+import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_KIND_PARAM;
+import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_TABLE_ID_PARAM;
+import static google.registry.export.UpdateSnapshotViewAction.UPDATE_SNAPSHOT_VIEWNAME_PARAM;
 import static google.registry.export.UploadDatastoreBackupAction.BACKUP_DATASET;
-import static google.registry.export.UploadDatastoreBackupAction.LATEST_BACKUP_VIEW_NAME;
-import static google.registry.export.UploadDatastoreBackupAction.PATH;
-import static google.registry.export.UploadDatastoreBackupAction.QUEUE;
-import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_FOLDER_PARAM;
-import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_ID_PARAM;
-import static google.registry.export.UploadDatastoreBackupAction.UPLOAD_BACKUP_KINDS_PARAM;
-import static google.registry.export.UploadDatastoreBackupAction.enqueueUploadBackupTask;
 import static google.registry.export.UploadDatastoreBackupAction.getBackupInfoFileForKind;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -38,16 +38,20 @@ import com.google.api.services.bigquery.Bigquery;
 import com.google.api.services.bigquery.model.Dataset;
 import com.google.api.services.bigquery.model.Job;
 import com.google.api.services.bigquery.model.JobConfigurationLoad;
-import com.google.api.services.bigquery.model.JobReference;
-import com.google.appengine.api.taskqueue.QueueFactory;
-import com.google.common.collect.ImmutableSet;
+import com.google.cloud.tasks.v2.HttpMethod;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Iterables;
+import com.google.protobuf.util.Timestamps;
 import google.registry.bigquery.CheckedBigquery;
-import google.registry.export.BigqueryPollJobAction.BigqueryPollJobEnqueuer;
 import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.testing.AppEngineExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
+import google.registry.testing.FakeClock;
+import google.registry.util.CloudTasksUtils;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.ObjectInputStream;
 import java.util.List;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -67,8 +71,9 @@ public class UploadDatastoreBackupActionTest {
   private final Bigquery.Datasets bigqueryDatasets = mock(Bigquery.Datasets.class);
   private final Bigquery.Datasets.Insert bigqueryDatasetsInsert =
       mock(Bigquery.Datasets.Insert.class);
-  private final BigqueryPollJobEnqueuer bigqueryPollEnqueuer = mock(BigqueryPollJobEnqueuer.class);
   private UploadDatastoreBackupAction action;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
+  private CloudTasksUtils cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
 
   @BeforeEach
   void beforeEach() throws Exception {
@@ -80,25 +85,14 @@ public class UploadDatastoreBackupActionTest {
         .thenReturn(bigqueryDatasetsInsert);
     action = new UploadDatastoreBackupAction();
     action.checkedBigquery = checkedBigquery;
-    action.bigqueryPollEnqueuer = bigqueryPollEnqueuer;
     action.projectId = "Project-Id";
     action.backupFolderUrl = "gs://bucket/path";
     action.backupId = "2018-12-05T17:46:39_92612";
     action.backupKinds = "one,two,three";
+    action.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
+    action.clock = new FakeClock();
   }
 
-  @Test
-  void testSuccess_enqueueLoadTask() {
-    enqueueUploadBackupTask("id12345", "gs://bucket/path", ImmutableSet.of("one", "two", "three"));
-    assertTasksEnqueued(
-        QUEUE,
-        new TaskMatcher()
-            .url(PATH)
-            .method("POST")
-            .param(UPLOAD_BACKUP_ID_PARAM, "id12345")
-            .param(UPLOAD_BACKUP_FOLDER_PARAM, "gs://bucket/path")
-            .param(UPLOAD_BACKUP_KINDS_PARAM, "one,two,three"));
-  }
 
   @Test
   void testSuccess_doPost() throws Exception {
@@ -153,33 +147,91 @@ public class UploadDatastoreBackupActionTest {
     verify(bigqueryJobsInsert, times(3)).execute();
 
     // Check that the poll tasks for each load job were enqueued.
-    verify(bigqueryPollEnqueuer)
-        .enqueuePollTask(
-            new JobReference()
-                .setProjectId("Project-Id")
-                .setJobId("load-backup-2018_12_05T17_46_39_92612-one"),
-            UpdateSnapshotViewAction.createViewUpdateTask(
-                BACKUP_DATASET, "2018_12_05T17_46_39_92612_one", "one", LATEST_BACKUP_VIEW_NAME),
-            QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
-    verify(bigqueryPollEnqueuer)
-        .enqueuePollTask(
-            new JobReference()
-                .setProjectId("Project-Id")
-                .setJobId("load-backup-2018_12_05T17_46_39_92612-two"),
-            UpdateSnapshotViewAction.createViewUpdateTask(
-                BACKUP_DATASET, "2018_12_05T17_46_39_92612_two", "two", LATEST_BACKUP_VIEW_NAME),
-            QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
-    verify(bigqueryPollEnqueuer)
-        .enqueuePollTask(
-            new JobReference()
-                .setProjectId("Project-Id")
-                .setJobId("load-backup-2018_12_05T17_46_39_92612-three"),
-            UpdateSnapshotViewAction.createViewUpdateTask(
-                BACKUP_DATASET,
-                "2018_12_05T17_46_39_92612_three",
-                "three",
-                LATEST_BACKUP_VIEW_NAME),
-            QueueFactory.getQueue(UpdateSnapshotViewAction.QUEUE));
+    cloudTasksHelper.assertTasksEnqueued(
+        BigqueryPollJobAction.QUEUE,
+        new TaskMatcher()
+            .method(HttpMethod.POST)
+            .header(PROJECT_ID_HEADER, "Project-Id")
+            .header(JOB_ID_HEADER, "load-backup-2018_12_05T17_46_39_92612-one")
+            .header(CHAINED_TASK_QUEUE_HEADER, UpdateSnapshotViewAction.QUEUE)
+            .scheduleTime(
+                Timestamps.fromMillis(
+                    action.clock.nowUtc().plus(BigqueryPollJobAction.POLL_COUNTDOWN).getMillis())),
+        new TaskMatcher()
+            .method(HttpMethod.POST)
+            .header(PROJECT_ID_HEADER, "Project-Id")
+            .header(JOB_ID_HEADER, "load-backup-2018_12_05T17_46_39_92612-two")
+            .header(CHAINED_TASK_QUEUE_HEADER, UpdateSnapshotViewAction.QUEUE)
+            .scheduleTime(
+                Timestamps.fromMillis(
+                    action.clock.nowUtc().plus(BigqueryPollJobAction.POLL_COUNTDOWN).getMillis())),
+        new TaskMatcher()
+            .method(HttpMethod.POST)
+            .header(PROJECT_ID_HEADER, "Project-Id")
+            .header(JOB_ID_HEADER, "load-backup-2018_12_05T17_46_39_92612-three")
+            .header(CHAINED_TASK_QUEUE_HEADER, UpdateSnapshotViewAction.QUEUE)
+            .scheduleTime(
+                Timestamps.fromMillis(
+                    action.clock.nowUtc().plus(BigqueryPollJobAction.POLL_COUNTDOWN).getMillis())));
+
+    // assert the chained task of each enqueud task is correct
+    assertThat(
+            cloudTasksHelper.getTestTasksFor(BigqueryPollJobAction.QUEUE).stream()
+                .map(
+                    testTask -> {
+                      try {
+                        return new ObjectInputStream(
+                                new ByteArrayInputStream(
+                                    testTask.getAppEngineHttpRequest().getBody().toByteArray()))
+                            .readObject();
+                      } catch (ClassNotFoundException | IOException e) {
+                        return null;
+                      }
+                    }))
+        .containsExactly(
+            cloudTasksHelper
+                .getTestCloudTasksUtils()
+                .createPostTask(
+                    UpdateSnapshotViewAction.PATH,
+                    "BACKEND",
+                    ImmutableMultimap.of(
+                        UPDATE_SNAPSHOT_DATASET_ID_PARAM,
+                        "datastore_backups",
+                        UPDATE_SNAPSHOT_TABLE_ID_PARAM,
+                        "2018_12_05T17_46_39_92612_one",
+                        UPDATE_SNAPSHOT_KIND_PARAM,
+                        "one",
+                        UPDATE_SNAPSHOT_VIEWNAME_PARAM,
+                        "latest_datastore_export")),
+            cloudTasksHelper
+                .getTestCloudTasksUtils()
+                .createPostTask(
+                    UpdateSnapshotViewAction.PATH,
+                    "BACKEND",
+                    ImmutableMultimap.of(
+                        UPDATE_SNAPSHOT_DATASET_ID_PARAM,
+                        "datastore_backups",
+                        UPDATE_SNAPSHOT_TABLE_ID_PARAM,
+                        "2018_12_05T17_46_39_92612_two",
+                        UPDATE_SNAPSHOT_KIND_PARAM,
+                        "two",
+                        UPDATE_SNAPSHOT_VIEWNAME_PARAM,
+                        "latest_datastore_export")),
+            cloudTasksHelper
+                .getTestCloudTasksUtils()
+                .createPostTask(
+                    UpdateSnapshotViewAction.PATH,
+                    "BACKEND",
+                    ImmutableMultimap.of(
+                        UPDATE_SNAPSHOT_DATASET_ID_PARAM,
+                        "datastore_backups",
+                        UPDATE_SNAPSHOT_TABLE_ID_PARAM,
+                        "2018_12_05T17_46_39_92612_three",
+                        UPDATE_SNAPSHOT_KIND_PARAM,
+                        "three",
+                        UPDATE_SNAPSHOT_VIEWNAME_PARAM,
+                        "latest_datastore_export")))
+        .inOrder();
   }
 
   @Test

core/src/test/java/google/registry/flows/EppPointInTimeTest.java

@@ -72,7 +72,7 @@ class EppPointInTimeTest {
     SessionMetadata sessionMetadata = new HttpSessionMetadata(new FakeHttpSession());
     sessionMetadata.setRegistrarId("TheRegistrar");
     DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock)))
+        .fakesAndMocksModule(FakesAndMocksModule.create(clock))
         .build()
         .startRequest()
         .flowComponentBuilder()

core/src/test/java/google/registry/flows/EppTestCase.java

@@ -224,12 +224,14 @@ public class EppTestCase {
     EppRequestHandler handler = new EppRequestHandler();
     FakeResponse response = new FakeResponse();
     handler.response = response;
-    eppMetricBuilder = EppMetric.builderForRequest(clock);
-    handler.eppController = DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, eppMetricBuilder))
-        .build()
-        .startRequest()
-        .eppController();
+    FakesAndMocksModule fakesAndMocksModule = FakesAndMocksModule.create(clock);
+    eppMetricBuilder = fakesAndMocksModule.getMetricBuilder();
+    handler.eppController =
+        DaggerEppTestComponent.builder()
+            .fakesAndMocksModule(fakesAndMocksModule)
+            .build()
+            .startRequest()
+            .eppController();
     handler.executeEpp(
         sessionMetadata,
         credentials,

core/src/test/java/google/registry/flows/EppTestComponent.java

@@ -15,8 +15,6 @@
 package google.registry.flows;
 
 import static org.joda.time.Duration.standardSeconds;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import dagger.Component;
 import dagger.Module;
@@ -33,12 +31,12 @@ import google.registry.flows.domain.DomainFlowTmchUtils;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.request.RequestScope;
 import google.registry.request.lock.LockHandler;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeLockHandler;
 import google.registry.testing.FakeSleeper;
 import google.registry.tmch.TmchCertificateAuthority;
 import google.registry.tmch.TmchXmlSignature;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.Clock;
 import google.registry.util.Sleeper;
 import javax.inject.Singleton;
@@ -60,35 +58,32 @@ public interface EppTestComponent {
     private EppMetric.Builder metricBuilder;
     private FakeClock clock;
     private FakeLockHandler lockHandler;
-    private AppEngineServiceUtils appEngineServiceUtils;
     private Sleeper sleeper;
+    private CloudTasksHelper cloudTasksHelper;
 
-    public static FakesAndMocksModule create() {
-      FakeClock clock = new FakeClock();
-      return create(clock, EppMetric.builderForRequest(clock));
+    public CloudTasksHelper getCloudTasksHelper() {
+      return cloudTasksHelper;
     }
 
-    public static FakesAndMocksModule create(FakeClock clock, EppMetric.Builder metricBuilder) {
-      return create(
-          clock,
-          metricBuilder,
-          new TmchXmlSignature(new TmchCertificateAuthority(TmchCaMode.PILOT, clock)));
+    public EppMetric.Builder getMetricBuilder() {
+      return metricBuilder;
     }
 
-    public static FakesAndMocksModule create(
-        FakeClock clock, EppMetric.Builder eppMetricBuilder, TmchXmlSignature tmchXmlSignature) {
+    public static FakesAndMocksModule create(FakeClock clock) {
       FakesAndMocksModule instance = new FakesAndMocksModule();
-      AppEngineServiceUtils appEngineServiceUtils = mock(AppEngineServiceUtils.class);
-      when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
+      CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
       instance.asyncTaskEnqueuer =
-          AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, standardSeconds(90));
+          AsyncTaskEnqueuerTest.createForTesting(
+              cloudTasksHelper.getTestCloudTasksUtils(), clock, standardSeconds(90));
       instance.clock = clock;
-      instance.domainFlowTmchUtils = new DomainFlowTmchUtils(tmchXmlSignature);
-      instance.sleeper = new FakeSleeper(clock);
+      instance.domainFlowTmchUtils =
+          new DomainFlowTmchUtils(
+              new TmchXmlSignature(new TmchCertificateAuthority(TmchCaMode.PILOT, clock)));
+      instance.sleeper = new FakeSleeper(instance.clock);
       instance.dnsQueue = DnsQueue.create();
-      instance.metricBuilder = eppMetricBuilder;
-      instance.appEngineServiceUtils = appEngineServiceUtils;
+      instance.metricBuilder = EppMetric.builderForRequest(clock);
       instance.lockHandler = new FakeLockHandler(true);
+      instance.cloudTasksHelper = cloudTasksHelper;
       return instance;
     }
 
@@ -127,11 +122,6 @@ public interface EppTestComponent {
       return metricBuilder;
     }
 
-    @Provides
-    AppEngineServiceUtils provideAppEngineServiceUtils() {
-      return appEngineServiceUtils;
-    }
-
     @Provides
     Sleeper provideSleeper() {
       return sleeper;

core/src/test/java/google/registry/flows/FlowTestCase.java

@@ -33,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.collect.ObjectArrays;
-import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
 import google.registry.flows.EppTestComponent.FakesAndMocksModule;
 import google.registry.flows.picker.FlowPicker;
 import google.registry.model.billing.BillingEvent;
@@ -45,14 +44,13 @@ import google.registry.model.ofy.Ofy;
 import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.EppLoader;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeHttpSession;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TestDataHelper;
-import google.registry.tmch.TmchCertificateAuthority;
-import google.registry.tmch.TmchXmlSignature;
 import google.registry.util.TypeUtils.TypeInstantiator;
 import google.registry.xml.ValidationMode;
 import java.util.Arrays;
@@ -88,7 +86,7 @@ public abstract class FlowTestCase<F extends Flow> {
   protected FakeClock clock = new FakeClock(DateTime.now(UTC));
   protected TransportCredentials credentials = new PasswordOnlyTransportCredentials();
   protected EppRequestSource eppRequestSource = EppRequestSource.UNIT_TEST;
-  private TmchXmlSignature testTmchXmlSignature = null;
+  protected CloudTasksHelper cloudTasksHelper;
 
   private EppMetric.Builder eppMetricBuilder;
 
@@ -229,13 +227,12 @@ public abstract class FlowTestCase<F extends Flow> {
     // Assert that the xml triggers the flow we expect.
     assertThat(FlowPicker.getFlowClass(eppLoader.getEpp()))
         .isEqualTo(new TypeInstantiator<F>(getClass()){}.getExactType());
+
+    FakesAndMocksModule fakesAndMocksModule = FakesAndMocksModule.create(clock);
+    cloudTasksHelper = fakesAndMocksModule.getCloudTasksHelper();
     // Run the flow.
-    TmchXmlSignature tmchXmlSignature =
-        testTmchXmlSignature != null
-            ? testTmchXmlSignature
-            : new TmchXmlSignature(new TmchCertificateAuthority(tmchCaMode, clock));
     return DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, eppMetricBuilder, tmchXmlSignature))
+        .fakesAndMocksModule(fakesAndMocksModule)
         .build()
         .startRequest()
         .flowComponentBuilder()
@@ -300,8 +297,6 @@ public abstract class FlowTestCase<F extends Flow> {
     return output;
   }
 
-  private TmchCaMode tmchCaMode = TmchCaMode.PILOT;
-
   public EppOutput dryRunFlowAssertResponse(String xml, String... ignoredPaths) throws Exception {
     List<Object> beforeEntities = DatabaseHelper.loadAllEntities();
     EppOutput output =

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -856,7 +856,8 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     assertAboutDomains()
         .that(domain)
         .hasExactlyDsData(
-            DelegationSignerData.create(12345, 3, 1, base16().decode("49FD46E6C4B45C55D4AC"))
+            DelegationSignerData.create(
+                    12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))
                 .cloneWithDomainRepoId(domain.getRepoId()));
   }
 

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -54,14 +54,13 @@ import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
 import static google.registry.testing.TaskQueueHelper.assertDnsTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.money.CurrencyUnit.USD;
 import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -102,10 +101,10 @@ import google.registry.model.tld.Registry.TldType;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import java.util.Map;
@@ -314,17 +313,17 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
     clock.advanceOneMilli();
     runFlowAssertResponse(loadFile("domain_delete_response_pending.xml"));
     Duration when = standardDays(3);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .param(PARAM_RESAVE_TIMES, clock.nowUtc().plusDays(5).toString())
-            .etaDelta(when.minus(standardSeconds(30)), when.plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(when)));
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -43,12 +43,11 @@ import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
 import static google.registry.testing.HostResourceSubject.assertAboutHosts;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.money.CurrencyUnit.USD;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -106,10 +105,10 @@ import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import java.util.Map;
@@ -514,18 +513,16 @@ class DomainTransferRequestFlowTest
     assertPollMessagesEmitted(expectedExpirationTime, implicitTransferTime);
     assertAboutDomainAfterAutomaticTransfer(
         expectedExpirationTime, implicitTransferTime, Period.create(1, Unit.YEARS));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
-            .etaDelta(
-                registry.getAutomaticTransferLength().minus(standardSeconds(30)),
-                registry.getAutomaticTransferLength().plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(registry.getAutomaticTransferLength())));
   }
 
   private void doSuccessfulTest(

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -123,13 +123,17 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, DomainBase> {
 
   private static final DelegationSignerData SOME_DSDATA =
-      DelegationSignerData.create(1, 2, 2, base16().decode("0123"));
+      DelegationSignerData.create(
+          1,
+          2,
+          2,
+          base16().decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   private static final ImmutableMap<String, String> OTHER_DSDATA_TEMPLATE_MAP =
       ImmutableMap.of(
           "KEY_TAG", "12346",
           "ALG", "3",
           "DIGEST_TYPE", "1",
-          "DIGEST", "38EC35D5B3A34B44C39B");
+          "DIGEST", "A94A8FE5CCB19BA61C4C0873D391E987982FBBD3");
 
   private ContactResource sh8013Contact;
   private ContactResource mak21Contact;
@@ -523,9 +527,17 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         null,
         ImmutableSet.of(
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableMap.of(
-            "KEY_TAG", "12346", "ALG", "3", "DIGEST_TYPE", "1", "DIGEST", "38EC35D5B3A34B44C39B"));
+            "KEY_TAG",
+            "12346",
+            "ALG",
+            "3",
+            "DIGEST_TYPE",
+            "1",
+            "DIGEST",
+            "A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"));
   }
 
   @TestOfyAndSql
@@ -535,9 +547,17 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableMap.of(
-            "KEY_TAG", "12346", "ALG", "3", "DIGEST_TYPE", "1", "DIGEST", "38EC35D5B3A34B44C39B"));
+            "KEY_TAG",
+            "12346",
+            "ALG",
+            "3",
+            "DIGEST_TYPE",
+            "1",
+            "DIGEST",
+            "A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"));
   }
 
   @TestOfyAndSql
@@ -546,7 +566,15 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "0123"));
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
@@ -555,8 +583,23 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(
-            SOME_DSDATA, DelegationSignerData.create(1, 8, 4, base16().decode("4567"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "8", "DIGEST_TYPE", "4", "DIGEST", "4567"));
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                8,
+                4,
+                base16()
+                    .decode(
+                        "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "8",
+            "DIGEST_TYPE",
+            "4",
+            "DIGEST",
+            "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"));
   }
 
   // Changing any of the four fields in DelegationSignerData should result in a new object
@@ -566,8 +609,22 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(
-            SOME_DSDATA, DelegationSignerData.create(12346, 2, 2, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "12346", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "0123"));
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                12346,
+                2,
+                2,
+                base16()
+                    .decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "12346",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
@@ -575,8 +632,23 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 8, 2, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "8", "DIGEST_TYPE", "2", "DIGEST", "0123"));
+        ImmutableSet.of(
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                8,
+                2,
+                base16()
+                    .decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "8",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
@@ -584,8 +656,24 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 2, 4, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "4", "DIGEST", "0123"));
+        ImmutableSet.of(
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                2,
+                4,
+                base16()
+                    .decode(
+                        "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "4",
+            "DIGEST",
+            "768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9"));
   }
 
   @TestOfyAndSql
@@ -593,15 +681,35 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 2, 2, base16().decode("4567"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "4567"));
+        ImmutableSet.of(
+            SOME_DSDATA,
+            DelegationSignerData.create(
+                1,
+                2,
+                2,
+                base16()
+                    .decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"))),
+        ImmutableMap.of(
+            "KEY_TAG",
+            "1",
+            "ALG",
+            "2",
+            "DIGEST_TYPE",
+            "2",
+            "DIGEST",
+            "9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08"));
   }
 
   @TestOfyAndSql
   void testSuccess_secDnsAddToMaxRecords() throws Exception {
     ImmutableSet.Builder<DelegationSignerData> builder = new ImmutableSet.Builder<>();
     for (int i = 0; i < 7; ++i) {
-      builder.add(DelegationSignerData.create(i, 2, 2, new byte[] {0, 1, 2}));
+      builder.add(
+          DelegationSignerData.create(
+              i,
+              2,
+              2,
+              base16().decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08")));
     }
     ImmutableSet<DelegationSignerData> commonDsData = builder.build();
 
@@ -613,7 +721,10 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
                 commonDsData,
                 ImmutableSet.of(
                     DelegationSignerData.create(
-                        12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))))));
+                        12346,
+                        3,
+                        1,
+                        base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))))));
   }
 
   @TestOfyAndSql
@@ -622,7 +733,8 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_rem.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of(SOME_DSDATA));
   }
 
@@ -633,7 +745,8 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_rem_all.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))),
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of());
   }
 
@@ -643,17 +756,24 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add_rem.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))),
+            DelegationSignerData.create(
+                12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))));
+            DelegationSignerData.create(
+                12346, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))));
   }
 
   @TestOfyAndSql
   void testSuccess_secDnsAddRemoveToMaxRecords() throws Exception {
     ImmutableSet.Builder<DelegationSignerData> builder = new ImmutableSet.Builder<>();
     for (int i = 0; i < 7; ++i) {
-      builder.add(DelegationSignerData.create(i, 2, 2, new byte[] {0, 1, 2}));
+      builder.add(
+          DelegationSignerData.create(
+              i,
+              2,
+              2,
+              base16().decode("9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08")));
     }
     ImmutableSet<DelegationSignerData> commonDsData = builder.build();
 
@@ -664,13 +784,19 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
                 commonDsData,
                 ImmutableSet.of(
                     DelegationSignerData.create(
-                        12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))))),
+                        12345,
+                        3,
+                        1,
+                        base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))))),
         ImmutableSet.copyOf(
             union(
                 commonDsData,
                 ImmutableSet.of(
                     DelegationSignerData.create(
-                        12346, 3, 1, base16().decode("38EC35D5B3A34B44C39B"))))));
+                        12346,
+                        3,
+                        1,
+                        base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))))));
   }
 
   @TestOfyAndSql
@@ -680,10 +806,12 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add_rem_same.xml",
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))),
+            DelegationSignerData.create(
+                12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))),
         ImmutableSet.of(
             SOME_DSDATA,
-            DelegationSignerData.create(12345, 3, 1, base16().decode("38EC35D5B3A34B33C99B"))));
+            DelegationSignerData.create(
+                12345, 3, 1, base16().decode("A94A8FE5CCB19BA61C4C0873D391E987982FBBD3"))));
   }
 
   @TestOfyAndSql
@@ -852,6 +980,41 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
+  @TestOfyAndSql
+  void testFailure_secDnsInvalidDigestLength() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(ImmutableSet.of(DelegationSignerData.create(1, 2, 1, new byte[] {0, 1, 2})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains("Domain contains DS record(s) with an invalid digest length");
+  }
+
+  @TestOfyAndSql
+  void testFailure_secDnsMultipleInvalidDigestLengths() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(
+                ImmutableSet.of(
+                    DelegationSignerData.create(1, 2, 1, new byte[] {0, 1, 2, 3, 4}),
+                    DelegationSignerData.create(2, 2, 2, new byte[] {5, 6, 7})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertThat(thrown).hasMessageThat().contains("0, 1, 2, 3, 4");
+    assertThat(thrown).hasMessageThat().contains("5, 6, 7");
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains("Domain contains DS record(s) with an invalid digest length");
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
   @TestOfyAndSql
   void testFailure_secDnsInvalidAlgorithm() throws Exception {
     setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);

core/src/test/java/google/registry/model/common/ClassPathManagerTest.java

@@ -41,6 +41,7 @@ import google.registry.model.rde.RdeRevision;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.replay.LastSqlTransaction;
+import google.registry.model.replay.ReplayGap;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.server.Lock;
 import google.registry.model.server.ServerSecret;
@@ -74,6 +75,7 @@ public class ClassPathManagerTest {
     assertThat(ClassPathManager.getClass("HostResource")).isEqualTo(HostResource.class);
     assertThat(ClassPathManager.getClass("Recurring")).isEqualTo(Recurring.class);
     assertThat(ClassPathManager.getClass("Registrar")).isEqualTo(Registrar.class);
+    assertThat(ClassPathManager.getClass("ReplayGap")).isEqualTo(ReplayGap.class);
     assertThat(ClassPathManager.getClass("ContactResource")).isEqualTo(ContactResource.class);
     assertThat(ClassPathManager.getClass("Cancellation")).isEqualTo(Cancellation.class);
     assertThat(ClassPathManager.getClass("RegistrarContact")).isEqualTo(RegistrarContact.class);
@@ -141,6 +143,7 @@ public class ClassPathManagerTest {
     assertThat(ClassPathManager.getClassName(HostResource.class)).isEqualTo("HostResource");
     assertThat(ClassPathManager.getClassName(Recurring.class)).isEqualTo("Recurring");
     assertThat(ClassPathManager.getClassName(Registrar.class)).isEqualTo("Registrar");
+    assertThat(ClassPathManager.getClassName(ReplayGap.class)).isEqualTo("ReplayGap");
     assertThat(ClassPathManager.getClassName(ContactResource.class)).isEqualTo("ContactResource");
     assertThat(ClassPathManager.getClassName(Cancellation.class)).isEqualTo("Cancellation");
     assertThat(ClassPathManager.getClassName(RegistrarContact.class)).isEqualTo("RegistrarContact");

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -21,18 +21,19 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.testing.DatabaseHelper.insertInDb;
 import static google.registry.testing.LogsSubject.assertAboutLogs;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.junit.Assert.assertThrows;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.testing.TestLogHandler;
 import com.google.common.truth.Truth8;
+import com.googlecode.objectify.Key;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
 import google.registry.model.domain.token.AllocationToken;
@@ -40,6 +41,8 @@ import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.server.Lock;
+import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManagerImpl;
 import google.registry.persistence.transaction.TransactionEntity;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DatabaseHelper;
@@ -49,9 +52,13 @@ import google.registry.testing.InjectExtension;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestObject;
 import google.registry.util.RequestStatusChecker;
+import java.lang.reflect.Proxy;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityManagerFactory;
+import javax.persistence.EntityTransaction;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.AfterEach;
@@ -63,6 +70,7 @@ import org.junitpioneer.jupiter.RetryingTest;
 public class ReplicateToDatastoreActionTest {
 
   private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ"));
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   @RegisterExtension
   public final AppEngineExtension appEngine =
@@ -203,41 +211,135 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @RetryingTest(4)
-  void testMissingTransactions() {
-    assumeTrue(ReplayExtension.replayTestsEnabled());
-
-    // Write a transaction (should have a transaction id of 1).
+  void testNoTransactionIdUpdate() {
+    // Create an object.
     TestObject foo = TestObject.create("foo");
     insertInDb(foo);
 
-    // Force the last transaction id back to -1 so that we look for transaction 0.
-    ofyTm().transact(() -> ofyTm().insert(new LastSqlTransaction(-1)));
+    // Fail two transactions.
+    for (int i = 0; i < 2; ++i) {
+      try {
+        jpaTm()
+            .transact(
+                () -> {
+                  jpaTm().delete(foo.key());
+                  // Explicitly save the transaction entity to force the id update.
+                  jpaTm().insert(new TransactionEntity(new byte[] {1, 2, 3}));
+                  throw new RuntimeException("fail!!!");
+                });
+      } catch (Exception e) {
+        logger.atInfo().log("Got expected exception.");
+      }
+    }
 
+    TestObject bar = TestObject.create("bar");
+    insertInDb(bar);
+
+    // Make sure we have only the expected transaction ids.
     List<TransactionEntity> txns = action.getTransactionBatchAtSnapshot();
-    assertThat(txns).hasSize(1);
-    assertThat(assertThrows(IllegalStateException.class, () -> applyTransaction(txns.get(0))))
-        .hasMessageThat()
-        .isEqualTo("Missing transaction: last txn id = -1, next available txn = 1");
+    assertThat(txns).hasSize(2);
+    for (TransactionEntity txn : txns) {
+      assertThat(txn.getId()).isNotEqualTo(2);
+      assertThat(txn.getId()).isNotEqualTo(3);
+      applyTransaction(txn);
+    }
+
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(foo.key()))).isEqualTo(foo);
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(bar.key()))).isEqualTo(bar);
+    assertThat(ofyTm().transact(() -> LastSqlTransaction.load()).getTransactionId()).isEqualTo(4);
   }
 
   @Test
-  void testMissingTransactions_fullTask() {
-    assumeTrue(ReplayExtension.replayTestsEnabled());
+  void testTransactionGapReplay() {
+    insertInDb(TestObject.create("foo"));
+    DeferredCommit deferred = new DeferredCommit(fakeClock);
+    TestObject bar = TestObject.create("bar");
+    deferred.transact(() -> jpaTm().insert(bar));
+    TestObject baz = TestObject.create("baz");
+    insertInDb(baz);
 
-    // Write a transaction (should have a transaction id of 1).
-    TestObject foo = TestObject.create("foo");
-    insertInDb(foo);
-
-    // Force the last transaction id back to -1 so that we look for transaction 0.
-    ofyTm().transact(() -> ofyTm().insert(new LastSqlTransaction(-1)));
     action.run();
-    assertAboutLogs()
-        .that(logHandler)
-        .hasSevereLogWithCause(
-            new IllegalStateException(
-                "Missing transaction: last txn id = -1, next available txn = 1"));
-    assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
-    assertThat(response.getPayload()).isEqualTo("Errored out replaying files.");
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(bar.key())).isPresent()).isFalse();
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(baz.key()))).isEqualTo(baz);
+    VKey<ReplayGap> gapKey = VKey.createOfy(ReplayGap.class, Key.create(ReplayGap.class, 2));
+    Truth8.assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(gapKey))).isPresent();
+
+    deferred.commit();
+    resetAction();
+    action.run();
+
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(bar.key()))).isEqualTo(bar);
+    // Verify that the gap record has been cleaned up.
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(gapKey).isPresent())).isFalse();
+  }
+
+  /** Verify that we can handle creation and deletion of > 25 gap records. */
+  @Test
+  void testLargeNumberOfGaps() {
+    // Fail thirty transactions.
+    for (int i = 0; i < 30; ++i) {
+      try {
+        jpaTm()
+            .transact(
+                () -> {
+                  insertInDb(TestObject.create("foo"));
+                  // Explicitly save the transaction entity to force the id update.
+                  jpaTm().insert(new TransactionEntity(new byte[] {1, 2, 3}));
+                  throw new RuntimeException("fail!!!");
+                });
+      } catch (Exception e) {
+        ;
+      }
+    }
+
+    TestObject bar = TestObject.create("bar");
+    insertInDb(bar);
+
+    // Verify that the transaction was successfully applied and that we have generated 30 gap
+    // records.
+    action.run();
+    Truth8.assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(bar.key()))).isPresent();
+    assertThat(ofyTm().loadAllOf(ReplayGap.class).size()).isEqualTo(30);
+
+    // Verify that we can clean up this many gap records after expiration.
+    fakeClock.advanceBy(Duration.millis(ReplicateToDatastoreAction.MAX_GAP_RETENTION_MILLIS + 1));
+    resetAction();
+    action.run();
+    assertThat(ofyTm().loadAllOf(ReplayGap.class).size()).isEqualTo(0);
+  }
+
+  @Test
+  void testGapRecordExpiration() {
+    insertInDb(TestObject.create("foo"));
+
+    // Fail a transaction, create a gap.
+    try {
+      jpaTm()
+          .transact(
+              () -> {
+                jpaTm().insert(TestObject.create("other"));
+                // Explicitly save the transaction entity to force the id update.
+                jpaTm().insert(new TransactionEntity(new byte[] {1, 2, 3}));
+                throw new RuntimeException("fail!!!");
+              });
+    } catch (Exception e) {
+      logger.atInfo().log("Got expected exception.");
+    }
+
+    insertInDb(TestObject.create("bar"));
+
+    action.run();
+
+    // Verify that the gap record has been created
+    VKey<ReplayGap> gapKey = VKey.createOfy(ReplayGap.class, Key.create(ReplayGap.class, 2));
+    Truth8.assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(gapKey))).isPresent();
+
+    fakeClock.advanceBy(Duration.millis(ReplicateToDatastoreAction.MAX_GAP_RETENTION_MILLIS + 1));
+    resetAction();
+    action.run();
+
+    // Verify that the gap record has been destroyed.
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(gapKey)).isPresent()).isFalse();
   }
 
   @Test
@@ -326,4 +428,63 @@ public class ReplicateToDatastoreActionTest {
     when(requestStatusChecker.getLogId()).thenReturn("logId");
     action = new ReplicateToDatastoreAction(fakeClock, requestStatusChecker, response);
   }
+
+  /**
+   * Deep fake of EntityManagerFactory -> EntityManager -> EntityTransaction that allows us to defer
+   * the actual commit until after the other transactions are replayed.
+   */
+  static class DeferredCommit {
+
+    FakeClock clock;
+    EntityTransaction realTransaction;
+
+    DeferredCommit(FakeClock clock) {
+      this.clock = clock;
+    }
+
+    private static <T> T makeProxy(
+        Class<T> iface, T delegate, String method, Supplier<?> supplier) {
+      return (T)
+          Proxy.newProxyInstance(
+              delegate.getClass().getClassLoader(),
+              new Class[] {iface},
+              (proxy, meth, args) -> {
+                if (meth.getName().equals(method)) {
+                  return supplier.get();
+                } else {
+                  return meth.invoke(delegate, args);
+                }
+              });
+    }
+
+    EntityManager createEntityManagerProxy(EntityManager orgEntityManager) {
+      return makeProxy(
+          EntityManager.class,
+          orgEntityManager,
+          "getTransaction",
+          () ->
+              makeProxy(
+                  EntityTransaction.class,
+                  realTransaction = orgEntityManager.getTransaction(),
+                  "commit",
+                  () -> null));
+    }
+
+    void commit() {
+      realTransaction.commit();
+    }
+
+    void transact(Runnable runnable) {
+      EntityManagerFactory orgEmf =
+          jpaTm().transact(() -> jpaTm().getEntityManager().getEntityManagerFactory());
+      EntityManagerFactory emfProxy =
+          makeProxy(
+              EntityManagerFactory.class,
+              orgEmf,
+              "createEntityManager",
+              () -> createEntityManagerProxy(orgEmf.createEntityManager()));
+
+      new JpaTransactionManagerImpl(emfProxy, clock).transact(runnable);
+    }
+  }
 }

core/src/test/java/google/registry/rde/BrdaCopyActionTest.java

@@ -16,9 +16,13 @@ package google.registry.rde;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
+import static google.registry.model.common.Cursor.CursorType.BRDA;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SystemInfo.hasCommand;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 import com.google.cloud.storage.BlobId;
@@ -28,8 +32,11 @@ import com.google.common.io.CharStreams;
 import com.google.common.io.Files;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.Keyring;
+import google.registry.model.common.Cursor;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeRevision;
+import google.registry.model.tld.Registry;
+import google.registry.request.HttpException.NoContentException;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.FakeKeyringModule;
@@ -104,6 +111,7 @@ public class BrdaCopyActionTest {
 
   @BeforeEach
   void beforeEach() throws Exception {
+    createTld("lol");
     action.gcsUtils = gcsUtils;
     action.tld = "lol";
     action.watermark = DateTime.parse("2010-10-17TZ");
@@ -116,6 +124,20 @@ public class BrdaCopyActionTest {
             () -> {
               RdeRevision.saveRevision("lol", DateTime.parse("2010-10-17TZ"), RdeMode.THIN, 0);
             });
+    persistResource(Cursor.create(BRDA, action.watermark.plusDays(1), Registry.get("lol")));
+  }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"", "job-name/"})
+  void testRun_stagingNotFinished_throws204(String prefix) throws Exception {
+    persistResource(Cursor.create(BRDA, action.watermark, Registry.get("lol")));
+    NoContentException thrown = assertThrows(NoContentException.class, () -> runAction(prefix));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Waiting on RdeStagingAction for TLD lol to copy BRDA deposit for"
+                + " 2010-10-17T00:00:00.000Z to GCS; last BRDA staging completion was before"
+                + " 2010-10-17T00:00:00.000Z");
   }
 
   @ParameterizedTest

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -14,7 +14,6 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@@ -29,20 +28,13 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
-import com.google.common.base.Ascii;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSource;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.common.Cursor;
@@ -57,20 +49,21 @@ import google.registry.testing.FakeClock;
 import google.registry.testing.FakeKeyringModule;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.Retrier;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
-import java.util.Map;
 import java.util.Optional;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
 
 /** Unit tests for {@link RdeReportAction}. */
 @DualDatabaseTest
@@ -89,20 +82,21 @@ public class RdeReportActionTest {
 
   private final FakeResponse response = new FakeResponse();
   private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
-  private final URLFetchService urlFetchService = mock(URLFetchService.class);
-  private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
-  private final HTTPResponse httpResponse = mock(HTTPResponse.class);
   private final PGPPublicKey encryptKey =
       new FakeKeyringModule().get().getRdeStagingEncryptionKey();
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
   private final BlobId reportFile =
       BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
 
   private RdeReportAction createAction() {
     RdeReporter reporter = new RdeReporter();
     reporter.reportUrlPrefix = "https://rde-report.example";
-    reporter.urlFetchService = urlFetchService;
     reporter.password = "foo";
+    reporter.urlConnectionService = urlConnectionService;
     reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
     RdeReportAction action = new RdeReportAction();
     action.gcsUtils = gcsUtils;
@@ -127,6 +121,7 @@ public class RdeReportActionTest {
         Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
     gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
   }
 
   @TestOfyAndSql
@@ -142,24 +137,22 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
-    assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
+    verify(httpUrlConnection).setRequestMethod("PUT");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -167,9 +160,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_withPrefix() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     RdeReportAction action = createAction();
     action.prefix = Optional.of("job-name/");
     gcsUtils.delete(reportFile);
@@ -182,16 +174,14 @@ public class RdeReportActionTest {
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
-    assertThat(headers)
-        .containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
+    verify(httpUrlConnection).setRequestMethod("PUT");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -204,9 +194,8 @@ public class RdeReportActionTest {
     PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
     gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
   }
@@ -239,9 +228,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream());
     InternalServerErrorException thrown =
         assertThrows(
             InternalServerErrorException.class,
@@ -252,18 +240,17 @@ public class RdeReportActionTest {
   @TestOfyAndSql
   void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
     class ExpectedThrownException extends RuntimeException {}
-    when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
+    when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException());
     assertThrows(
         ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
   }
 
   @TestOfyAndSql
   void testRunWithLock_socketTimeout_doesRetry() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture()))
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
+    when(httpUrlConnection.getResponseCode())
         .thenThrow(new SocketTimeoutException())
-        .thenReturn(httpResponse);
+        .thenReturn(SC_OK);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@@ -274,14 +261,6 @@ public class RdeReportActionTest {
     return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime();
   }
 
-  private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
-    ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
-    for (HTTPHeader header : headers) {
-      builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
-    }
-    return builder.build();
-  }
-
   private static XjcRdeReportReport parseReport(byte[] data) {
     try {
       return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));

core/src/test/java/google/registry/rde/RdeUploadActionTest.java

@@ -403,7 +403,7 @@ public class RdeUploadActionTest {
         .hasMessageThat()
         .isEqualTo(
             "Waiting on RdeStagingAction for TLD tld to send 2010-10-17T00:00:00.000Z upload; last"
-                + " RDE staging completion was at 1970-01-01T00:00:00.000Z");
+                + " RDE staging completion was before 1970-01-01T00:00:00.000Z");
     cloudTasksHelper.assertNoTasksEnqueued("rde-upload");
     assertThat(folder.list()).isEmpty();
   }
@@ -420,7 +420,7 @@ public class RdeUploadActionTest {
         .hasMessageThat()
         .isEqualTo(
             "Waiting on RdeStagingAction for TLD tld to send 2010-10-17T00:00:00.000Z upload; "
-                + "last RDE staging completion was at 2010-10-17T00:00:00.000Z");
+                + "last RDE staging completion was before 2010-10-17T00:00:00.000Z");
   }
 
   @TestOfyAndSql
@@ -437,8 +437,9 @@ public class RdeUploadActionTest {
     assertThat(thrown)
         .hasMessageThat()
         .isEqualTo(
-            "Waiting on 120 minute SFTP cooldown for TLD tld to send 2010-10-17T00:00:00.000Z "
-                + "upload; last upload attempt was at 2010-10-16T22:23:00.000Z (97 minutes ago)");
+            "Waiting on 120 minute SFTP cooldown for TLD tld to send 2010-10-17T00:00:00.000Z"
+                + " upload; last upload attempt was at 2010-10-16T22:23:00.000Z (97 minutes"
+                + " ago)");
   }
 
   private String slurp(InputStream is) throws IOException {

core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.MediaType.CSV_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
+import static google.registry.request.UrlConnectionUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
@@ -27,22 +27,19 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import google.registry.testing.AppEngineExtension;
+import java.io.ByteArrayOutputStream;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
+import javax.net.ssl.HttpsURLConnection;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockito.ArgumentCaptor;
 
-/** Unit tests for {@link UrlFetchUtils}. */
-class UrlFetchUtilsTest {
-
-  @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().build();
+/** Tests for {@link UrlConnectionUtils}. */
+public class UrlConnectionUtilsTest {
 
   private final Random random = mock(Random.class);
 
@@ -58,25 +55,28 @@ class UrlFetchUtilsTest {
   }
 
   @Test
-  void testSetPayloadMultipart() {
-    HTTPRequest request = mock(HTTPRequest.class);
+  void testSetPayloadMultipart() throws Exception {
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
+    ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+    when(connection.getOutputStream()).thenReturn(connectionOutputStream);
     setPayloadMultipart(
-        request,
+        connection,
         "lol",
         "cat",
         CSV_UTF_8,
         "The nice people at the store say hello. ヘ(◕。◕ヘ)",
         random);
-    ArgumentCaptor<HTTPHeader> headerCaptor = ArgumentCaptor.forClass(HTTPHeader.class);
-    verify(request, times(2)).addHeader(headerCaptor.capture());
-    List<HTTPHeader> addedHeaders = headerCaptor.getAllValues();
-    assertThat(addedHeaders.get(0).getName()).isEqualTo(CONTENT_TYPE);
-    assertThat(addedHeaders.get(0).getValue())
-        .isEqualTo(
-            "multipart/form-data; "
-                + "boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"");
-    assertThat(addedHeaders.get(1).getName()).isEqualTo(CONTENT_LENGTH);
-    assertThat(addedHeaders.get(1).getValue()).isEqualTo("294");
+    ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
+    ArgumentCaptor<String> valueCaptor = ArgumentCaptor.forClass(String.class);
+    verify(connection, times(2)).setRequestProperty(keyCaptor.capture(), valueCaptor.capture());
+    List<String> addedKeys = keyCaptor.getAllValues();
+    assertThat(addedKeys).containsExactly(CONTENT_TYPE, CONTENT_LENGTH);
+    List<String> addedValues = valueCaptor.getAllValues();
+    assertThat(addedValues)
+        .containsExactly(
+            "multipart/form-data;"
+                + " boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"",
+            "294");
     String payload =
         "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n"
             + "Content-Disposition: form-data; name=\"lol\"; filename=\"cat\"\r\n"
@@ -84,18 +84,20 @@ class UrlFetchUtilsTest {
             + "\r\n"
             + "The nice people at the store say hello. ヘ(◕。◕ヘ)\r\n"
             + "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r\n";
-    verify(request).setPayload(payload.getBytes(UTF_8));
-    verifyNoMoreInteractions(request);
+    verify(connection).setDoOutput(true);
+    verify(connection).getOutputStream();
+    assertThat(connectionOutputStream.toByteArray()).isEqualTo(payload.getBytes(UTF_8));
+    verifyNoMoreInteractions(connection);
   }
 
   @Test
   void testSetPayloadMultipart_boundaryInPayload() {
-    HTTPRequest request = mock(HTTPRequest.class);
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
     String payload = "I screamed------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHHH";
     IllegalStateException thrown =
         assertThrows(
             IllegalStateException.class,
-            () -> setPayloadMultipart(request, "lol", "cat", CSV_UTF_8, payload, random));
+            () -> setPayloadMultipart(connection, "lol", "cat", CSV_UTF_8, payload, random));
     assertThat(thrown)
         .hasMessageThat()
         .contains(

core/src/test/java/google/registry/testing/FakeURLFetchService.java

@@ -1,46 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.collect.ImmutableList;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * A fake {@link URLFetchService} that serves constructed {@link HTTPResponse} objects from
- * a simple {@link Map} ({@link URL} to {@link HTTPResponse}) lookup.
- */
-public class FakeURLFetchService extends ForwardingURLFetchService {
-
-  private Map<URL, HTTPResponse> backingMap;
-
-  public FakeURLFetchService(Map<URL, HTTPResponse> backingMap) {
-    this.backingMap = backingMap;
-  }
-
-  @Override
-  public HTTPResponse fetch(HTTPRequest request) {
-    URL requestURL = request.getURL();
-    if (backingMap.containsKey(requestURL)) {
-      return backingMap.get(requestURL);
-    } else {
-      return new HTTPResponse(HttpURLConnection.HTTP_NOT_FOUND, null, null, ImmutableList.of());
-    }
-  }
-}

core/src/test/java/google/registry/testing/FakeUrlConnectionService.java

@@ -0,0 +1,46 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.testing;
+
+import static org.mockito.Mockito.when;
+
+import google.registry.request.UrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+/** A fake {@link UrlConnectionService} with a mocked HTTP connection for testing. */
+public class FakeUrlConnectionService implements UrlConnectionService {
+
+  private final HttpURLConnection mockConnection;
+  private final List<URL> connectedUrls;
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection) {
+    this(mockConnection, new ArrayList<>());
+  }
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection, List<URL> connectedUrls) {
+    this.mockConnection = mockConnection;
+    this.connectedUrls = connectedUrls;
+  }
+
+  @Override
+  public HttpURLConnection createConnection(URL url) {
+    connectedUrls.add(url);
+    when(mockConnection.getURL()).thenReturn(url);
+    return mockConnection;
+  }
+}

core/src/test/java/google/registry/testing/ForwardingURLFetchService.java

@@ -1,49 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.util.concurrent.Futures;
-import java.io.IOException;
-import java.net.URL;
-import java.util.concurrent.Future;
-
-/**
- * An implementation of the {@link URLFetchService} interface that forwards all requests through
- * a synchronous fetch call.
- */
-public abstract class ForwardingURLFetchService implements URLFetchService {
-
-  @Override
-  public HTTPResponse fetch(URL url) throws IOException {
-    return fetch(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(URL url) {
-    return fetchAsync(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(HTTPRequest request) {
-    try {
-      return Futures.immediateFuture(fetch(request));
-    } catch (Exception e) {
-      return Futures.immediateFailedFuture(e);
-    }
-  }
-}

core/src/test/java/google/registry/tmch/NordnUploadActionTest.java

@@ -19,21 +19,22 @@ import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.FORM_DATA;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistDomainAndEnqueueLordn;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.startsWith;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -43,10 +44,6 @@ import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.VerifyException;
 import com.google.common.collect.ImmutableList;
@@ -57,11 +54,15 @@ import google.registry.model.tld.Registry;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
@@ -69,17 +70,9 @@ import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.mockito.junit.jupiter.MockitoSettings;
-import org.mockito.quality.Strictness;
 
 /** Unit tests for {@link NordnUploadAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnUploadActionTest {
 
   private static final String CLAIMS_CSV =
@@ -101,29 +94,30 @@ class NordnUploadActionTest {
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
-
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
+  
   private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z"));
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnUploadAction action = new NordnUploadAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
     inject.setStaticField(Ofy.class, "clock", clock);
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
-    when(httpResponse.getContent()).thenReturn("Success".getBytes(US_ASCII));
-    when(httpResponse.getResponseCode()).thenReturn(SC_ACCEPTED);
-    when(httpResponse.getHeadersUncombined())
-        .thenReturn(ImmutableList.of(new HTTPHeader(LOCATION, "http://trololol")));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream("Success".getBytes(UTF_8)));
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_ACCEPTED);
+    when(httpUrlConnection.getHeaderField(LOCATION)).thenReturn("http://trololol");
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
     persistResource(loadRegistrar("TheRegistrar").asBuilder().setIanaIdentifier(99999L).build());
     createTld("tld");
     persistResource(Registry.get("tld").asBuilder().setLordnUsername("lolcat").build());
     action.clock = clock;
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.phase = "claims";
     action.taskQueueUtils = new TaskQueueUtils(new Retrier(new FakeSleeper(clock), 3));
@@ -133,7 +127,6 @@ class NordnUploadActionTest {
     action.retrier = new Retrier(new FakeSleeper(clock), 3);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv() {
     List<TaskHandle> tasks =
@@ -145,7 +138,6 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_dedupesDuplicates() {
     List<TaskHandle> tasks =
@@ -158,14 +150,12 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_doesntFailOnEmptyTasks() {
     assertThat(NordnUploadAction.convertTasksToCsv(ImmutableList.of(), clock.nowUtc(), "col1,col2"))
         .isEqualTo("1,2010-05-01T10:11:12.000Z,0\ncol1,col2\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_throwsNpeOnNullTasks() {
     assertThrows(
@@ -173,7 +163,6 @@ class NordnUploadActionTest {
         () -> NordnUploadAction.convertTasksToCsv(null, clock.nowUtc(), "header"));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @SuppressWarnings("unchecked")
   @Test
   void test_loadAllTasks_retryLogic_thirdTrysTheCharm() {
@@ -186,7 +175,6 @@ class NordnUploadActionTest {
     assertThat(action.loadAllTasks(queue, "tld")).containsExactly(task);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_loadAllTasks_retryLogic_allFailures() {
     Queue queue = mock(Queue.class);
@@ -201,47 +189,47 @@ class NordnUploadActionTest {
   void testRun_claimsMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
   }
 
   @Test
   void testRun_sunriseMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistSunriseModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
   }
 
   @Test
   void testRun_usesMultipartContentType() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), CONTENT_TYPE).get())
-        .startsWith("multipart/form-data; boundary=");
+    verify(httpUrlConnection)
+        .setRequestProperty(eq(CONTENT_TYPE), startsWith("multipart/form-data; boundary="));
+    verify(httpUrlConnection).setRequestMethod("POST");
   }
 
   @Test
-  void testRun_hasPassword_setsAuthorizationHeader() throws Exception {
+  void testRun_hasPassword_setsAuthorizationHeader() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+    verify(httpUrlConnection)
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testRun_noPassword_doesntSendAuthorizationHeader() throws Exception {
+  void testRun_noPassword_doesntSendAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
-  void testRun_claimsMode_payloadMatchesClaimsCsv() throws Exception {
+  void testRun_claimsMode_payloadMatchesClaimsCsv() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(CLAIMS_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(CLAIMS_CSV);
   }
 
   @Test
@@ -257,19 +245,19 @@ class NordnUploadActionTest {
   }
 
   @Test
-  void testRun_sunriseMode_payloadMatchesSunriseCsv() throws Exception {
+  void testRun_sunriseMode_payloadMatchesSunriseCsv() {
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
   void test_noResponseContent_stillWorksNormally() throws Exception {
     // Returning null only affects logging.
-    when(httpResponse.getContent()).thenReturn(null);
+    when(httpUrlConnection.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[] {}));
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
@@ -284,7 +272,6 @@ class NordnUploadActionTest {
             .header(CONTENT_TYPE, FORM_DATA.toString()));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void testFailure_nullRegistryUser() {
     persistClaimsModeDomain();
@@ -293,17 +280,11 @@ class NordnUploadActionTest {
     assertThat(thrown).hasMessageThat().contains("lordnUsername is not set for tld.");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
-  void testFetchFailure() {
+  void testFetchFailure() throws Exception {
     persistClaimsModeDomain();
-    when(httpResponse.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
-    assertThrows(UrlFetchException.class, action::run);
-  }
-
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
+    assertThrows(UrlConnectionException.class, action::run);
   }
 
   private void persistClaimsModeDomain() {

core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java

@@ -16,39 +16,34 @@ package google.registry.tmch;
 
 import static com.google.common.net.HttpHeaders.AUTHORIZATION;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.model.tld.Registry;
 import google.registry.request.HttpException.ConflictException;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeUrlConnectionService;
+import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Unit tests for {@link NordnVerifyAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnVerifyActionTest {
 
   private static final String LOG_ACCEPTED =
@@ -81,84 +76,83 @@ class NordnVerifyActionTest {
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
   private final FakeResponse response = new FakeResponse();
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnVerifyAction action = new NordnVerifyAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(LOG_ACCEPTED.getBytes(UTF_8));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
     createTld("gtld");
     persistResource(Registry.get("gtld").asBuilder().setLordnUsername("lolcat").build());
     action.tld = "gtld";
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ACCEPTED.getBytes(UTF_8)));
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.response = response;
     action.url = new URL("http://127.0.0.1/blobio");
   }
 
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
-  }
-
   @Test
   void testSuccess_sendHttpRequest_urlIsCorrect() throws Exception {
     action.run();
-    assertThat(getCapturedHttpRequest().getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
   }
 
   @Test
-  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() throws Exception {
+  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() {
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+    verify(httpUrlConnection)
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() throws Exception {
+  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
   void successVerifyRejected() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_REJECTED.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_REJECTED.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.getStatus()).isEqualTo(LordnLog.Status.REJECTED);
   }
 
   @Test
   void successVerifyWarnings() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_WARNINGS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_WARNINGS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
   void successVerifyErrors() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_ERRORS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ERRORS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
-  void failureVerifyUnauthorized() {
-    when(httpResponse.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
+  void failureVerifyUnauthorized() throws Exception {
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
     assertThrows(Exception.class, action::run);
   }
 
   @Test
-  void failureVerifyNotReady() {
-    when(httpResponse.getResponseCode()).thenReturn(SC_NO_CONTENT);
+  void failureVerifyNotReady() throws Exception {
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_NO_CONTENT);
     ConflictException thrown = assertThrows(ConflictException.class, action::run);
     assertThat(thrown).hasMessageThat().contains("Not ready");
   }

core/src/test/java/google/registry/tmch/TmchActionTestCase.java

@@ -15,21 +15,19 @@
 package google.registry.tmch;
 
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.FakeClock;
+import google.registry.testing.FakeUrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Common code for unit tests of classes that extend {@link Marksdb}. */
@@ -46,19 +44,19 @@ abstract class TmchActionTestCase {
   @RegisterExtension
   public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension();
 
-  @Mock URLFetchService fetchService;
-  @Mock HTTPResponse httpResponse;
-  @Captor ArgumentCaptor<HTTPRequest> httpRequest;
-
   final FakeClock clock = new FakeClock();
   final Marksdb marksdb = new Marksdb();
 
+  protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  protected final ArrayList<URL> connectedUrls = new ArrayList<>();
+  protected FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection, connectedUrls);
+
   @BeforeEach
   public void beforeEachTmchActionTestCase() throws Exception {
-    marksdb.fetchService = fetchService;
     marksdb.tmchMarksdbUrl = MARKSDB_URL;
     marksdb.marksdbPublicKey = TmchData.loadPublicKey(TmchTestData.loadBytes("pubkey"));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    marksdb.urlConnectionService = urlConnectionService;
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
   }
 }

core/src/test/java/google/registry/tmch/TmchCrlActionTest.java

@@ -22,6 +22,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
+import java.io.ByteArrayInputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.security.SignatureException;
@@ -44,19 +45,22 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testSuccess() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent()).thenReturn(
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     newTmchCrlAction(TmchCaMode.PRODUCTION).run();
-    verify(httpResponse).getContent();
-    verify(fetchService).fetch(httpRequest.capture());
-    assertThat(httpRequest.getValue().getURL().toString()).isEqualTo("http://sloth.lol/tmch.crl");
+    verify(httpUrlConnection).getInputStream();
+    assertThat(connectedUrls).containsExactly(new URL("http://sloth.lol/tmch.crl"));
   }
 
   @Test
   void testFailure_crlTooOld() throws Exception {
     clock.setTo(DateTime.parse("2020-01-01TZ"));
-    when(httpResponse.getContent())
-        .thenReturn(loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read()));
     TmchCrlAction action = newTmchCrlAction(TmchCaMode.PILOT);
     Exception e = assertThrows(Exception.class, action::run);
     assertThat(e).hasCauseThat().isInstanceOf(CRLException.class);
@@ -69,8 +73,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotSignedByRoot() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent())
-        .thenReturn(readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(SignatureException.class);
     assertThat(e).hasCauseThat().hasMessageThat().isEqualTo("Signature does not match.");
@@ -79,8 +85,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotYetValid() throws Exception {
     clock.setTo(DateTime.parse("1984-01-01TZ"));
-    when(httpResponse.getContent()).thenReturn(
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(CertificateNotYetValidException.class);
   }

core/src/test/java/google/registry/tmch/TmchDnlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static org.mockito.Mockito.times;
@@ -22,6 +23,8 @@ import static org.mockito.Mockito.when;
 
 import google.registry.model.tmch.ClaimsList;
 import google.registry.model.tmch.ClaimsListDao;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -39,15 +42,13 @@ class TmchDnlActionTest extends TmchActionTestCase {
   @Test
   void testDnl() throws Exception {
     assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
-    when(httpResponse.getContent())
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.csv").read()))
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.sig").read()));
     newTmchDnlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.csv");
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
+    verify(httpUrlConnection, times(2)).getInputStream();
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
+        .containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig");
 
     // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
     ClaimsList claimsList = ClaimsListDao.get();

core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.tmch.TmchTestData.loadBytes;
 import static org.mockito.Mockito.times;
@@ -21,6 +22,8 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.model.smd.SignedMarkRevocationList;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -42,15 +45,14 @@ class TmchSmdrlActionTest extends TmchActionTestCase {
     SignedMarkRevocationList smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isFalse();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();
-    when(httpResponse.getContent())
-        .thenReturn(loadBytes("smdrl-latest.csv").read())
-        .thenReturn(loadBytes("smdrl-latest.sig").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.csv").read()))
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.sig").read()));
     newTmchSmdrlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.csv");
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.sig");
+    verify(httpUrlConnection, times(2)).getInputStream();
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
+        .containsExactly(
+            MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig");
     smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isTrue();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();

core/src/test/java/google/registry/tools/CreateDomainCommandTest.java

@@ -50,7 +50,8 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
         "--admins=crr-admin",
         "--techs=crr-tech",
         "--password=2fooBAR",
-        "--ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--ds_records=60485 5  2  D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
         "example.tld");
     eppVerifier.verifySent("domain_create_complete.xml");
@@ -66,7 +67,8 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
         "--admins=crr-admin",
         "--techs=crr-tech",
         "--password=2fooBAR",
-        "--ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--ds_records=60485 5  2  D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
         "example.tld");
     eppVerifier.verifySent("domain_create_complete.xml");
@@ -330,6 +332,22 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized digest type: 3");
   }
 
+  @Test
+  void testFailure_invalidDigestLength() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar",
+                    "--registrant=crr-admin",
+                    "--admins=crr-admin",
+                    "--techs=crr-tech",
+                    "--ds_records=1 2 1 abcd",
+                    "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record has an invalid digest length: ABCD");
+  }
+
   @Test
   void testFailure_invalidAlgorithm() {
     IllegalArgumentException thrown =
@@ -341,7 +359,9 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
                     "--registrant=crr-admin",
                     "--admins=crr-admin",
                     "--techs=crr-tech",
-                    "--ds_records=1 999 4 abcd",
+                    "--ds_records=1 999 4"
+                        + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1C"
+                        + "CB126255D196047DFEDF17A0A9",
                     "example.tld"));
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized algorithm: 999");
   }

core/src/test/java/google/registry/tools/DomainLockUtilsTest.java

@@ -31,8 +31,6 @@ import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STAT
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardHours;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
@@ -55,7 +53,6 @@ import google.registry.testing.FakeClock;
 import google.registry.testing.SqlHelper;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.UserInfo;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.StringGenerator.Alphabets;
 import java.util.Optional;
 import java.util.Set;
@@ -96,8 +93,6 @@ public final class DomainLockUtilsTest {
     HostResource host = persistActiveHost("ns1.example.net");
     domain = persistResource(newDomainBase(DOMAIN_NAME, host));
 
-    AppEngineServiceUtils appEngineServiceUtils = mock(AppEngineServiceUtils.class);
-    when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
     domainLockUtils =
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),

core/src/test/java/google/registry/tools/GenerateEscrowDepositCommandTest.java

@@ -17,17 +17,14 @@ package google.registry.tools;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.when;
 
 import com.beust.jcommander.ParameterException;
 import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.InjectExtension;
-import google.registry.util.AppEngineServiceUtils;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
 
@@ -38,8 +35,6 @@ public class GenerateEscrowDepositCommandTest
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  @Mock AppEngineServiceUtils appEngineServiceUtils;
-
   CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
 
   @BeforeEach
@@ -47,10 +42,7 @@ public class GenerateEscrowDepositCommandTest
     createTld("tld");
     createTld("anothertld");
     command = new GenerateEscrowDepositCommand();
-    command.appEngineServiceUtils = appEngineServiceUtils;
     command.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
-    when(appEngineServiceUtils.getCurrentVersionHostname("backend"))
-        .thenReturn("backend.test.localhost");
   }
 
   @Test

core/src/test/java/google/registry/tools/UniformRapidSuspensionCommandTest.java

@@ -78,7 +78,7 @@ class UniformRapidSuspensionCommandTest
     runCommandForced(
         "--domain_name=evil.tld",
         "--hosts=urs1.example.com,urs2.example.com",
-        "--dsdata=1 1 1 abcd",
+        "--dsdata=1 1 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--renew_one_year=false");
     eppVerifier
         .expectRegistrarId("CharlestonRoad")
@@ -149,7 +149,7 @@ class UniformRapidSuspensionCommandTest
     runCommandForced(
         "--domain_name=evil.tld",
         "--hosts=urs1.example.com,urs2.example.com",
-        "--dsdata=1 1 1 abcd",
+        "--dsdata=1 1 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--renew_one_year=false");
     eppVerifier
         .expectRegistrarId("CharlestonRoad")

core/src/test/java/google/registry/tools/UpdateDomainCommandTest.java

@@ -86,12 +86,14 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--remove_nameservers=ns3.zdns.google,ns4.zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld");
@@ -106,12 +108,14 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--remove_nameservers=ns[3-4].zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld");
@@ -128,12 +132,14 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--remove_nameservers=ns[3-4].zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld",
@@ -186,7 +192,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--add_ds_records=1 2 2 D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A,4"
+            + " 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "example.tld");
     eppVerifier.verifySent("domain_update_add.xml");
   }
@@ -199,7 +206,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3,6 5 4"
+            + " 768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9",
         "example.tld");
     eppVerifier.verifySent("domain_update_remove.xml");
   }
@@ -277,7 +285,11 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
 
   @TestOfyAndSql
   void testSuccess_setDsRecords() throws Exception {
-    runCommandForced("--client=NewRegistrar", "--ds_records=1 2 2 abcd,4 5 1 EF01", "example.tld");
+    runCommandForced(
+        "--client=NewRegistrar",
+        "--ds_records=1 2 2 D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
+        "example.tld");
     eppVerifier.verifySent("domain_update_set_ds_records.xml");
   }
 
@@ -285,7 +297,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
   void testSuccess_setDsRecords_withUnneededClear() throws Exception {
     runCommandForced(
         "--client=NewRegistrar",
-        "--ds_records=1 2 2 abcd,4 5 1 EF01",
+        "--ds_records=1 2 2 D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A,4 5 1"
+            + " A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
         "--clear_ds_records",
         "example.tld");
     eppVerifier.verifySent("domain_update_set_ds_records.xml");
@@ -636,7 +649,10 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             IllegalArgumentException.class,
             () ->
                 runCommandForced(
-                    "--client=NewRegistrar", "--add_ds_records=1 299 2 abcd", "example.tld"));
+                    "--client=NewRegistrar",
+                    "--add_ds_records=1 299 2"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
+                    "example.tld"));
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized algorithm: 299");
   }
 
@@ -647,10 +663,29 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             IllegalArgumentException.class,
             () ->
                 runCommandForced(
-                    "--client=NewRegistrar", "--add_ds_records=1 2 3 abcd", "example.tld"));
+                    "--client=NewRegistrar",
+                    "--add_ds_records=1 2 3"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
+                    "example.tld"));
     assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized digest type: 3");
   }
 
+  @TestOfyAndSql
+  void testFailure_invalidDigestLength() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar",
+                    "--registrant=crr-admin",
+                    "--admins=crr-admin",
+                    "--techs=crr-tech",
+                    "--ds_records=1 2 1 abcd",
+                    "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record has an invalid digest length: ABCD");
+  }
+
   @TestOfyAndSql
   void testFailure_provideDsRecordsAndAddDsRecords() {
     IllegalArgumentException thrown =
@@ -659,8 +694,9 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--add_ds_records=1 2 2 abcd",
-                    "--ds_records=4 5 1 EF01",
+                    "--add_ds_records=1 2 2"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
+                    "--ds_records=4 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
                     "example.tld"));
     assertThat(thrown)
         .hasMessageThat()
@@ -677,8 +713,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--remove_ds_records=7 8 1 12ab",
-                    "--ds_records=4 5 1 EF01",
+                    "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
+                    "--ds_records=4 5 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
                     "example.tld"));
     assertThat(thrown)
         .hasMessageThat()
@@ -695,7 +731,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--add_ds_records=1 2 2 abcd",
+                    "--add_ds_records=1 2 2"
+                        + " D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
                     "--clear_ds_records",
                     "example.tld"));
     assertThat(thrown)
@@ -713,7 +750,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--remove_ds_records=7 8 1 12ab",
+                    "--remove_ds_records=7 8 1 A94A8FE5CCB19BA61C4C0873D391E987982FBBD3",
                     "--clear_ds_records",
                     "example.tld"));
     assertThat(thrown)

core/src/test/java/google/registry/ui/server/registrar/RegistrarSettingsActionTestCase.java

@@ -50,7 +50,6 @@ import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
 import google.registry.ui.server.SendEmailUtils;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import java.io.PrintWriter;
@@ -88,7 +87,6 @@ public abstract class RegistrarSettingsActionTestCase {
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  @Mock AppEngineServiceUtils appEngineServiceUtils;
   @Mock HttpServletRequest req;
   @Mock HttpServletResponse rsp;
   @Mock SendEmailService emailService;
@@ -112,8 +110,6 @@ public abstract class RegistrarSettingsActionTestCase {
         getOnlyElement(loadRegistrar(CLIENT_ID).getContactsOfType(RegistrarContact.Type.TECH));
 
     action.registrarAccessor = null;
-    action.appEngineServiceUtils = appEngineServiceUtils;
-    when(appEngineServiceUtils.getCurrentVersionHostname("backend")).thenReturn("backend.hostname");
     action.jsonActionRunner =
         new JsonActionRunner(ImmutableMap.of(), new JsonResponse(new ResponseImpl(rsp)));
     action.sendEmailUtils =

core/src/test/java/google/registry/webdriver/TestServerExtension.java

@@ -33,6 +33,7 @@ import google.registry.testing.UserInfo;
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
@@ -53,6 +54,8 @@ import org.junit.jupiter.api.extension.ExtensionContext;
  */
 public final class TestServerExtension implements BeforeEachCallback, AfterEachCallback {
 
+  private static final Duration SERVER_STATUS_POLLING_INTERVAL = Duration.ofSeconds(5);
+
   private final ImmutableList<Fixture> fixtures;
   private final AppEngineExtension appEngineExtension;
   private final BlockingQueue<FutureTask<?>> jobs = new LinkedBlockingDeque<>();
@@ -107,8 +110,11 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
     serverThread = new Thread(server);
     synchronized (this) {
       serverThread.start();
-      while (!server.isRunning) {
-        this.wait();
+      while (server.serverStatus.equals(ServerStatus.NOT_STARTED)) {
+        this.wait(SERVER_STATUS_POLLING_INTERVAL.toMillis());
+      }
+      if (server.serverStatus.equals(ServerStatus.FAILED)) {
+        throw new RuntimeException("TestServer failed to start. See log for details.");
       }
     }
   }
@@ -163,6 +169,12 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
     return job.get();
   }
 
+  enum ServerStatus {
+    NOT_STARTED,
+    RUNNING,
+    FAILED
+  }
+
   private final class Server implements Runnable {
 
     private ExtensionContext context;
@@ -171,7 +183,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
       this.context = context;
     }
 
-    private volatile boolean isRunning = false;
+    private volatile ServerStatus serverStatus = ServerStatus.NOT_STARTED;
 
     @Override
     public void run() {
@@ -185,6 +197,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
           appEngineExtension.afterEach(context);
         }
       } catch (Throwable e) {
+        serverStatus = ServerStatus.FAILED;
         throw new RuntimeException(e);
       }
     }
@@ -196,7 +209,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
       testServer.start();
       System.out.printf("TestServerExtension is listening on: %s\n", testServer.getUrl("/"));
       synchronized (TestServerExtension.this) {
-        isRunning = true;
+        serverStatus = ServerStatus.RUNNING;
         TestServerExtension.this.notify();
       }
       try {

core/src/test/resources/google/registry/flows/domain/domain_create_dsdata_8_records.xml

@@ -26,49 +26,49 @@
           <secDNS:keyTag>12345</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12346</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12347</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12348</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12349</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12350</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12351</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>12352</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
       </secDNS:create>
     </extension>

core/src/test/resources/google/registry/flows/domain/domain_create_dsdata_no_maxsiglife.xml

@@ -26,7 +26,7 @@
           <secDNS:keyTag>12345</secDNS:keyTag>
           <secDNS:alg>3</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
       </secDNS:create>
     </extension>

core/src/test/resources/google/registry/flows/domain/domain_update_dsdata_add_rem.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12345</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B33C99B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -24,7 +24,7 @@
             <secDNS:keyTag>12346</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B44C39B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/flows/domain/domain_update_dsdata_add_rem_same.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12345</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B33C99B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -24,7 +24,7 @@
             <secDNS:keyTag>12345</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B33C99B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/flows/domain/domain_update_dsdata_rem.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12346</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B44C39B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
       </secDNS:update>

core/src/test/resources/google/registry/flows/domain_update_dsdata_add.xml

@@ -16,7 +16,7 @@
             <secDNS:keyTag>12346</secDNS:keyTag>
             <secDNS:alg>3</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>38EC35D5B3A34B44C39B</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/model/schema.txt

@@ -641,6 +641,10 @@ class google.registry.model.replay.LastSqlTransaction {
   @Id long id;
   long transactionId;
 }
+class google.registry.model.replay.ReplayGap {
+  @Id long transactionId;
+  org.joda.time.DateTime timestamp;
+}
 class google.registry.model.reporting.DomainTransactionRecord {
   google.registry.model.reporting.DomainTransactionRecord$TransactionReportField reportField;
   java.lang.Integer reportAmount;

core/src/test/resources/google/registry/tools/server/domain_create_complete.xml

@@ -26,13 +26,13 @@
           <secDNS:keyTag>1</secDNS:keyTag>
           <secDNS:alg>2</secDNS:alg>
           <secDNS:digestType>2</secDNS:digestType>
-          <secDNS:digest>ABCD</secDNS:digest>
+          <secDNS:digest>9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>4</secDNS:keyTag>
           <secDNS:alg>5</secDNS:alg>
           <secDNS:digestType>1</secDNS:digestType>
-          <secDNS:digest>EF01</secDNS:digest>
+          <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>60485</secDNS:keyTag>

core/src/test/resources/google/registry/tools/server/domain_update_add.xml

@@ -23,13 +23,13 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
             <secDNS:digestType>2</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>EF01</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_update_complete.xml

@@ -38,13 +38,13 @@
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>12AB</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>6</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>4</secDNS:digestType>
-            <secDNS:digest>34CD</secDNS:digest>
+            <secDNS:digest>768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -52,13 +52,13 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
             <secDNS:digestType>2</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>EF01</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_update_complete_abc.xml

@@ -38,13 +38,13 @@
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>12AB</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>6</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>4</secDNS:digestType>
-            <secDNS:digest>34CD</secDNS:digest>
+            <secDNS:digest>768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
         <secDNS:add>
@@ -52,13 +52,13 @@
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
             <secDNS:digestType>2</secDNS:digestType>
-            <secDNS:digest>ABCD</secDNS:digest>
+            <secDNS:digest>9F86D081884C7D659A2FEAA0C55AD015A3BF4F1B2B0B822CD15D6C15B0F00A08</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>EF01</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>
       </secDNS:update>

core/src/test/resources/google/registry/tools/server/domain_update_remove.xml

@@ -22,13 +22,13 @@
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
             <secDNS:digestType>1</secDNS:digestType>
-            <secDNS:digest>12AB</secDNS:digest>
+            <secDNS:digest>A94A8FE5CCB19BA61C4C0873D391E987982FBBD3</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>6</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
             <secDNS:digestType>4</secDNS:digestType>
-            <secDNS:digest>34CD</secDNS:digest>
+            <secDNS:digest>768412320F7B0AA5812FCE428DC4706B3CAE50E02A64CAA16A782249BFE8EFC4B7EF1CCB126255D196047DFEDF17A0A9</secDNS:digest>
           </secDNS:dsData>
         </secDNS:rem>
       </secDNS:update>