Use built-in Java URL connections instead of UrlFetchService (#1535)

- Use the standard HttpsURLConnection to write/read data
- Rewrite RdeReporter, Nordn*Action, and Marksdb classes and related
  tests to conform to the new format
- Remove FakeURLFetchService and ForwardingUrlFetchService as they weren't used
- Refactor UrlFetchException to UrlConnectionException
- Refactor UrlFetchUtils to UrlConnectionUtils

I will need to test this on Alpha. Fortunately the connections that
don't require auth (e.g. TMDB downloading) should be testable.

core/build.gradle

@@ -707,10 +707,7 @@ createToolTask(
     'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
 
 createToolTask(
-    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
-
-createToolTask(
-    'validateDatastorePipeline', 'google.registry.beam.comparedb.ValidateDatastorePipeline')
+    'validateDatabasePipeline', 'google.registry.beam.comparedb.ValidateDatabasePipeline')
 
 
 createToolTask(
@@ -797,15 +794,10 @@ if (environment == 'alpha') {
               mainClass: 'google.registry.beam.rde.RdePipeline',
               metaData : 'google/registry/beam/rde_pipeline_metadata.json'
           ],
-      validateDatastore  :
+      validateDatabase  :
           [
-              mainClass: 'google.registry.beam.comparedb.ValidateDatastorePipeline',
-              metaData: 'google/registry/beam/validate_datastore_pipeline_metadata.json'
-          ],
-      validateSql  :
-          [
-              mainClass: 'google.registry.beam.comparedb.ValidateSqlPipeline',
-              metaData: 'google/registry/beam/validate_sql_pipeline_metadata.json'
+              mainClass: 'google.registry.beam.comparedb.ValidateDatabasePipeline',
+              metaData: 'google/registry/beam/validate_database_pipeline_metadata.json'
           ],
   ]
   project.tasks.create("stageBeamPipelines") {

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -76,7 +76,10 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private static final Duration LEASE_LENGTH = standardHours(1);
+  public static final String REPLAY_TO_SQL_LOCK_NAME =
+      ReplayCommitLogsToSqlAction.class.getSimpleName();
+
+  public static final Duration REPLAY_TO_SQL_LOCK_LEASE_LENGTH = standardHours(1);
   // Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
   // request timeouts
   private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
@@ -115,7 +118,11 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
     }
     Optional<Lock> lock =
         Lock.acquireSql(
-            this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
+            REPLAY_TO_SQL_LOCK_NAME,
+            null,
+            REPLAY_TO_SQL_LOCK_LEASE_LENGTH,
+            requestStatusChecker,
+            false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
       logger.atSevere().log(message);

core/src/main/java/google/registry/backup/SyncDatastoreToSqlSnapshotAction.java

@@ -29,6 +29,8 @@ import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Sleeper;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -48,8 +50,7 @@ import org.joda.time.Duration;
  * </ul>
  *
  * The caller may release the replication lock upon receiving the response from this action. Please
- * refer to {@link google.registry.tools.ValidateDatastoreWithSqlCommand} for more information on
- * usage.
+ * refer to {@link google.registry.tools.ValidateDatastoreCommand} for more information on usage.
  *
  * <p>This action plays SQL transactions up to the user-specified snapshot, creates a new CommitLog
  * checkpoint, and exports all CommitLogs to GCS up to this checkpoint. The timestamp of this
@@ -115,9 +116,22 @@ public class SyncDatastoreToSqlSnapshotAction implements Runnable {
       response.setPayload(
           String.format(SUCCESS_RESPONSE_TEMPLATE, sqlSnapshotId, checkpoint.getCheckpointTime()));
       return;
-    } catch (Exception e) {
+    } catch (Throwable e) {
+      logger.atSevere().withCause(e).log("Failed to sync Datastore to SQL.");
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
-      response.setPayload(e.getMessage());
+      response.setPayload(getStackTrace(e));
+    }
+  }
+
+  private static String getStackTrace(Throwable e) {
+    try {
+      ByteArrayOutputStream bis = new ByteArrayOutputStream();
+      PrintStream printStream = new PrintStream(bis);
+      e.printStackTrace(printStream);
+      printStream.close();
+      return bis.toString();
+    } catch (RuntimeException re) {
+      return re.getMessage();
     }
   }
 

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -22,15 +22,17 @@ import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Multimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
-import google.registry.util.AppEngineServiceUtils;
+import google.registry.request.Action.Service;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.Retrier;
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -59,25 +61,23 @@ public final class AsyncTaskEnqueuer {
   private static final Duration MAX_ASYNC_ETA = Duration.standardDays(30);
 
   private final Duration asyncDeleteDelay;
-  private final Queue asyncActionsPushQueue;
   private final Queue asyncDeletePullQueue;
   private final Queue asyncDnsRefreshPullQueue;
-  private final AppEngineServiceUtils appEngineServiceUtils;
   private final Retrier retrier;
 
+  private CloudTasksUtils cloudTasksUtils;
+
   @Inject
   public AsyncTaskEnqueuer(
-      @Named(QUEUE_ASYNC_ACTIONS) Queue asyncActionsPushQueue,
       @Named(QUEUE_ASYNC_DELETE) Queue asyncDeletePullQueue,
       @Named(QUEUE_ASYNC_HOST_RENAME) Queue asyncDnsRefreshPullQueue,
       @Config("asyncDeleteFlowMapreduceDelay") Duration asyncDeleteDelay,
-      AppEngineServiceUtils appEngineServiceUtils,
+      CloudTasksUtils cloudTasksUtils,
       Retrier retrier) {
-    this.asyncActionsPushQueue = asyncActionsPushQueue;
     this.asyncDeletePullQueue = asyncDeletePullQueue;
     this.asyncDnsRefreshPullQueue = asyncDnsRefreshPullQueue;
     this.asyncDeleteDelay = asyncDeleteDelay;
-    this.appEngineServiceUtils = appEngineServiceUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
     this.retrier = retrier;
   }
 
@@ -103,19 +103,17 @@ public final class AsyncTaskEnqueuer {
           entityKey, firstResave, MAX_ASYNC_ETA);
       return;
     }
-    logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
-    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
-    TaskOptions task =
-        TaskOptions.Builder.withUrl(ResaveEntityAction.PATH)
-            .method(Method.POST)
-            .header("Host", backendHostname)
-            .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.stringify())
-            .param(PARAM_REQUESTED_TIME, now.toString());
+    Multimap<String, String> params = ArrayListMultimap.create();
+    params.put(PARAM_RESOURCE_KEY, entityKey.stringify());
+    params.put(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
-      task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
+      params.put(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
     }
-    addTaskToQueueWithRetry(asyncActionsPushQueue, task);
+    logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
+    cloudTasksUtils.enqueue(
+        QUEUE_ASYNC_ACTIONS,
+        cloudTasksUtils.createPostTaskWithDelay(
+            ResaveEntityAction.PATH, Service.BACKEND.toString(), params, etaDuration));
   }
 
   /** Enqueues a task to asynchronously delete a contact or host, by key. */
@@ -152,32 +150,6 @@ public final class AsyncTaskEnqueuer {
             .param(PARAM_REQUESTED_TIME, now.toString()));
   }
 
-  /**
-   * Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked.
-   *
-   * <p>Note: the relockDuration must be present on the lock object.
-   */
-  public void enqueueDomainRelock(RegistryLock lock) {
-    checkArgument(
-        lock.getRelockDuration().isPresent(),
-        "Lock with ID %s not configured for relock",
-        lock.getRevisionId());
-    enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
-  }
-
-  /** Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked. */
-  void enqueueDomainRelock(Duration countdown, long lockRevisionId, int previousAttempts) {
-    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
-    addTaskToQueueWithRetry(
-        asyncActionsPushQueue,
-        TaskOptions.Builder.withUrl(RelockDomainAction.PATH)
-            .method(Method.POST)
-            .header("Host", backendHostname)
-            .param(RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM, String.valueOf(lockRevisionId))
-            .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, String.valueOf(previousAttempts))
-            .countdownMillis(countdown.getMillis()));
-  }
-
   /**
    * Adds a task to a queue with retrying, to avoid aborting the entire flow over a transient issue
    * enqueuing a task.

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -88,7 +88,6 @@ public class RelockDomainAction implements Runnable {
   private final SendEmailService sendEmailService;
   private final DomainLockUtils domainLockUtils;
   private final Response response;
-  private final AsyncTaskEnqueuer asyncTaskEnqueuer;
 
   @Inject
   public RelockDomainAction(
@@ -99,8 +98,7 @@ public class RelockDomainAction implements Runnable {
       @Config("supportEmail") String supportEmail,
       SendEmailService sendEmailService,
       DomainLockUtils domainLockUtils,
-      Response response,
-      AsyncTaskEnqueuer asyncTaskEnqueuer) {
+      Response response) {
     this.oldUnlockRevisionId = oldUnlockRevisionId;
     this.previousAttempts = previousAttempts;
     this.alertRecipientAddress = alertRecipientAddress;
@@ -109,7 +107,6 @@ public class RelockDomainAction implements Runnable {
     this.sendEmailService = sendEmailService;
     this.domainLockUtils = domainLockUtils;
     this.response = response;
-    this.asyncTaskEnqueuer = asyncTaskEnqueuer;
   }
 
   @Override
@@ -245,8 +242,7 @@ public class RelockDomainAction implements Runnable {
       }
     }
     Duration timeBeforeRetry = previousAttempts < ATTEMPTS_BEFORE_SLOWDOWN ? TEN_MINUTES : ONE_HOUR;
-    asyncTaskEnqueuer.enqueueDomainRelock(
-        timeBeforeRetry, oldUnlockRevisionId, previousAttempts + 1);
+    domainLockUtils.enqueueDomainRelock(timeBeforeRetry, oldUnlockRevisionId, previousAttempts + 1);
   }
 
   private void sendSuccessEmail(RegistryLock oldLock) {

core/src/main/java/google/registry/beam/comparedb/ValidateDatabasePipeline.java

@@ -0,0 +1,206 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.util.SystemClock;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * Validates the asynchronous data replication process between Datastore and Cloud SQL.
+ *
+ * <p>This pipeline is to be launched by {@link google.registry.tools.ValidateDatastoreCommand} or
+ * {@link google.registry.tools.ValidateSqlCommand}.
+ */
+@DeleteAfterMigration
+public class ValidateDatabasePipeline {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final Duration COMMITLOG_START_TIME_MARGIN = Duration.standardMinutes(10);
+
+  private final ValidateDatabasePipelineOptions options;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+
+  public ValidateDatabasePipeline(
+      ValidateDatabasePipelineOptions options,
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
+    this.options = options;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    DateTime latestCommitLogTime = DateTime.parse(options.getLatestCommitLogTimestamp());
+    DatastoreSnapshotInfo mostRecentExport =
+        datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
+
+    logger.atInfo().log(
+        "Comparing datastore export at %s and commitlog timestamp %s.",
+        mostRecentExport.exportDir(), latestCommitLogTime);
+
+    Optional<String> outputPath =
+        Optional.ofNullable(options.getDiffOutputGcsBucket())
+            .map(
+                bucket ->
+                    String.format(
+                        "gs://%s/validate_database/%s/diffs.txt",
+                        bucket, new SystemClock().nowUtc()));
+    outputPath.ifPresent(path -> logger.atInfo().log("Discrepancies will be logged to %s", path));
+
+    setupPipeline(
+        pipeline,
+        Optional.ofNullable(options.getSqlSnapshotId()),
+        mostRecentExport,
+        latestCommitLogTime,
+        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse),
+        outputPath);
+
+    pipeline.run();
+  }
+
+  static void setupPipeline(
+      Pipeline pipeline,
+      Optional<String> sqlSnapshotId,
+      DatastoreSnapshotInfo mostRecentExport,
+      DateTime latestCommitLogTime,
+      Optional<DateTime> compareStartTime,
+      Optional<String> diffOutputPath) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minus(COMMITLOG_START_TIME_MARGIN),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS,
+            compareStartTime);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId, compareStartTime);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    PCollectionList<String> diffLogs = PCollectionList.empty(pipeline);
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      diffLogs =
+          diffLogs.and(
+              PCollectionList.of(datastoreSnapshot.get(tag))
+                  .and(cloudSqlSnapshot.get(tag))
+                  .apply(
+                      "Combine from both snapshots: " + clazz.getSimpleName(),
+                      Flatten.pCollections())
+                  .apply(
+                      "Assign primary key to merged " + clazz.getSimpleName(),
+                      WithKeys.of(ValidateDatabasePipeline::getPrimaryKeyString)
+                          .withKeyType(strings()))
+                  .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+                  .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity())));
+    }
+    if (diffOutputPath.isPresent()) {
+      diffLogs
+          .apply("Gather diff logs", Flatten.pCollections())
+          .apply(
+              "Output diffs",
+              TextIO.write()
+                  .to(diffOutputPath.get())
+                  /**
+                   * Output to a single file for ease of use since diffs should be few. If this
+                   * assumption turns out not to be false, user should abort the pipeline and
+                   * investigate why.
+                   */
+                  .withoutSharding()
+                  .withDelimiter((Strings.repeat("-", 80) + "\n").toCharArray()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  public static void main(String[] args) {
+    ValidateDatabasePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args)
+            .withValidation()
+            .as(ValidateDatabasePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Set up JPA in the pipeline harness (the locally executed part of the main() method). Reuse
+    // code in RegistryPipelineWorkerInitializer, which only applies to pipeline worker VMs.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+
+    new ValidateDatabasePipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateDatabasePipelineOptions.java

@@ -14,14 +14,15 @@
 
 package google.registry.beam.comparedb;
 
+import google.registry.beam.common.RegistryPipelineOptions;
 import google.registry.model.annotations.DeleteAfterMigration;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.Validation;
 
-/** BEAM pipeline options for {@link ValidateDatastorePipelineOptions}. */
+/** BEAM pipeline options for {@link ValidateDatabasePipeline}. */
 @DeleteAfterMigration
-public interface ValidateDatastorePipelineOptions extends ValidateSqlPipelineOptions {
+public interface ValidateDatabasePipelineOptions extends RegistryPipelineOptions {
 
   @Description(
       "The id of the SQL snapshot to be compared with Datastore. "
@@ -36,4 +37,19 @@ public interface ValidateDatastorePipelineOptions extends ValidateSqlPipelineOpt
   String getLatestCommitLogTimestamp();
 
   void setLatestCommitLogTimestamp(String commitLogEndTimestamp);
+
+  @Description(
+      "For history entries and EPP resources, only those modified strictly after this time are "
+          + "included in comparison. Value is in ISO8601 format. "
+          + "Other entity types are not affected.")
+  @Nullable
+  String getComparisonStartTimestamp();
+
+  void setComparisonStartTimestamp(String comparisonStartTimestamp);
+
+  @Description("The GCS bucket where discrepancies found during comparison should be logged.")
+  @Nullable
+  String getDiffOutputGcsBucket();
+
+  void setDiffOutputGcsBucket(String gcsBucket);
 }

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipeline.java

@@ -1,83 +0,0 @@
-// Copyright 2022 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.comparedb;
-
-import google.registry.beam.common.RegistryPipelineOptions;
-import google.registry.beam.common.RegistryPipelineWorkerInitializer;
-import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
-import google.registry.model.annotations.DeleteAfterMigration;
-import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
-import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
-import java.util.Optional;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.joda.time.DateTime;
-
-/**
- * Validates the asynchronous data replication process from Cloud SQL (primary) to Datastore
- * (secondary).
- *
- * <p>This pipeline simply compares the snapshots provided by an invoker, which is responsible for
- * obtaining two consistent snapshots for the same point of time.
- */
-// TODO(weiminyu): Implement the invoker action in a followup PR.
-@DeleteAfterMigration
-public class ValidateDatastorePipeline {
-
-  private final ValidateDatastorePipelineOptions options;
-  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
-
-  public ValidateDatastorePipeline(
-      ValidateDatastorePipelineOptions options,
-      LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
-    this.options = options;
-    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
-  }
-
-  void run(Pipeline pipeline) {
-    DateTime latestCommitLogTime = DateTime.parse(options.getLatestCommitLogTimestamp());
-    DatastoreSnapshotInfo mostRecentExport =
-        datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
-
-    ValidateSqlPipeline.setupPipeline(
-        pipeline,
-        Optional.ofNullable(options.getSqlSnapshotId()),
-        mostRecentExport,
-        latestCommitLogTime,
-        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
-
-    pipeline.run();
-  }
-
-  public static void main(String[] args) {
-    ValidateDatastorePipelineOptions options =
-        PipelineOptionsFactory.fromArgs(args)
-            .withValidation()
-            .as(ValidateDatastorePipelineOptions.class);
-    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
-
-    // Defensively set important options.
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
-    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
-
-    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
-    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
-
-    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
-        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
-            .datastoreSnapshotInfoFinder();
-    new ValidateDatastorePipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
-  }
-}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -1,264 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.comparedb;
-
-import static com.google.common.base.Verify.verify;
-import static org.apache.beam.sdk.values.TypeDescriptors.strings;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
-import com.google.common.flogger.FluentLogger;
-import google.registry.beam.common.DatabaseSnapshot;
-import google.registry.beam.common.RegistryPipelineWorkerInitializer;
-import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
-import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
-import google.registry.model.annotations.DeleteAfterMigration;
-import google.registry.model.common.DatabaseMigrationStateSchedule;
-import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
-import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.DomainHistory;
-import google.registry.model.replay.SqlEntity;
-import google.registry.model.replay.SqlReplayCheckpoint;
-import google.registry.model.server.Lock;
-import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
-import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
-import google.registry.persistence.transaction.TransactionManagerFactory;
-import google.registry.util.RequestStatusChecker;
-import google.registry.util.SystemClock;
-import java.io.Serializable;
-import java.util.Optional;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult.State;
-import org.apache.beam.sdk.coders.SerializableCoder;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.values.PCollectionList;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.beam.sdk.values.TupleTag;
-import org.joda.time.DateTime;
-import org.joda.time.Duration;
-
-/**
- * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
- * (secondary storage).
- */
-@DeleteAfterMigration
-public class ValidateSqlPipeline {
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  /** Specifies the extra CommitLogs to load before the start of a Database export. */
-  private static final Duration COMMITLOG_START_TIME_MARGIN = Duration.standardMinutes(10);
-
-  /**
-   * Name of the lock used by the commitlog replay process.
-   *
-   * <p>See {@link google.registry.backup.ReplayCommitLogsToSqlAction} for more information.
-   */
-  private static final String COMMITLOG_REPLAY_LOCK_NAME = "ReplayCommitLogsToSqlAction";
-
-  private static final Duration REPLAY_LOCK_LEASE_LENGTH = Duration.standardHours(1);
-  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_TIMEOUT =
-      java.time.Duration.ofMinutes(6);
-  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_DELAY =
-      java.time.Duration.ofSeconds(30);
-
-  private final ValidateSqlPipelineOptions options;
-  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
-
-  public ValidateSqlPipeline(
-      ValidateSqlPipelineOptions options, LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
-    this.options = options;
-    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
-  }
-
-  @VisibleForTesting
-  void run(Pipeline pipeline) {
-    Optional<Lock> lock = acquireCommitLogReplayLock();
-    if (lock.isPresent()) {
-      logger.atInfo().log("Acquired CommitLog Replay lock.");
-    } else {
-      throw new RuntimeException("Failed to acquire CommitLog Replay lock.");
-    }
-
-    try {
-      DateTime latestCommitLogTime =
-          TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
-      DatastoreSnapshotInfo mostRecentExport =
-          datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
-      Preconditions.checkState(
-          latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
-          "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
-      try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
-        // Eagerly release the commitlog replay lock so that replay can resume.
-        lock.ifPresent(Lock::releaseSql);
-        lock = Optional.empty();
-
-        logger.atInfo().log(
-            "Starting comparison with export at %s and latestCommitLogTime at %s",
-            mostRecentExport.exportDir(), latestCommitLogTime);
-
-        setupPipeline(
-            pipeline,
-            Optional.of(databaseSnapshot.getSnapshotId()),
-            mostRecentExport,
-            latestCommitLogTime,
-            Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
-        State state = pipeline.run().waitUntilFinish();
-        if (!State.DONE.equals(state)) {
-          throw new IllegalStateException("Unexpected pipeline state: " + state);
-        }
-      }
-    } finally {
-      lock.ifPresent(Lock::releaseSql);
-    }
-  }
-
-  static void setupPipeline(
-      Pipeline pipeline,
-      Optional<String> sqlSnapshotId,
-      DatastoreSnapshotInfo mostRecentExport,
-      DateTime latestCommitLogTime,
-      Optional<DateTime> compareStartTime) {
-    pipeline
-        .getCoderRegistry()
-        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
-
-    PCollectionTuple datastoreSnapshot =
-        DatastoreSnapshots.loadDatastoreSnapshotByKind(
-            pipeline,
-            mostRecentExport.exportDir(),
-            mostRecentExport.commitLogDir(),
-            mostRecentExport.exportInterval().getStart().minus(COMMITLOG_START_TIME_MARGIN),
-            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
-            // this parameter is exclusive.
-            latestCommitLogTime.plusMillis(1),
-            DatastoreSnapshots.ALL_DATASTORE_KINDS,
-            compareStartTime);
-
-    PCollectionTuple cloudSqlSnapshot =
-        SqlSnapshots.loadCloudSqlSnapshotByType(
-            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId, compareStartTime);
-
-    verify(
-        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
-        "Expecting the same set of types in both snapshots.");
-
-    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
-      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
-      verify(
-          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
-      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
-      PCollectionList.of(datastoreSnapshot.get(tag))
-          .and(cloudSqlSnapshot.get(tag))
-          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
-          .apply(
-              "Assign primary key to merged " + clazz.getSimpleName(),
-              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
-          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
-          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
-    }
-  }
-
-  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
-    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
-    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
-    // DomainHistory. See BulkQueryEntities.java for more information.
-    if (sqlEntity instanceof DomainBase) {
-      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
-    }
-    if (sqlEntity instanceof DomainHistory) {
-      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
-    }
-    return sqlEntity.getPrimaryKeyString();
-  }
-
-  private static Optional<Lock> acquireCommitLogReplayLock() {
-    Stopwatch stopwatch = Stopwatch.createStarted();
-    while (stopwatch.elapsed().minus(REPLAY_LOCK_ACQUIRE_TIMEOUT).isNegative()) {
-      Optional<Lock> lock = tryAcquireCommitLogReplayLock();
-      if (lock.isPresent()) {
-        return lock;
-      }
-      logger.atInfo().log("Failed to acquired CommitLog Replay lock. Will retry...");
-      try {
-        Thread.sleep(REPLAY_LOCK_ACQUIRE_DELAY.toMillis());
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        throw new RuntimeException("Interrupted.");
-      }
-    }
-    return Optional.empty();
-  }
-
-  private static Optional<Lock> tryAcquireCommitLogReplayLock() {
-    return Lock.acquireSql(
-        COMMITLOG_REPLAY_LOCK_NAME,
-        null,
-        REPLAY_LOCK_LEASE_LENGTH,
-        getLockingRequestStatusChecker(),
-        false);
-  }
-
-  /**
-   * Returns a fake implementation of {@link RequestStatusChecker} that is required for lock
-   * acquisition. The default implementation is AppEngine-specific and is unusable on GCE.
-   */
-  private static RequestStatusChecker getLockingRequestStatusChecker() {
-    return new RequestStatusChecker() {
-      @Override
-      public String getLogId() {
-        return "ValidateSqlPipeline";
-      }
-
-      LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
-          DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
-              .datastoreSnapshotInfoFinder();
-
-      @Override
-      public boolean isRunning(String requestLogId) {
-        return true;
-      }
-    };
-  }
-
-  public static void main(String[] args) {
-    ValidateSqlPipelineOptions options =
-        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
-
-    // Defensively set important options.
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
-    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
-
-    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
-    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
-
-    MigrationState state =
-        DatabaseMigrationStateSchedule.getValueAtTime(new SystemClock().nowUtc());
-    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
-      throw new IllegalStateException("This pipeline is not designed for migration phase " + state);
-    }
-
-    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
-        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
-            .datastoreSnapshotInfoFinder();
-
-    new ValidateSqlPipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
-  }
-}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -1,34 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.comparedb;
-
-import google.registry.beam.common.RegistryPipelineOptions;
-import google.registry.model.annotations.DeleteAfterMigration;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.options.Description;
-
-/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
-@DeleteAfterMigration
-public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {
-
-  @Description(
-      "For history entries and EPP resources, only those modified strictly after this time are "
-          + "included in comparison. Value is in ISO8601 format. "
-          + "Other entity types are not affected.")
-  @Nullable
-  String getComparisonStartTimestamp();
-
-  void setComparisonStartTimestamp(String comparisonStartTimestamp);
-}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -20,7 +20,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.flogger.FluentLogger;
 import google.registry.beam.initsql.Transforms;
 import google.registry.config.RegistryEnvironment;
 import google.registry.model.EppResource;
@@ -37,6 +36,7 @@ import google.registry.model.poll.PollMessage;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.util.DiffUtils;
 import java.lang.reflect.Field;
 import java.math.BigInteger;
 import java.util.HashMap;
@@ -51,10 +51,9 @@ import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
 
-/** Helpers for use by {@link ValidateSqlPipeline}. */
+/** Helpers for use by {@link ValidateDatabasePipeline}. */
 @DeleteAfterMigration
 final class ValidateSqlUtils {
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   private ValidateSqlUtils() {}
 
@@ -66,7 +65,7 @@ final class ValidateSqlUtils {
    * Query template for finding the median value of the {@code history_revision_id} column in one of
    * the History tables.
    *
-   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * <p>The {@link ValidateDatabasePipeline} uses this query to parallelize the query to some of the
    * history tables. Although the {@code repo_id} column is the leading column in the primary keys
    * of these tables, in practice and with production data, division by {@code history_revision_id}
    * works slightly faster for unknown reasons.
@@ -98,15 +97,13 @@ final class ValidateSqlUtils {
     return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
   }
 
-  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, String> {
     private final HashMap<String, Counter> totalCounters = new HashMap<>();
     private final HashMap<String, Counter> missingCounters = new HashMap<>();
     private final HashMap<String, Counter> unequalCounters = new HashMap<>();
     private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
     private final HashMap<String, Counter> duplicateEntityCounters = new HashMap<>();
 
-    private volatile boolean logPrinted = false;
-
     private String getCounterKey(Class<?> clazz) {
       return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
     }
@@ -124,39 +121,36 @@ final class ValidateSqlUtils {
           counterKey, Metrics.counter("CompareDB", "Duplicate Entities:" + counterKey));
     }
 
+    String duplicateEntityLog(String key, ImmutableList<SqlEntity> entities) {
+      return String.format("%s: %d entities.", key, entities.size());
+    }
+
+    String unmatchedEntityLog(String key, SqlEntity entry) {
+      // For a PollMessage only found in Datastore, key is not enough to query for it.
+      return String.format("Missing in one DB:\n%s", entry instanceof PollMessage ? entry : key);
+    }
+
     /**
      * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
      * This will be removed when we start exporting such entities to GCS.
      */
-    void logDiff(String key, Object entry0, Object entry1) {
-      if (logPrinted) {
-        return;
-      }
-      logPrinted = true;
+    String unEqualEntityLog(String key, SqlEntity entry0, SqlEntity entry1) {
       Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
       Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
-      StringBuilder sb = new StringBuilder();
-      fields0.forEach(
-          (field, value) -> {
-            if (fields1.containsKey(field)) {
-              if (!Objects.equals(value, fields1.get(field))) {
-                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
-              }
-            } else {
-              sb.append(field + "Not found in entity 2\n");
-            }
-          });
-      fields1.forEach(
-          (field, value) -> {
-            if (!fields0.containsKey(field)) {
-              sb.append(field + "Not found in entity 1\n");
-            }
-          });
-      logger.atWarning().log(key + "  " + sb.toString());
+      return key + "  " + DiffUtils.prettyPrintEntityDeepDiff(fields0, fields1);
+    }
+
+    String badEntitiesLog(String key, SqlEntity entry0, SqlEntity entry1) {
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      return String.format(
+          "Failed to parse one or both entities for key %s:\n%s\n",
+          key, DiffUtils.prettyPrintEntityDeepDiff(fields0, fields1));
     }
 
     @ProcessElement
-    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+    public void processElement(
+        @Element KV<String, Iterable<SqlEntity>> kv, OutputReceiver<String> out) {
       ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
 
       verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
@@ -169,6 +163,7 @@ final class ValidateSqlUtils {
         // Duplicates may happen with Cursors if imported across projects. Its key in Datastore, the
         // id field, encodes the project name and is not fixed by the importing job.
         duplicateEntityCounters.get(counterKey).inc();
+        out.output(duplicateEntityLog(kv.getKey(), entities) + "\n");
         return;
       }
 
@@ -177,11 +172,7 @@ final class ValidateSqlUtils {
           return;
         }
         missingCounters.get(counterKey).inc();
-        // Temporary debugging help. See logDiff() above.
-        if (!logPrinted) {
-          logPrinted = true;
-          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
-        }
+        out.output(unmatchedEntityLog(kv.getKey(), entities.get(0)) + "\n");
         return;
       }
       SqlEntity entity0 = entities.get(0);
@@ -198,17 +189,14 @@ final class ValidateSqlUtils {
         entity0 = normalizeEntity(entity0);
         entity1 = normalizeEntity(entity1);
       } catch (Exception e) {
-        // Temporary debugging help. See logDiff() above.
-        if (!logPrinted) {
-          logPrinted = true;
-          badEntityCounters.get(counterKey).inc();
-        }
+        badEntityCounters.get(counterKey).inc();
+        out.output(badEntitiesLog(kv.getKey(), entity0, entity1));
         return;
       }
 
       if (!Objects.equals(entity0, entity1)) {
         unequalCounters.get(counterKey).inc();
-        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+        out.output(unEqualEntityLog(kv.getKey(), entities.get(0), entities.get(1)));
       }
     }
   }

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -779,6 +779,10 @@ public class DomainContent extends EppResource
    */
   void setContactFields(Set<DesignatedContact> contacts, boolean includeRegistrant) {
     // Set the individual contact fields.
+    billingContact = techContact = adminContact = null;
+    if (includeRegistrant) {
+      registrantContact = null;
+    }
     for (DesignatedContact contact : contacts) {
       switch (contact.getType()) {
         case BILLING:

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -825,7 +825,6 @@ public class Registrar extends ImmutableObject
 
     public Builder setAllowedTlds(Set<String> allowedTlds) {
       getInstance().allowedTlds = ImmutableSortedSet.copyOf(assertTldsExist(allowedTlds));
-      getInstance().lastUpdateTime = UpdateAutoTimestamp.create(null);
       return this;
     }
 

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -128,13 +128,10 @@ public class ReplicateToDatastoreAction implements Runnable {
                 LastSqlTransaction lastSqlTxn = LastSqlTransaction.load();
                 long nextTxnId = lastSqlTxn.getTransactionId() + 1;
                 if (nextTxnId < txnEntity.getId()) {
-                  // We're missing a transaction.  This is bad.  Transaction ids are supposed to
-                  // increase monotonically, so we abort rather than applying anything out of
-                  // order.
-                  throw new IllegalStateException(
-                      String.format(
-                          "Missing transaction: last txn id = %s, next available txn = %s",
-                          nextTxnId - 1, txnEntity.getId()));
+                  // Missing transaction id.  This can happen normally.  If a transaction gets
+                  // rolled back, the sequence counter doesn't.
+                  logger.atWarning().log(
+                      "Ignoring transaction %s, which does not exist.", nextTxnId);
                 } else if (nextTxnId > txnEntity.getId()) {
                   // We've already replayed this transaction.  This shouldn't happen, as GAE cron
                   // is supposed to avoid overruns and this action shouldn't be executed from any

core/src/main/java/google/registry/module/backend/BackendComponent.java

@@ -43,7 +43,7 @@ import google.registry.rde.JSchModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.URLFetchServiceModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
@@ -80,7 +80,7 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       SheetsServiceModule.class,
       StackdriverModule.class,
-      URLFetchServiceModule.class,
+      UrlConnectionServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
       VoidDnsWriterModule.class,

core/src/main/java/google/registry/module/frontend/FrontendComponent.java

@@ -34,7 +34,6 @@ import google.registry.monitoring.whitebox.StackdriverModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.ui.ConsoleDebug.ConsoleConfigModule;
@@ -46,6 +45,7 @@ import javax.inject.Singleton;
 @Component(
     modules = {
       AuthModule.class,
+      CloudTasksUtilsModule.class,
       ConfigModule.class,
       ConsoleConfigModule.class,
       CredentialModule.class,
@@ -64,7 +64,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -34,7 +34,6 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -62,7 +61,6 @@ import javax.inject.Singleton;
       SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/module/tools/ToolsComponent.java

@@ -36,7 +36,6 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
 import google.registry.util.UtilsModule;
@@ -66,7 +65,6 @@ import javax.inject.Singleton;
       ServerTridProviderModule.class,
       StackdriverModule.class,
       ToolsRequestComponentModule.class,
-      UrlFetchTransportModule.class,
       UserServiceModule.class,
       UtilsModule.class
     })

core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java

@@ -14,6 +14,7 @@
 
 package google.registry.persistence.transaction;
 
+import com.google.common.annotations.VisibleForTesting;
 import google.registry.model.ImmutableObject;
 import google.registry.model.replay.SqlOnlyEntity;
 import javax.persistence.Entity;
@@ -39,7 +40,8 @@ public class TransactionEntity extends ImmutableObject implements SqlOnlyEntity
 
   TransactionEntity() {}
 
-  TransactionEntity(byte[] contents) {
+  @VisibleForTesting
+  public TransactionEntity(byte[] contents) {
     this.contents = contents;
   }
 

core/src/main/java/google/registry/rde/RdeReporter.java

@@ -14,26 +14,24 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
-import static com.google.common.io.BaseEncoding.base64;
-import static com.google.common.net.HttpHeaders.AUTHORIZATION;
-import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
+import static google.registry.request.UrlConnectionUtils.setPayload;
 import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
+import com.google.api.client.http.HttpMethods;
 import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.request.HttpException.InternalServerErrorException;
+import google.registry.request.UrlConnectionService;
 import google.registry.util.Retrier;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
 import google.registry.xjc.iirdea.XjcIirdeaResult;
@@ -41,6 +39,7 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.SocketTimeoutException;
 import java.net.URL;
@@ -55,12 +54,15 @@ public class RdeReporter {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  /** @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
-   *     ICANN Registry Interfaces - Interface details</a>*/
-  private static final String REPORT_MIME = "text/xml";
+  /**
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
+   *     ICANN Registry Interfaces - Interface details</a>
+   */
+  private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
 
   @Inject Retrier retrier;
-  @Inject URLFetchService urlFetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
   @Inject @Key("icannReportingPassword") String password;
   @Inject RdeReporter() {}
@@ -74,29 +76,24 @@ public class RdeReporter {
     // Send a PUT request to ICANN's HTTPS server.
     URL url = makeReportUrl(header.getTld(), report.getId());
     String username = header.getTld() + "_ry";
-    String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
-    final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
-    req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
-    req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    req.setPayload(reportBytes);
     logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
-    HTTPResponse rsp =
+    byte[] responseBytes =
         retrier.callWithRetry(
             () -> {
-              HTTPResponse rsp1 = urlFetchService.fetch(req);
-              switch (rsp1.getResponseCode()) {
-                case SC_OK:
-                case SC_BAD_REQUEST:
-                  break;
-                default:
-                  throw new UrlFetchException("PUT failed", req, rsp1);
+              HttpURLConnection connection = urlConnectionService.createConnection(url);
+              connection.setRequestMethod(HttpMethods.PUT);
+              setBasicAuth(connection, username, password);
+              setPayload(connection, reportBytes, MEDIA_TYPE.toString());
+              int responseCode = connection.getResponseCode();
+              if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) {
+                return getResponseBytes(connection);
               }
-              return rsp1;
+              throw new UrlConnectionException("PUT failed", connection);
             },
             SocketTimeoutException.class);
 
     // Ensure the XML response is valid.
-    XjcIirdeaResult result = parseResult(rsp);
+    XjcIirdeaResult result = parseResult(responseBytes);
     if (result.getCode().getValue() != 1000) {
       logger.atWarning().log(
           "PUT rejected: %d %s\n%s",
@@ -108,11 +105,11 @@ public class RdeReporter {
   /**
    * Unmarshals IIRDEA XML result object from {@link HTTPResponse} payload.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
+   * @see <a
+   *     href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4.1">
    *     ICANN Registry Interfaces - IIRDEA Result Object</a>
    */
-  private XjcIirdeaResult parseResult(HTTPResponse rsp) throws XmlException {
-    byte[] responseBytes = rsp.getContent();
+  private XjcIirdeaResult parseResult(byte[] responseBytes) throws XmlException {
     logger.atInfo().log("Received response:\n%s", new String(responseBytes, UTF_8));
     XjcIirdeaResponseElement response = XjcXmlTransformer.unmarshal(
         XjcIirdeaResponseElement.class, new ByteArrayInputStream(responseBytes));

core/src/main/java/google/registry/request/Modules.java

@@ -23,12 +23,11 @@ import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.appengine.api.datastore.DatastoreService;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
 import com.google.appengine.api.users.UserService;
 import com.google.appengine.api.users.UserServiceFactory;
 import dagger.Module;
 import dagger.Provides;
+import java.net.HttpURLConnection;
 import javax.inject.Singleton;
 
 /** Dagger modules for App Engine services and other vendor classes. */
@@ -45,14 +44,12 @@ public final class Modules {
     }
   }
 
-  /** Dagger module for {@link URLFetchService}. */
+  /** Dagger module for {@link UrlConnectionService}. */
   @Module
-  public static final class URLFetchServiceModule {
-    private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService();
-
+  public static final class UrlConnectionServiceModule {
     @Provides
-    static URLFetchService provideURLFetchService() {
-      return fetchService;
+    static UrlConnectionService provideUrlConnectionService() {
+      return url -> (HttpURLConnection) url.openConnection();
     }
   }
 

core/src/main/java/google/registry/request/UrlConnectionService.java

@@ -0,0 +1,25 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.request;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/** Functional interface for opening a connection from a URL, injectable for testing. */
+public interface UrlConnectionService {
+
+  HttpURLConnection createConnection(URL url) throws IOException;
+}

core/src/main/java/google/registry/request/UrlConnectionUtils.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.io.BaseEncoding.base64;
@@ -22,36 +22,41 @@ import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.common.base.Ascii;
 import com.google.common.base.Strings;
+import com.google.common.io.ByteStreams;
 import com.google.common.net.MediaType;
-import java.util.Optional;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.URLConnection;
 import java.util.Random;
 
-/** Helper methods for the App Engine URL fetch service. */
-public final class UrlFetchUtils {
+/** Utilities for common functionality relating to {@link java.net.URLConnection}s. */
+public class UrlConnectionUtils {
 
-  /** Returns value of first header matching {@code name}. */
-  public static Optional<String> getHeaderFirst(HTTPResponse rsp, String name) {
-    return getHeaderFirstInternal(rsp.getHeadersUncombined(), name);
+  /** Retrieves the response from the given connection as a byte array. */
+  public static byte[] getResponseBytes(URLConnection connection) throws IOException {
+    return ByteStreams.toByteArray(connection.getInputStream());
   }
 
-  /** Returns value of first header matching {@code name}. */
-  public static Optional<String> getHeaderFirst(HTTPRequest req, String name) {
-    return getHeaderFirstInternal(req.getHeaders(), name);
+  /** Sets auth on the given connection with the given username/password. */
+  public static void setBasicAuth(URLConnection connection, String username, String password) {
+    setBasicAuth(connection, String.format("%s:%s", username, password));
   }
 
-  private static Optional<String> getHeaderFirstInternal(Iterable<HTTPHeader> hdrs, String name) {
-    name = Ascii.toLowerCase(name);
-    for (HTTPHeader header : hdrs) {
-      if (Ascii.toLowerCase(header.getName()).equals(name)) {
-        return Optional.of(header.getValue());
-      }
+  /** Sets auth on the given connection with the given string, formatted "username:password". */
+  public static void setBasicAuth(URLConnection connection, String usernameAndPassword) {
+    String token = base64().encode(usernameAndPassword.getBytes(UTF_8));
+    connection.setRequestProperty(AUTHORIZATION, "Basic " + token);
+  }
+
+  /** Sets the given byte[] payload on the given connection with a particular content type. */
+  public static void setPayload(URLConnection connection, byte[] bytes, String contentType)
+      throws IOException {
+    connection.setRequestProperty(CONTENT_TYPE, contentType);
+    connection.setDoOutput(true);
+    try (DataOutputStream dataStream = new DataOutputStream(connection.getOutputStream())) {
+      dataStream.write(bytes);
     }
-    return Optional.empty();
   }
 
   /**
@@ -62,16 +67,16 @@ public final class UrlFetchUtils {
    * @see <a href="http://www.ietf.org/rfc/rfc2388.txt">RFC2388 - Returning Values from Forms</a>
    */
   public static void setPayloadMultipart(
-      HTTPRequest request,
+      URLConnection connection,
       String name,
       String filename,
       MediaType contentType,
       String data,
-      Random random) {
+      Random random)
+      throws IOException {
     String boundary = createMultipartBoundary(random);
     checkState(
-        !data.contains(boundary),
-        "Multipart data contains autogenerated boundary: %s", boundary);
+        !data.contains(boundary), "Multipart data contains autogenerated boundary: %s", boundary);
     String multipart =
         String.format("--%s\r\n", boundary)
             + String.format(
@@ -83,11 +88,9 @@ public final class UrlFetchUtils {
             + "\r\n"
             + String.format("--%s--\r\n", boundary);
     byte[] payload = multipart.getBytes(UTF_8);
-    request.addHeader(
-        new HTTPHeader(
-            CONTENT_TYPE, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary)));
-    request.addHeader(new HTTPHeader(CONTENT_LENGTH, Integer.toString(payload.length)));
-    request.setPayload(payload);
+    connection.setRequestProperty(CONTENT_LENGTH, Integer.toString(payload.length));
+    setPayload(
+        connection, payload, String.format("multipart/form-data;" + " boundary=\"%s\"", boundary));
   }
 
   private static String createMultipartBoundary(Random random) {
@@ -98,12 +101,4 @@ public final class UrlFetchUtils {
     // See https://tools.ietf.org/html/rfc2046#section-5.1.1
     return Strings.repeat("-", 30) + base64().encode(rand);
   }
-
-  /** Sets the HTTP Basic Authentication header on an {@link HTTPRequest}. */
-  public static void setAuthorizationHeader(HTTPRequest req, Optional<String> login) {
-    if (login.isPresent()) {
-      String token = base64().encode(login.get().getBytes(UTF_8));
-      req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
-    }
-  }
 }

core/src/main/java/google/registry/tmch/LordnRequestInitializer.java

@@ -15,12 +15,12 @@
 package google.registry.tmch;
 
 import static com.google.common.base.Verify.verifyNotNull;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
 import com.google.common.flogger.FluentLogger;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.tld.Registry;
+import google.registry.request.UrlConnectionUtils;
+import java.net.HttpURLConnection;
 import java.util.Optional;
 import javax.inject.Inject;
 
@@ -37,8 +37,9 @@ final class LordnRequestInitializer {
   }
 
   /** Initializes a URL fetch request for talking to the MarksDB server. */
-  void initialize(HTTPRequest request, String tld) {
-    setAuthorizationHeader(request, getMarksDbLordnCredentials(tld));
+  void initialize(HttpURLConnection connection, String tld) {
+    getMarksDbLordnCredentials(tld)
+        .ifPresent(login -> UrlConnectionUtils.setBasicAuth(connection, login));
   }
 
   /** Returns the username and password for the current TLD to login to the MarksDB server. */

core/src/main/java/google/registry/tmch/Marksdb.java

@@ -14,25 +14,23 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
 import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
+import static google.registry.request.UrlConnectionUtils.setBasicAuth;
 import static google.registry.util.HexDumper.dumpHex;
-import static google.registry.util.UrlFetchUtils.setAuthorizationHeader;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
-import google.registry.util.UrlFetchException;
+import google.registry.request.UrlConnectionService;
+import google.registry.util.UrlConnectionException;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.Security;
 import java.security.SignatureException;
@@ -57,7 +55,8 @@ public final class Marksdb {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final int MAX_DNL_LOGGING_LENGTH = 500;
 
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Key("marksdbPublicKey") PGPPublicKey marksdbPublicKey;
   @Inject Marksdb() {}
@@ -112,19 +111,16 @@ public final class Marksdb {
   }
 
   byte[] fetch(URL url, Optional<String> loginAndPassword) throws IOException {
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
-    setAuthorizationHeader(req, loginAndPassword);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
+    loginAndPassword.ifPresent(auth -> setBasicAuth(connection, auth));
     try {
-      rsp = fetchService.fetch(req);
+      if (connection.getResponseCode() != SC_OK) {
+        throw new UrlConnectionException("Failed to fetch from MarksDB", connection);
+      }
+      return getResponseBytes(connection);
     } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    if (rsp.getResponseCode() != SC_OK) {
-      throw new UrlFetchException("Failed to fetch from MarksDB", req, rsp);
-    }
-    return rsp.getContent();
   }
 
   List<String> fetchSignedCsv(Optional<String> loginAndPassword, String csvPath, String sigPath)

core/src/main/java/google/registry/tmch/NordnUploadAction.java

@@ -16,28 +16,23 @@ package google.registry.tmch;
 
 import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.POST;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.CSV_UTF_8;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_CLAIMS;
 import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 
+import com.google.api.client.http.HttpMethods;
 import com.google.appengine.api.taskqueue.LeaseOptions;
 import com.google.appengine.api.taskqueue.Queue;
 import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
@@ -49,16 +44,18 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
+import google.registry.request.UrlConnectionService;
+import google.registry.request.UrlConnectionUtils;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
-import java.util.Optional;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
@@ -97,7 +94,8 @@ public final class NordnUploadAction implements Runnable {
   @Inject Retrier retrier;
   @Inject SecureRandom random;
   @Inject LordnRequestInitializer lordnRequestInitializer;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Config("tmchMarksdbUrl") String tmchMarksdbUrl;
   @Inject @Parameter(LORDN_PHASE_PARAM) String phase;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -193,47 +191,48 @@ public final class NordnUploadAction implements Runnable {
    * <p>Idempotency: If the exact same LORDN report is uploaded twice, the MarksDB server will
    * return the same confirmation number.
    *
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">
-   *     TMCH functional specifications - LORDN File</a>
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">TMCH
+   *     functional specifications - LORDN File</a>
    */
   private void uploadCsvToLordn(String urlPath, String csvData) throws IOException {
     String url = tmchMarksdbUrl + urlPath;
     logger.atInfo().log(
         "LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData);
-    HTTPRequest req = new HTTPRequest(new URL(url), POST, validateCertificate().setDeadline(60d));
-    lordnRequestInitializer.initialize(req, tld);
-    setPayloadMultipart(req, "file", "claims.csv", CSV_UTF_8, csvData, random);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(new URL(url));
+    connection.setRequestMethod(HttpMethods.POST);
+    lordnRequestInitializer.initialize(connection, tld);
+    UrlConnectionUtils.setPayloadMultipart(
+        connection, "file", "claims.csv", CSV_UTF_8, csvData, random);
     try {
-      rsp = fetchService.fetch(req);
+      int responseCode = connection.getResponseCode();
+      if (logger.atInfo().isEnabled()) {
+        String responseContent = new String(getResponseBytes(connection), US_ASCII);
+        if (responseContent.isEmpty()) {
+          responseContent = "(null)";
+        }
+        logger.atInfo().log(
+            "LORDN upload task %s response: HTTP response code %d, response data: %s",
+            actionLogId, responseCode, responseContent);
+      }
+      if (responseCode != SC_ACCEPTED) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB",
+                actionLogId),
+            connection);
+      }
+      String location = connection.getHeaderField(LOCATION);
+      if (location == null) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN upload task %s error: MarksDB failed to provide a Location header",
+                actionLogId),
+            connection);
+      }
+      getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location)));
     } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    if (logger.atInfo().isEnabled()) {
-      String response =
-          (rsp.getContent() == null) ? "(null)" : new String(rsp.getContent(), US_ASCII);
-      logger.atInfo().log(
-          "LORDN upload task %s response: HTTP response code %d, response data: %s",
-          actionLogId, rsp.getResponseCode(), response);
-    }
-    if (rsp.getResponseCode() != SC_ACCEPTED) {
-      throw new UrlFetchException(
-          String.format(
-              "LORDN upload task %s error: Failed to upload LORDN claims to MarksDB", actionLogId),
-          req,
-          rsp);
-    }
-    Optional<String> location = getHeaderFirst(rsp, LOCATION);
-    if (!location.isPresent()) {
-      throw new UrlFetchException(
-          String.format(
-              "LORDN upload task %s error: MarksDB failed to provide a Location header",
-              actionLogId),
-          req,
-          rsp);
-    }
-    getQueue(NordnVerifyAction.QUEUE).add(makeVerifyTask(new URL(location.get())));
   }
 
   private TaskOptions makeVerifyTask(URL url) {

core/src/main/java/google/registry/tmch/NordnVerifyAction.java

@@ -14,15 +14,11 @@
 
 package google.registry.tmch;
 
-import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
-import static com.google.appengine.api.urlfetch.HTTPMethod.GET;
+import static google.registry.request.UrlConnectionUtils.getResponseBytes;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteSource;
@@ -32,9 +28,11 @@ import google.registry.request.HttpException.ConflictException;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
+import google.registry.request.UrlConnectionService;
 import google.registry.request.auth.Auth;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
 import java.io.IOException;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Map.Entry;
 import javax.inject.Inject;
@@ -68,7 +66,8 @@ public final class NordnVerifyAction implements Runnable {
 
   @Inject LordnRequestInitializer lordnRequestInitializer;
   @Inject Response response;
-  @Inject URLFetchService fetchService;
+  @Inject UrlConnectionService urlConnectionService;
+
   @Inject @Header(URL_HEADER) URL url;
   @Inject @Header(HEADER_ACTION_LOG_ID) String actionLogId;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
@@ -96,51 +95,49 @@ public final class NordnVerifyAction implements Runnable {
   @VisibleForTesting
   LordnLog verify() throws IOException {
     logger.atInfo().log("LORDN verify task %s: Sending request to URL %s", actionLogId, url);
-    HTTPRequest req = new HTTPRequest(url, GET, validateCertificate().setDeadline(60d));
-    lordnRequestInitializer.initialize(req, tld);
-    HTTPResponse rsp;
+    HttpURLConnection connection = urlConnectionService.createConnection(url);
+    lordnRequestInitializer.initialize(connection, tld);
     try {
-      rsp = fetchService.fetch(req);
-    } catch (IOException e) {
-      throw new IOException(
-          String.format("Error connecting to MarksDB at URL %s", url), e);
-    }
-    logger.atInfo().log(
-        "LORDN verify task %s response: HTTP response code %d, response data: %s",
-        actionLogId, rsp.getResponseCode(), rsp.getContent());
-    if (rsp.getResponseCode() == SC_NO_CONTENT) {
-      // Send a 400+ status code so App Engine will retry the task.
-      throw new ConflictException("Not ready");
-    }
-    if (rsp.getResponseCode() != SC_OK) {
-      throw new UrlFetchException(
-          String.format("LORDN verify task %s: Failed to verify LORDN upload to MarksDB.",
-              actionLogId),
-          req, rsp);
-    }
-    LordnLog log =
-        LordnLog.parse(ByteSource.wrap(rsp.getContent()).asCharSource(UTF_8).readLines());
-    if (log.getStatus() == LordnLog.Status.ACCEPTED) {
-      logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
-    } else {
-      logger.atSevere().log(
-          "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
-    }
-    for (Entry<String, LordnLog.Result> result : log) {
-      switch (result.getValue().getOutcome()) {
-        case OK:
-          break;
-        case WARNING:
-          // fall through
-        case ERROR:
-          logger.atWarning().log(result.toString());
-          break;
-        default:
-          logger.atWarning().log(
-              "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
-          break;
+      int responseCode = connection.getResponseCode();
+      logger.atInfo().log(
+          "LORDN verify task %s response: HTTP response code %d", actionLogId, responseCode);
+      if (responseCode == SC_NO_CONTENT) {
+        // Send a 400+ status code so App Engine will retry the task.
+        throw new ConflictException("Not ready");
       }
+      if (responseCode != SC_OK) {
+        throw new UrlConnectionException(
+            String.format(
+                "LORDN verify task %s: Failed to verify LORDN upload to MarksDB.", actionLogId),
+            connection);
+      }
+      LordnLog log =
+          LordnLog.parse(
+              ByteSource.wrap(getResponseBytes(connection)).asCharSource(UTF_8).readLines());
+      if (log.getStatus() == LordnLog.Status.ACCEPTED) {
+        logger.atInfo().log("LORDN verify task %s: Upload accepted.", actionLogId);
+      } else {
+        logger.atSevere().log(
+            "LORDN verify task %s: Upload rejected with reason: %s", actionLogId, log);
+      }
+      for (Entry<String, LordnLog.Result> result : log) {
+        switch (result.getValue().getOutcome()) {
+          case OK:
+            break;
+          case WARNING:
+            // fall through
+          case ERROR:
+            logger.atWarning().log(result.toString());
+            break;
+          default:
+            logger.atWarning().log(
+                "LORDN verify task %s: Unexpected outcome: %s", actionLogId, result);
+            break;
+        }
+      }
+      return log;
+    } catch (IOException e) {
+      throw new IOException(String.format("Error connecting to MarksDB at URL %s", url), e);
     }
-    return log;
   }
 }

core/src/main/java/google/registry/tools/DomainLockUtils.java

@@ -15,14 +15,16 @@
 package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
 import static google.registry.model.EppResourceUtils.loadByForeignKeyCached;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
-import google.registry.batch.AsyncTaskEnqueuer;
+import google.registry.batch.RelockDomainAction;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -32,6 +34,8 @@ import google.registry.model.domain.RegistryLock;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
 import google.registry.model.tld.RegistryLockDao;
+import google.registry.request.Action.Service;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.StringGenerator;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -53,16 +57,16 @@ public final class DomainLockUtils {
 
   private final StringGenerator stringGenerator;
   private final String registryAdminRegistrarId;
-  private final AsyncTaskEnqueuer asyncTaskEnqueuer;
+  private CloudTasksUtils cloudTasksUtils;
 
   @Inject
   public DomainLockUtils(
       @Named("base58StringGenerator") StringGenerator stringGenerator,
       @Config("registryAdminClientId") String registryAdminRegistrarId,
-      AsyncTaskEnqueuer asyncTaskEnqueuer) {
+      CloudTasksUtils cloudTasksUtils) {
     this.stringGenerator = stringGenerator;
     this.registryAdminRegistrarId = registryAdminRegistrarId;
-    this.asyncTaskEnqueuer = asyncTaskEnqueuer;
+    this.cloudTasksUtils = cloudTasksUtils;
   }
 
   /**
@@ -203,10 +207,38 @@ public final class DomainLockUtils {
 
   private void submitRelockIfNecessary(RegistryLock lock) {
     if (lock.getRelockDuration().isPresent()) {
-      asyncTaskEnqueuer.enqueueDomainRelock(lock);
+      enqueueDomainRelock(lock);
     }
   }
 
+  /**
+   * Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked.
+   *
+   * <p>Note: the relockDuration must be present on the lock object.
+   */
+  public void enqueueDomainRelock(RegistryLock lock) {
+    checkArgument(
+        lock.getRelockDuration().isPresent(),
+        "Lock with ID %s not configured for relock",
+        lock.getRevisionId());
+    enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
+  }
+
+  /** Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked. */
+  public void enqueueDomainRelock(Duration countdown, long lockRevisionId, int previousAttempts) {
+    cloudTasksUtils.enqueue(
+        QUEUE_ASYNC_ACTIONS,
+        cloudTasksUtils.createPostTaskWithDelay(
+            RelockDomainAction.PATH,
+            Service.BACKEND.toString(),
+            ImmutableMultimap.of(
+                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
+                String.valueOf(lockRevisionId),
+                RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM,
+                String.valueOf(previousAttempts)),
+            countdown));
+  }
+
   private void setAsRelock(RegistryLock newLock) {
     jpaTm()
         .transact(

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -123,9 +123,10 @@ public final class RegistryTool {
           .put("update_server_locks", UpdateServerLocksCommand.class)
           .put("update_tld", UpdateTldCommand.class)
           .put("upload_claims_list", UploadClaimsListCommand.class)
-          .put("validate_datastore_with_sql", ValidateDatastoreWithSqlCommand.class)
+          .put("validate_datastore", ValidateDatastoreCommand.class)
           .put("validate_escrow_deposit", ValidateEscrowDepositCommand.class)
           .put("validate_login_credentials", ValidateLoginCredentialsCommand.class)
+          .put("validate_sql", ValidateSqlCommand.class)
           .put("verify_ote", VerifyOteCommand.class)
           .put("whois_query", WhoisQueryCommand.class)
           .build();

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -38,8 +38,7 @@ import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
-import google.registry.request.Modules.URLFetchServiceModule;
-import google.registry.request.Modules.UrlFetchTransportModule;
+import google.registry.request.Modules.UrlConnectionServiceModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
@@ -79,8 +78,7 @@ import javax.inject.Singleton;
       RegistryToolDataflowModule.class,
       RequestFactoryModule.class,
       SecretManagerModule.class,
-      URLFetchServiceModule.class,
-      UrlFetchTransportModule.class,
+      UrlConnectionServiceModule.class,
       UserServiceModule.class,
       UtilsModule.class,
       VoidDnsWriterModule.class,
@@ -171,12 +169,14 @@ interface RegistryToolComponent {
 
   void inject(UpdateTldCommand command);
 
-  void inject(ValidateDatastoreWithSqlCommand command);
+  void inject(ValidateDatastoreCommand command);
 
   void inject(ValidateEscrowDepositCommand command);
 
   void inject(ValidateLoginCredentialsCommand command);
 
+  void inject(ValidateSqlCommand command);
+
   void inject(WhoisQueryCommand command);
 
   AppEngineConnection appEngineConnection();

core/src/main/java/google/registry/tools/ValidateDatabaseMigrationCommand.java

@@ -0,0 +1,232 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.beam.BeamUtils.createJobName;
+
+import com.beust.jcommander.Parameter;
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.Job;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.tools.params.DateTimeParameter;
+import google.registry.util.Clock;
+import google.registry.util.RequestStatusChecker;
+import google.registry.util.Sleeper;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.UUID;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/** Shared setup for commands that validate the data replication between Datastore and Cloud SQL. */
+abstract class ValidateDatabaseMigrationCommand
+    implements CommandWithConnection, CommandWithRemoteApi {
+
+  private static final String PIPELINE_NAME = "validate_database_pipeline";
+
+  private static final String MANUAL_PIPELINE_LAUNCH_COMMAND_TEMPLATE =
+      "gcloud dataflow flex-template run "
+          + "\"%s-${USER}-$(date +%%Y%%m%%dt%%H%%M%%S)\" "
+          + "--template-file-gcs-location %s "
+          + "--project %s "
+          + "--region=%s "
+          + "--worker-machine-type=n2-standard-8 --num-workers=8 "
+          + "--parameters registryEnvironment=%s "
+          + "--parameters sqlSnapshotId=%s "
+          + "--parameters latestCommitLogTimestamp=%s "
+          + "--parameters diffOutputGcsBucket=%s ";
+
+  // States indicating a job is not finished yet.
+  static final ImmutableSet<String> DATAFLOW_JOB_RUNNING_STATES =
+      ImmutableSet.of(
+          "JOB_STATE_UNKNOWN",
+          "JOB_STATE_RUNNING",
+          "JOB_STATE_STOPPED",
+          "JOB_STATE_PENDING",
+          "JOB_STATE_QUEUED");
+
+  static final Duration JOB_POLLING_INTERVAL = Duration.standardSeconds(60);
+
+  @Parameter(
+      names = {"-m", "--manual"},
+      description =
+          "If true, let user launch the comparison pipeline manually out of band. "
+              + "Command will wait for user key-press to exit after syncing Datastore.")
+  boolean manualLaunchPipeline;
+
+  @Parameter(
+      names = {"-r", "--release"},
+      description = "The release tag of the BEAM pipeline to run. It defaults to 'live'.")
+  String release = "live";
+
+  @Parameter(
+      names = {"-c", "--comparisonStartTimestamp"},
+      description =
+          "When comparing History and Epp Resource entities, ignore those that have not"
+              + " changed since this time.",
+      converter = DateTimeParameter.class)
+  DateTime comparisonStartTimestamp;
+
+  @Parameter(
+      names = {"-o", "--outputBucket"},
+      description =
+          "The GCS bucket where data discrepancies are logged. "
+              + "It defaults to ${projectId}-beam")
+  String outputBucket;
+
+  @Inject Clock clock;
+  @Inject Dataflow dataflow;
+
+  @Inject
+  @Config("defaultJobRegion")
+  String jobRegion;
+
+  @Inject
+  @Config("beamStagingBucketUrl")
+  String stagingBucketUrl;
+
+  @Inject
+  @Config("projectId")
+  String projectId;
+
+  @Inject Sleeper sleeper;
+
+  AppEngineConnection connection;
+
+  @Override
+  public void setConnection(AppEngineConnection connection) {
+    this.connection = connection;
+  }
+
+  String getDataflowJobStatus(String jobId) {
+    try {
+      return dataflow
+          .projects()
+          .locations()
+          .jobs()
+          .get(projectId, jobRegion, jobId)
+          .execute()
+          .getCurrentState();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  void launchPipelineAndWaitUntilFinish(
+      String pipelineName, DatabaseSnapshot snapshot, String latestCommitTimestamp) {
+    Job pipelineJob =
+        launchComparisonPipeline(pipelineName, snapshot.getSnapshotId(), latestCommitTimestamp)
+            .getJob();
+    String jobId = pipelineJob.getId();
+
+    System.out.printf("Launched comparison pipeline %s (%s).\n", pipelineJob.getName(), jobId);
+
+    while (DATAFLOW_JOB_RUNNING_STATES.contains(getDataflowJobStatus(jobId))) {
+      sleeper.sleepInterruptibly(JOB_POLLING_INTERVAL);
+    }
+    System.out.printf(
+        "Pipeline ended with %s state. Please check counters for results.\n",
+        getDataflowJobStatus(jobId));
+  }
+
+  String getOutputBucket() {
+    return Optional.ofNullable(outputBucket).orElse(projectId + "-beam");
+  }
+
+  String getContainerSpecGcsPath() {
+    return String.format(
+        "%s/%s_metadata.json", stagingBucketUrl.replace("live", release), PIPELINE_NAME);
+  }
+
+  String getManualLaunchCommand(
+      String jobName, String snapshotId, String latestCommitLogTimestamp) {
+    String baseCommand =
+        String.format(
+            MANUAL_PIPELINE_LAUNCH_COMMAND_TEMPLATE,
+            jobName,
+            getContainerSpecGcsPath(),
+            projectId,
+            jobRegion,
+            RegistryToolEnvironment.get().name(),
+            snapshotId,
+            latestCommitLogTimestamp,
+            getOutputBucket());
+    if (comparisonStartTimestamp == null) {
+      return baseCommand;
+    }
+    return baseCommand + "--parameters comparisonStartTimestamp=" + comparisonStartTimestamp;
+  }
+
+  LaunchFlexTemplateResponse launchComparisonPipeline(
+      String jobName, String sqlSnapshotId, String latestCommitLogTimestamp) {
+    try {
+      // Hardcode machine type and initial workers to force a quick start.
+      ImmutableMap.Builder<String, String> paramsBuilder =
+          new ImmutableMap.Builder()
+              .put("workerMachineType", "n2-standard-8")
+              .put("numWorkers", "8")
+              .put("sqlSnapshotId", sqlSnapshotId)
+              .put("latestCommitLogTimestamp", latestCommitLogTimestamp)
+              .put("registryEnvironment", RegistryToolEnvironment.get().name())
+              .put("diffOutputGcsBucket", getOutputBucket());
+      if (comparisonStartTimestamp != null) {
+        paramsBuilder.put("comparisonStartTimestamp", comparisonStartTimestamp.toString());
+      }
+      LaunchFlexTemplateParameter parameter =
+          new LaunchFlexTemplateParameter()
+              .setJobName(createJobName(Ascii.toLowerCase(jobName).replace('_', '-'), clock))
+              .setContainerSpecGcsPath(getContainerSpecGcsPath())
+              .setParameters(paramsBuilder.build());
+      return dataflow
+          .projects()
+          .locations()
+          .flexTemplates()
+          .launch(
+              projectId, jobRegion, new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
+          .execute();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * A fake implementation of {@link RequestStatusChecker} for managing SQL-backed locks from
+   * non-AppEngine platforms. This is only required until the Nomulus server is migrated off
+   * AppEngine.
+   */
+  static class FakeRequestStatusChecker implements RequestStatusChecker {
+
+    private final String logId =
+        ValidateDatastoreCommand.class.getSimpleName() + "-" + UUID.randomUUID();
+
+    @Override
+    public String getLogId() {
+      return logId;
+    }
+
+    @Override
+    public boolean isRunning(String requestLogId) {
+      return false;
+    }
+  }
+}

core/src/main/java/google/registry/tools/ValidateDatastoreCommand.java

@@ -0,0 +1,105 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.model.replay.ReplicateToDatastoreAction.REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH;
+import static google.registry.model.replay.ReplicateToDatastoreAction.REPLICATE_TO_DATASTORE_LOCK_NAME;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.net.MediaType;
+import google.registry.backup.SyncDatastoreToSqlSnapshotAction;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.server.Lock;
+import google.registry.request.Action.Service;
+import java.util.Optional;
+
+/**
+ * Validates asynchronously replicated data from the primary Cloud SQL database to Datastore.
+ *
+ * <p>This command suspends the replication process (by acquiring the replication lock), take a
+ * snapshot of the Cloud SQL database, invokes a Nomulus server action to sync Datastore to this
+ * snapshot (See {@link SyncDatastoreToSqlSnapshotAction} for details), and finally launches a BEAM
+ * pipeline to compare Datastore with the given SQL snapshot.
+ *
+ * <p>This command does not lock up the SQL database. Normal processing can proceed.
+ */
+@Parameters(commandDescription = "Validates Datastore with the primary Cloud SQL database.")
+public class ValidateDatastoreCommand extends ValidateDatabaseMigrationCommand {
+
+  private static final Service NOMULUS_SERVICE = Service.BACKEND;
+
+  @Override
+  public void run() throws Exception {
+    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc());
+    if (!state.getReplayDirection().equals(ReplayDirection.SQL_TO_DATASTORE)) {
+      throw new IllegalStateException("Cannot validate Datastore in migration step " + state);
+    }
+    Optional<Lock> lock =
+        Lock.acquireSql(
+            REPLICATE_TO_DATASTORE_LOCK_NAME,
+            null,
+            REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH,
+            new FakeRequestStatusChecker(),
+            false);
+    if (!lock.isPresent()) {
+      throw new IllegalStateException("Cannot acquire the async propagation lock.");
+    }
+
+    try {
+      try (DatabaseSnapshot snapshot = DatabaseSnapshot.createSnapshot()) {
+        System.out.printf("Obtained snapshot %s\n", snapshot.getSnapshotId());
+        AppEngineConnection connectionToService = connection.withService(NOMULUS_SERVICE);
+        String response =
+            connectionToService.sendPostRequest(
+                getNomulusEndpoint(snapshot.getSnapshotId()),
+                ImmutableMap.<String, String>of(),
+                MediaType.PLAIN_TEXT_UTF_8,
+                "".getBytes(UTF_8));
+        System.out.println(response);
+
+        lock.ifPresent(Lock::releaseSql);
+        lock = Optional.empty();
+
+        // See SyncDatastoreToSqlSnapshotAction for response format.
+        String latestCommitTimestamp =
+            response.substring(response.lastIndexOf('(') + 1, response.lastIndexOf(')'));
+
+        if (manualLaunchPipeline) {
+          System.out.printf(
+              "To launch the pipeline manually, use the following command:\n%s\n",
+              getManualLaunchCommand(
+                  "validate-datastore", snapshot.getSnapshotId(), latestCommitTimestamp));
+
+          System.out.print("\nEnter any key to continue when the pipeline ends:");
+          System.in.read();
+        } else {
+          launchPipelineAndWaitUntilFinish("validate-datastore", snapshot, latestCommitTimestamp);
+        }
+      }
+    } finally {
+      lock.ifPresent(Lock::releaseSql);
+    }
+  }
+
+  private static String getNomulusEndpoint(String sqlSnapshotId) {
+    return String.format(
+        "%s?sqlSnapshotId=%s", SyncDatastoreToSqlSnapshotAction.PATH, sqlSnapshotId);
+  }
+}

core/src/main/java/google/registry/tools/ValidateDatastoreWithSqlCommand.java

@@ -1,229 +0,0 @@
-// Copyright 2022 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools;
-
-import static google.registry.beam.BeamUtils.createJobName;
-import static google.registry.model.replay.ReplicateToDatastoreAction.REPLICATE_TO_DATASTORE_LOCK_NAME;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.api.services.dataflow.Dataflow;
-import com.google.api.services.dataflow.model.Job;
-import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
-import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
-import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.net.MediaType;
-import google.registry.backup.SyncDatastoreToSqlSnapshotAction;
-import google.registry.beam.common.DatabaseSnapshot;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.model.common.DatabaseMigrationStateSchedule;
-import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
-import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
-import google.registry.model.replay.ReplicateToDatastoreAction;
-import google.registry.model.server.Lock;
-import google.registry.request.Action.Service;
-import google.registry.util.Clock;
-import google.registry.util.RequestStatusChecker;
-import google.registry.util.Sleeper;
-import java.io.IOException;
-import java.util.Optional;
-import java.util.UUID;
-import javax.inject.Inject;
-import org.joda.time.Duration;
-
-/**
- * Validates asynchronously replicated data from the primary Cloud SQL database to Datastore.
- *
- * <p>This command suspends the replication process (by acquiring the replication lock), take a
- * snapshot of the Cloud SQL database, invokes a Nomulus server action to sync Datastore to this
- * snapshot (See {@link SyncDatastoreToSqlSnapshotAction} for details), and finally launches a BEAM
- * pipeline to compare Datastore with the given SQL snapshot.
- *
- * <p>This command does not lock up the SQL database. Normal processing can proceed.
- */
-@Parameters(commandDescription = "Validates Datastore with Cloud SQL.")
-public class ValidateDatastoreWithSqlCommand
-    implements CommandWithConnection, CommandWithRemoteApi {
-
-  private static final Service NOMULUS_SERVICE = Service.BACKEND;
-
-  private static final String PIPELINE_NAME = "validate_datastore_pipeline";
-
-  // States indicating a job is not finished yet.
-  private static final ImmutableSet<String> DATAFLOW_JOB_RUNNING_STATES =
-      ImmutableSet.of(
-          "JOB_STATE_RUNNING", "JOB_STATE_STOPPED", "JOB_STATE_PENDING", "JOB_STATE_QUEUED");
-
-  private static final Duration JOB_POLLING_INTERVAL = Duration.standardSeconds(60);
-
-  @Parameter(
-      names = {"-m", "--manual"},
-      description =
-          "If true, let user launch the comparison pipeline manually out of band. "
-              + "Command will wait for user key-press to exit after syncing Datastore.")
-  boolean manualLaunchPipeline;
-
-  @Inject Clock clock;
-  @Inject Dataflow dataflow;
-
-  @Inject
-  @Config("defaultJobRegion")
-  String jobRegion;
-
-  @Inject
-  @Config("beamStagingBucketUrl")
-  String stagingBucketUrl;
-
-  @Inject
-  @Config("projectId")
-  String projectId;
-
-  @Inject Sleeper sleeper;
-
-  private AppEngineConnection connection;
-
-  @Override
-  public void setConnection(AppEngineConnection connection) {
-    this.connection = connection;
-  }
-
-  @Override
-  public void run() throws Exception {
-    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc());
-    if (!state.getReplayDirection().equals(ReplayDirection.SQL_TO_DATASTORE)) {
-      throw new IllegalStateException("Cannot sync Datastore to SQL in migration step " + state);
-    }
-    Optional<Lock> lock =
-        Lock.acquireSql(
-            REPLICATE_TO_DATASTORE_LOCK_NAME,
-            null,
-            ReplicateToDatastoreAction.REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH,
-            new FakeRequestStatusChecker(),
-            false);
-    if (!lock.isPresent()) {
-      throw new IllegalStateException("Cannot acquire the async propagation lock.");
-    }
-
-    try {
-      try (DatabaseSnapshot snapshot = DatabaseSnapshot.createSnapshot()) {
-        System.out.printf("Obtained snapshot %s\n", snapshot.getSnapshotId());
-        AppEngineConnection connectionToService = connection.withService(NOMULUS_SERVICE);
-        String response =
-            connectionToService.sendPostRequest(
-                getNomulusEndpoint(snapshot.getSnapshotId()),
-                ImmutableMap.<String, String>of(),
-                MediaType.PLAIN_TEXT_UTF_8,
-                "".getBytes(UTF_8));
-        System.out.println(response);
-
-        lock.ifPresent(Lock::releaseSql);
-        lock = Optional.empty();
-
-        // See SyncDatastoreToSqlSnapshotAction for response format.
-        String latestCommitTimestamp =
-            response.substring(response.lastIndexOf('(') + 1, response.lastIndexOf(')'));
-
-        if (manualLaunchPipeline) {
-          System.out.print("\nEnter any key to continue when the pipeline ends:");
-          System.in.read();
-        } else {
-          Job pipelineJob =
-              launchComparisonPipeline(snapshot.getSnapshotId(), latestCommitTimestamp).getJob();
-          String jobId = pipelineJob.getId();
-
-          System.out.printf(
-              "Launched comparison pipeline %s (%s).\n", pipelineJob.getName(), jobId);
-
-          while (DATAFLOW_JOB_RUNNING_STATES.contains(getDataflowJobStatus(jobId))) {
-            sleeper.sleepInterruptibly(JOB_POLLING_INTERVAL);
-          }
-          System.out.printf(
-              "Pipeline ended with %s state. Please check counters for results.\n",
-              getDataflowJobStatus(jobId));
-        }
-      }
-    } finally {
-      lock.ifPresent(Lock::releaseSql);
-    }
-  }
-
-  private static String getNomulusEndpoint(String sqlSnapshotId) {
-    return String.format(
-        "%s?sqlSnapshotId=%s", SyncDatastoreToSqlSnapshotAction.PATH, sqlSnapshotId);
-  }
-
-  private LaunchFlexTemplateResponse launchComparisonPipeline(
-      String sqlSnapshotId, String latestCommitLogTimestamp) {
-    try {
-      LaunchFlexTemplateParameter parameter =
-          new LaunchFlexTemplateParameter()
-              .setJobName(createJobName("validate-datastore", clock))
-              .setContainerSpecGcsPath(
-                  String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-              .setParameters(
-                  ImmutableMap.of(
-                      "sqlSnapshotId",
-                      sqlSnapshotId,
-                      "latestCommitLogTimestamp",
-                      latestCommitLogTimestamp,
-                      "registryEnvironment",
-                      RegistryToolEnvironment.get().name()));
-      return dataflow
-          .projects()
-          .locations()
-          .flexTemplates()
-          .launch(
-              projectId, jobRegion, new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
-          .execute();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  private String getDataflowJobStatus(String jobId) {
-    try {
-      return dataflow
-          .projects()
-          .locations()
-          .jobs()
-          .get(projectId, jobRegion, jobId)
-          .execute()
-          .getCurrentState();
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  /**
-   * A fake implementation of {@link RequestStatusChecker} for managing SQL-backed locks from
-   * non-AppEngine platforms. This is only required until the Nomulus server is migrated off
-   * AppEngine.
-   */
-  static class FakeRequestStatusChecker implements RequestStatusChecker {
-
-    @Override
-    public String getLogId() {
-      return ValidateDatastoreWithSqlCommand.class.getSimpleName() + "-" + UUID.randomUUID();
-    }
-
-    @Override
-    public boolean isRunning(String requestLogId) {
-      return false;
-    }
-  }
-}

core/src/main/java/google/registry/tools/ValidateSqlCommand.java

@@ -0,0 +1,91 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.backup.ReplayCommitLogsToSqlAction.REPLAY_TO_SQL_LOCK_LEASE_LENGTH;
+import static google.registry.backup.ReplayCommitLogsToSqlAction.REPLAY_TO_SQL_LOCK_NAME;
+
+import com.beust.jcommander.Parameters;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.model.server.Lock;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import java.util.Optional;
+import org.joda.time.DateTime;
+
+/**
+ * Validates asynchronously replicated data from the primary Datastore to Cloud SQL.
+ *
+ * <p>This command suspends the replication process (by acquiring the replication lock), take a
+ * snapshot of the Cloud SQL database, finds the corresponding Datastore snapshot, and finally
+ * launches a BEAM pipeline to compare the two snapshots.
+ *
+ * <p>This command does not lock up either database. Normal processing can proceed.
+ */
+@Parameters(commandDescription = "Validates Cloud SQL with the primary Datastore.")
+public class ValidateSqlCommand extends ValidateDatabaseMigrationCommand {
+
+  @Override
+  public void run() throws Exception {
+    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc());
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      throw new IllegalStateException("Cannot validate SQL in migration step " + state);
+    }
+    Optional<Lock> lock =
+        Lock.acquireSql(
+            REPLAY_TO_SQL_LOCK_NAME,
+            null,
+            REPLAY_TO_SQL_LOCK_LEASE_LENGTH,
+            new FakeRequestStatusChecker(),
+            false);
+    if (!lock.isPresent()) {
+      throw new IllegalStateException("Cannot acquire the async propagation lock.");
+    }
+
+    try {
+      DateTime latestCommitLogTime =
+          TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+      try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+        // Eagerly release the commitlog replay lock so that replay can resume.
+        lock.ifPresent(Lock::releaseSql);
+        lock = Optional.empty();
+
+        System.out.printf(
+            "Start comparison with SQL snapshot (%s) and CommitLog timestamp (%s).\n",
+            databaseSnapshot.getSnapshotId(), latestCommitLogTime);
+
+        if (manualLaunchPipeline) {
+          System.out.printf(
+              "To launch the pipeline manually, use the following command:\n%s\n",
+              getManualLaunchCommand(
+                  "validate-sql",
+                  databaseSnapshot.getSnapshotId(),
+                  latestCommitLogTime.toString()));
+
+          System.out.print("\nEnter any key to continue when the pipeline ends:");
+          System.in.read();
+        } else {
+          launchPipelineAndWaitUntilFinish(
+              "validate-sql", databaseSnapshot, latestCommitLogTime.toString());
+        }
+      }
+    } finally {
+      lock.ifPresent(Lock::releaseSql);
+    }
+  }
+}

core/src/main/resources/google/registry/beam/validate_database_pipeline_metadata.json

@@ -1,6 +1,6 @@
 {
-  "name": "Validate Datastore with Cloud SQL",
-  "description": "An Apache Beam batch pipeline that compares Datastore with the primary Cloud SQL database.",
+  "name": "Validate Datastore and Cloud SQL",
+  "description": "An Apache Beam batch pipeline that compares the data in Datastore and Cloud SQL database.",
   "parameters": [
     {
       "name": "registryEnvironment",
@@ -24,7 +24,7 @@
       "name": "sqlSnapshotId",
       "label": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
       "helpText": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
-      "is_optional": true
+      "is_optional": false
     },
     {
       "name": "latestCommitLogTimestamp",
@@ -37,6 +37,12 @@
       "label": "Only entities updated at or after this time are included for validation.",
       "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
       "is_optional": true
+    },
+    {
+      "name": "diffOutputGcsBucket",
+      "label": "The GCS bucket where data discrepancies should be output to.",
+      "helpText": "The GCS bucket where data discrepancies should be output to.",
+      "is_optional": true
     }
   ]
 }

core/src/main/resources/google/registry/beam/validate_sql_pipeline_metadata.json

@@ -1,21 +0,0 @@
-{
-  "name": "Validate Cloud SQL with Datastore being primary",
-  "description": "An Apache Beam batch pipeline that compares Cloud SQL with the primary Datastore.",
-  "parameters": [
-    {
-      "name": "registryEnvironment",
-      "label": "The Registry environment.",
-      "helpText": "The Registry environment.",
-      "is_optional": false,
-      "regexes": [
-        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
-      ]
-    },
-    {
-      "name": "comparisonStartTimestamp",
-      "label": "Only entities updated at or after this time are included for validation.",
-      "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
-      "is_optional": true
-    }
-  ]
-}

core/src/test/java/google/registry/batch/AsyncTaskEnqueuerTest.java

@@ -15,7 +15,6 @@
 package google.registry.batch;
 
 import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.common.truth.Truth.assertThat;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
@@ -23,26 +22,20 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
-import static google.registry.testing.SqlHelper.saveRegistryLock;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestLogHandlerUtils.assertLogMessage;
-import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardHours;
 import static org.joda.time.Duration.standardSeconds;
-import static org.junit.Assert.assertThrows;
-import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSortedSet;
 import google.registry.model.contact.ContactResource;
-import google.registry.model.domain.RegistryLock;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.CapturingLogHandler;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.JdkLoggerConfig;
 import google.registry.util.Retrier;
 import java.util.logging.Level;
@@ -52,7 +45,6 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
@@ -67,27 +59,25 @@ public class AsyncTaskEnqueuerTest {
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  @Mock private AppEngineServiceUtils appEngineServiceUtils;
-
   private AsyncTaskEnqueuer asyncTaskEnqueuer;
   private final CapturingLogHandler logHandler = new CapturingLogHandler();
   private final FakeClock clock = new FakeClock(DateTime.parse("2015-05-18T12:34:56Z"));
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @BeforeEach
   void beforeEach() {
     JdkLoggerConfig.getConfig(AsyncTaskEnqueuer.class).addHandler(logHandler);
-    when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
-    asyncTaskEnqueuer = createForTesting(appEngineServiceUtils, clock, standardSeconds(90));
+    asyncTaskEnqueuer =
+        createForTesting(cloudTasksHelper.getTestCloudTasksUtils(), clock, standardSeconds(90));
   }
 
   public static AsyncTaskEnqueuer createForTesting(
-      AppEngineServiceUtils appEngineServiceUtils, FakeClock clock, Duration asyncDeleteDelay) {
+      CloudTasksUtils cloudTasksUtils, FakeClock clock, Duration asyncDeleteDelay) {
     return new AsyncTaskEnqueuer(
-        getQueue(QUEUE_ASYNC_ACTIONS),
         getQueue(QUEUE_ASYNC_DELETE),
         getQueue(QUEUE_ASYNC_HOST_RENAME),
         asyncDeleteDelay,
-        appEngineServiceUtils,
+        cloudTasksUtils,
         new Retrier(new FakeSleeper(clock), 1));
   }
 
@@ -96,18 +86,16 @@ public class AsyncTaskEnqueuerTest {
     ContactResource contact = persistActiveContact("jd23456");
     asyncTaskEnqueuer.enqueueAsyncResave(
         contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(5));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
-        new TaskMatcher()
+        new CloudTasksHelper.TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
-            .etaDelta(
-                standardDays(5).minus(standardSeconds(30)),
-                standardDays(5).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(Duration.standardDays(5))));
   }
 
   @Test
@@ -118,19 +106,17 @@ public class AsyncTaskEnqueuerTest {
         contact.createVKey(),
         now,
         ImmutableSortedSet.of(now.plusHours(24), now.plusHours(50), now.plusHours(75)));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, contact.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, now.toString())
             .param(PARAM_RESAVE_TIMES, "2015-05-20T14:34:56.000Z,2015-05-21T15:34:56.000Z")
-            .etaDelta(
-                standardHours(24).minus(standardSeconds(30)),
-                standardHours(24).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(Duration.standardHours(24))));
   }
 
   @MockitoSettings(strictness = Strictness.LENIENT)
@@ -139,62 +125,7 @@ public class AsyncTaskEnqueuerTest {
     ContactResource contact = persistActiveContact("jd23456");
     asyncTaskEnqueuer.enqueueAsyncResave(
         contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(31));
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
     assertLogMessage(logHandler, Level.INFO, "Ignoring async re-save");
   }
-
-  @Test
-  void testEnqueueRelock() {
-    RegistryLock lock =
-        saveRegistryLock(
-            new RegistryLock.Builder()
-                .setLockCompletionTime(clock.nowUtc())
-                .setUnlockRequestTime(clock.nowUtc())
-                .setUnlockCompletionTime(clock.nowUtc())
-                .isSuperuser(false)
-                .setDomainName("example.tld")
-                .setRepoId("repoId")
-                .setRelockDuration(standardHours(6))
-                .setRegistrarId("TheRegistrar")
-                .setRegistrarPocId("someone@example.com")
-                .setVerificationCode("hi")
-                .build());
-    asyncTaskEnqueuer.enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
-    assertTasksEnqueued(
-        QUEUE_ASYNC_ACTIONS,
-        new TaskMatcher()
-            .url(RelockDomainAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
-            .param(
-                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
-                String.valueOf(lock.getRevisionId()))
-            .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, "0")
-            .etaDelta(
-                standardHours(6).minus(standardSeconds(30)),
-                standardHours(6).plus(standardSeconds(30))));
-  }
-
-  @MockitoSettings(strictness = Strictness.LENIENT)
-  @Test
-  void testFailure_enqueueRelock_noDuration() {
-    RegistryLock lockWithoutDuration =
-        saveRegistryLock(
-            new RegistryLock.Builder()
-                .isSuperuser(false)
-                .setDomainName("example.tld")
-                .setRepoId("repoId")
-                .setRegistrarId("TheRegistrar")
-                .setRegistrarPocId("someone@example.com")
-                .setVerificationCode("hi")
-                .build());
-    assertThat(
-        assertThrows(
-            IllegalArgumentException.class,
-            () -> asyncTaskEnqueuer.enqueueDomainRelock(lockWithoutDuration)))
-        .hasMessageThat()
-        .isEqualTo(
-            String.format(
-                "Lock with ID %s not configured for relock", lockWithoutDuration.getRevisionId()));
-  }
 }

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -91,13 +91,13 @@ import google.registry.model.transfer.ContactTransferData;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.mapreduce.MapreduceTestCase;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.Retrier;
 import google.registry.util.Sleeper;
@@ -148,7 +148,7 @@ public class DeleteContactsAndHostsActionTest
     inject.setStaticField(Ofy.class, "clock", clock);
     enqueuer =
         AsyncTaskEnqueuerTest.createForTesting(
-            mock(AppEngineServiceUtils.class), clock, Duration.ZERO);
+            new CloudTasksHelper(clock).getTestCloudTasksUtils(), clock, Duration.ZERO);
     AsyncTaskMetrics asyncTaskMetricsMock = mock(AsyncTaskMetrics.class);
     action = new DeleteContactsAndHostsAction();
     action.asyncTaskMetrics = asyncTaskMetricsMock;

core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java

@@ -39,7 +39,6 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.persistence.transaction.QueryComposer.Comparator;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DualDatabaseTest;
@@ -78,8 +77,7 @@ class DeleteExpiredDomainsActionTest {
     createTld("tld");
     EppController eppController =
         DaggerEppTestComponent.builder()
-            .fakesAndMocksModule(
-                FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock)))
+            .fakesAndMocksModule(FakesAndMocksModule.create(clock))
             .build()
             .startRequest()
             .eppController();

core/src/test/java/google/registry/batch/RefreshDnsOnHostRenameActionTest.java

@@ -49,6 +49,7 @@ import google.registry.batch.RefreshDnsOnHostRenameAction.RefreshDnsOnHostRename
 import google.registry.dns.DnsQueue;
 import google.registry.model.host.HostResource;
 import google.registry.model.server.Lock;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
@@ -59,7 +60,6 @@ import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import google.registry.testing.TestSqlOnly;
 import google.registry.testing.mapreduce.MapreduceTestCase;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.Retrier;
 import google.registry.util.Sleeper;
@@ -89,7 +89,7 @@ public class RefreshDnsOnHostRenameActionTest
     createTld("tld");
     enqueuer =
         AsyncTaskEnqueuerTest.createForTesting(
-            mock(AppEngineServiceUtils.class), clock, Duration.ZERO);
+            new CloudTasksHelper(clock).getTestCloudTasksUtils(), clock, Duration.ZERO);
     AsyncTaskMetrics asyncTaskMetricsMock = mock(AsyncTaskMetrics.class);
     action = new RefreshDnsOnHostRenameAction();
     action.asyncTaskMetrics = asyncTaskMetricsMock;

core/src/test/java/google/registry/batch/RelockDomainActionTest.java

@@ -28,27 +28,26 @@ import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getMostRecentVerifiedRegistryLockByRepoId;
 import static google.registry.testing.SqlHelper.getRegistryLockByVerificationCode;
 import static google.registry.testing.SqlHelper.saveRegistryLock;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.joda.time.Duration.standardSeconds;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.host.HostResource;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.UserInfo;
 import google.registry.tools.DomainLockUtils;
@@ -78,12 +77,12 @@ public class RelockDomainActionTest {
 
   private final FakeResponse response = new FakeResponse();
   private final FakeClock clock = new FakeClock(DateTime.parse("2015-05-18T12:34:56Z"));
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
   private final DomainLockUtils domainLockUtils =
       new DomainLockUtils(
           new DeterministicStringGenerator(Alphabets.BASE_58),
           "adminreg",
-          AsyncTaskEnqueuerTest.createForTesting(
-              mock(AppEngineServiceUtils.class), clock, Duration.ZERO));
+          cloudTasksHelper.getTestCloudTasksUtils());
 
   @RegisterExtension
   public final AppEngineExtension appEngineExtension =
@@ -96,7 +95,6 @@ public class RelockDomainActionTest {
   private DomainBase domain;
   private RegistryLock oldLock;
   @Mock private SendEmailService sendEmailService;
-  private AsyncTaskEnqueuer asyncTaskEnqueuer;
   private RelockDomainAction action;
 
   @BeforeEach
@@ -118,8 +116,6 @@ public class RelockDomainActionTest {
         .when(appEngineServiceUtils.getServiceHostname("backend"))
         .thenReturn("backend.hostname.fake");
 
-    asyncTaskEnqueuer =
-        AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, Duration.ZERO);
     action = createAction(oldLock.getRevisionId());
   }
 
@@ -158,7 +154,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -170,7 +166,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -180,7 +176,7 @@ public class RelockDomainActionTest {
     assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
     assertThat(response.getPayload())
         .isEqualTo("Domain example.tld is already manually re-locked, skipping automated re-lock.");
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -192,7 +188,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -207,7 +203,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -253,7 +249,7 @@ public class RelockDomainActionTest {
     assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
     assertThat(response.getPayload())
         .isEqualTo("Domain example.tld is already manually re-locked, skipping automated re-lock.");
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -320,17 +316,16 @@ public class RelockDomainActionTest {
   }
 
   private void assertTaskEnqueued(int numAttempts, long oldUnlockRevisionId, Duration duration) {
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(RelockDomainAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
             .param(
                 RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
                 String.valueOf(oldUnlockRevisionId))
             .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, String.valueOf(numAttempts))
-            .etaDelta(duration.minus(standardSeconds(30)), duration.plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(duration)));
   }
 
   private RelockDomainAction createAction(Long oldUnlockRevisionId) throws Exception {
@@ -349,7 +344,6 @@ public class RelockDomainActionTest {
         "support@example.com",
         sendEmailService,
         domainLockUtils,
-        response,
-        asyncTaskEnqueuer);
+        response);
   }
 }

core/src/test/java/google/registry/batch/ResaveEntityActionTest.java

@@ -25,12 +25,10 @@ import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static google.registry.testing.DatabaseHelper.persistDomainWithDependentResources;
 import static google.registry.testing.DatabaseHelper.persistDomainWithPendingTransfer;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardSeconds;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import google.registry.model.domain.DomainBase;
@@ -40,10 +38,11 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.Ofy;
 import google.registry.request.Response;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.AppEngineServiceUtils;
 import org.joda.time.DateTime;
@@ -71,13 +70,14 @@ public class ResaveEntityActionTest {
   @Mock private Response response;
   private final FakeClock clock = new FakeClock(DateTime.parse("2016-02-11T10:00:00Z"));
   private AsyncTaskEnqueuer asyncTaskEnqueuer;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @BeforeEach
   void beforeEach() {
     inject.setStaticField(Ofy.class, "clock", clock);
-    when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
     asyncTaskEnqueuer =
-        AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, Duration.ZERO);
+        AsyncTaskEnqueuerTest.createForTesting(
+            cloudTasksHelper.getTestCloudTasksUtils(), clock, Duration.ZERO);
     createTld("tld");
   }
 
@@ -143,17 +143,15 @@ public class ResaveEntityActionTest {
     DomainBase resavedDomain = loadByEntity(domain);
     assertThat(resavedDomain.getGracePeriods()).isEmpty();
 
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, resavedDomain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, requestedTime.toString())
-            .etaDelta(
-                standardDays(5).minus(standardSeconds(30)),
-                standardDays(5).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(standardDays(5))));
   }
 }

core/src/test/java/google/registry/flows/EppPointInTimeTest.java

@@ -72,7 +72,7 @@ class EppPointInTimeTest {
     SessionMetadata sessionMetadata = new HttpSessionMetadata(new FakeHttpSession());
     sessionMetadata.setRegistrarId("TheRegistrar");
     DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, EppMetric.builderForRequest(clock)))
+        .fakesAndMocksModule(FakesAndMocksModule.create(clock))
         .build()
         .startRequest()
         .flowComponentBuilder()

core/src/test/java/google/registry/flows/EppTestCase.java

@@ -224,12 +224,14 @@ public class EppTestCase {
     EppRequestHandler handler = new EppRequestHandler();
     FakeResponse response = new FakeResponse();
     handler.response = response;
-    eppMetricBuilder = EppMetric.builderForRequest(clock);
-    handler.eppController = DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, eppMetricBuilder))
-        .build()
-        .startRequest()
-        .eppController();
+    FakesAndMocksModule fakesAndMocksModule = FakesAndMocksModule.create(clock);
+    eppMetricBuilder = fakesAndMocksModule.getMetricBuilder();
+    handler.eppController =
+        DaggerEppTestComponent.builder()
+            .fakesAndMocksModule(fakesAndMocksModule)
+            .build()
+            .startRequest()
+            .eppController();
     handler.executeEpp(
         sessionMetadata,
         credentials,

core/src/test/java/google/registry/flows/EppTestComponent.java

@@ -15,8 +15,6 @@
 package google.registry.flows;
 
 import static org.joda.time.Duration.standardSeconds;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import dagger.Component;
 import dagger.Module;
@@ -33,12 +31,12 @@ import google.registry.flows.domain.DomainFlowTmchUtils;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.request.RequestScope;
 import google.registry.request.lock.LockHandler;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeLockHandler;
 import google.registry.testing.FakeSleeper;
 import google.registry.tmch.TmchCertificateAuthority;
 import google.registry.tmch.TmchXmlSignature;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.Clock;
 import google.registry.util.Sleeper;
 import javax.inject.Singleton;
@@ -60,35 +58,32 @@ public interface EppTestComponent {
     private EppMetric.Builder metricBuilder;
     private FakeClock clock;
     private FakeLockHandler lockHandler;
-    private AppEngineServiceUtils appEngineServiceUtils;
     private Sleeper sleeper;
+    private CloudTasksHelper cloudTasksHelper;
 
-    public static FakesAndMocksModule create() {
-      FakeClock clock = new FakeClock();
-      return create(clock, EppMetric.builderForRequest(clock));
+    public CloudTasksHelper getCloudTasksHelper() {
+      return cloudTasksHelper;
     }
 
-    public static FakesAndMocksModule create(FakeClock clock, EppMetric.Builder metricBuilder) {
-      return create(
-          clock,
-          metricBuilder,
-          new TmchXmlSignature(new TmchCertificateAuthority(TmchCaMode.PILOT, clock)));
+    public EppMetric.Builder getMetricBuilder() {
+      return metricBuilder;
     }
 
-    public static FakesAndMocksModule create(
-        FakeClock clock, EppMetric.Builder eppMetricBuilder, TmchXmlSignature tmchXmlSignature) {
+    public static FakesAndMocksModule create(FakeClock clock) {
       FakesAndMocksModule instance = new FakesAndMocksModule();
-      AppEngineServiceUtils appEngineServiceUtils = mock(AppEngineServiceUtils.class);
-      when(appEngineServiceUtils.getServiceHostname("backend")).thenReturn("backend.hostname.fake");
+      CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
       instance.asyncTaskEnqueuer =
-          AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, standardSeconds(90));
+          AsyncTaskEnqueuerTest.createForTesting(
+              cloudTasksHelper.getTestCloudTasksUtils(), clock, standardSeconds(90));
       instance.clock = clock;
-      instance.domainFlowTmchUtils = new DomainFlowTmchUtils(tmchXmlSignature);
-      instance.sleeper = new FakeSleeper(clock);
+      instance.domainFlowTmchUtils =
+          new DomainFlowTmchUtils(
+              new TmchXmlSignature(new TmchCertificateAuthority(TmchCaMode.PILOT, clock)));
+      instance.sleeper = new FakeSleeper(instance.clock);
       instance.dnsQueue = DnsQueue.create();
-      instance.metricBuilder = eppMetricBuilder;
-      instance.appEngineServiceUtils = appEngineServiceUtils;
+      instance.metricBuilder = EppMetric.builderForRequest(clock);
       instance.lockHandler = new FakeLockHandler(true);
+      instance.cloudTasksHelper = cloudTasksHelper;
       return instance;
     }
 
@@ -127,11 +122,6 @@ public interface EppTestComponent {
       return metricBuilder;
     }
 
-    @Provides
-    AppEngineServiceUtils provideAppEngineServiceUtils() {
-      return appEngineServiceUtils;
-    }
-
     @Provides
     Sleeper provideSleeper() {
       return sleeper;

core/src/test/java/google/registry/flows/FlowTestCase.java

@@ -33,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.common.collect.ObjectArrays;
-import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
 import google.registry.flows.EppTestComponent.FakesAndMocksModule;
 import google.registry.flows.picker.FlowPicker;
 import google.registry.model.billing.BillingEvent;
@@ -45,14 +44,13 @@ import google.registry.model.ofy.Ofy;
 import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.EppLoader;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeHttpSession;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TestDataHelper;
-import google.registry.tmch.TmchCertificateAuthority;
-import google.registry.tmch.TmchXmlSignature;
 import google.registry.util.TypeUtils.TypeInstantiator;
 import google.registry.xml.ValidationMode;
 import java.util.Arrays;
@@ -88,7 +86,7 @@ public abstract class FlowTestCase<F extends Flow> {
   protected FakeClock clock = new FakeClock(DateTime.now(UTC));
   protected TransportCredentials credentials = new PasswordOnlyTransportCredentials();
   protected EppRequestSource eppRequestSource = EppRequestSource.UNIT_TEST;
-  private TmchXmlSignature testTmchXmlSignature = null;
+  protected CloudTasksHelper cloudTasksHelper;
 
   private EppMetric.Builder eppMetricBuilder;
 
@@ -229,13 +227,12 @@ public abstract class FlowTestCase<F extends Flow> {
     // Assert that the xml triggers the flow we expect.
     assertThat(FlowPicker.getFlowClass(eppLoader.getEpp()))
         .isEqualTo(new TypeInstantiator<F>(getClass()){}.getExactType());
+
+    FakesAndMocksModule fakesAndMocksModule = FakesAndMocksModule.create(clock);
+    cloudTasksHelper = fakesAndMocksModule.getCloudTasksHelper();
     // Run the flow.
-    TmchXmlSignature tmchXmlSignature =
-        testTmchXmlSignature != null
-            ? testTmchXmlSignature
-            : new TmchXmlSignature(new TmchCertificateAuthority(tmchCaMode, clock));
     return DaggerEppTestComponent.builder()
-        .fakesAndMocksModule(FakesAndMocksModule.create(clock, eppMetricBuilder, tmchXmlSignature))
+        .fakesAndMocksModule(fakesAndMocksModule)
         .build()
         .startRequest()
         .flowComponentBuilder()
@@ -300,8 +297,6 @@ public abstract class FlowTestCase<F extends Flow> {
     return output;
   }
 
-  private TmchCaMode tmchCaMode = TmchCaMode.PILOT;
-
   public EppOutput dryRunFlowAssertResponse(String xml, String... ignoredPaths) throws Exception {
     List<Object> beforeEntities = DatabaseHelper.loadAllEntities();
     EppOutput output =

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -54,14 +54,13 @@ import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
 import static google.registry.testing.TaskQueueHelper.assertDnsTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.money.CurrencyUnit.USD;
 import static org.joda.time.Duration.standardDays;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -102,10 +101,10 @@ import google.registry.model.tld.Registry.TldType;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import java.util.Map;
@@ -314,17 +313,17 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
     clock.advanceOneMilli();
     runFlowAssertResponse(loadFile("domain_delete_response_pending.xml"));
     Duration when = standardDays(3);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .param(PARAM_RESAVE_TIMES, clock.nowUtc().plusDays(5).toString())
-            .etaDelta(when.minus(standardSeconds(30)), when.plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(when)));
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -43,12 +43,11 @@ import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
 import static google.registry.testing.HostResourceSubject.assertAboutHosts;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.money.CurrencyUnit.USD;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.base.Predicates;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -106,10 +105,10 @@ import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import java.util.Map;
@@ -514,18 +513,16 @@ class DomainTransferRequestFlowTest
     assertPollMessagesEmitted(expectedExpirationTime, implicitTransferTime);
     assertAboutDomainAfterAutomaticTransfer(
         expectedExpirationTime, implicitTransferTime, Period.create(1, Unit.YEARS));
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(ResaveEntityAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .header("content-type", "application/x-www-form-urlencoded")
             .param(PARAM_RESOURCE_KEY, domain.createVKey().stringify())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
-            .etaDelta(
-                registry.getAutomaticTransferLength().minus(standardSeconds(30)),
-                registry.getAutomaticTransferLength().plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(registry.getAutomaticTransferLength())));
   }
 
   private void doSuccessfulTest(

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -109,7 +109,6 @@ import google.registry.persistence.VKey;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
-import google.registry.testing.ReplayExtension.NoDatabaseCompare;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
 import java.util.Optional;
@@ -955,7 +954,6 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     assertThat(thrown).hasMessageThat().contains("(sh8013)");
   }
 
-  @NoDatabaseCompare
   @TestOfyAndSql
   void testFailure_addingDuplicateContact() throws Exception {
     persistReferencedEntities();
@@ -1284,7 +1282,6 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   }
 
   // Contacts mismatch.
-  @NoDatabaseCompare
   @TestOfyAndSql
   void testFailure_sameContactAddedAndRemoved() throws Exception {
     setEppInput("domain_update_add_remove_same_contact.xml");

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -75,6 +75,7 @@ public class DomainBaseTest extends EntityTestCase {
   private VKey<BillingEvent.OneTime> oneTimeBillKey;
   private VKey<BillingEvent.Recurring> recurringBillKey;
   private Key<HistoryEntry> historyEntryKey;
+  private VKey<ContactResource> contact1Key, contact2Key;
 
   @BeforeEach
   void setUp() {
@@ -88,14 +89,14 @@ public class DomainBaseTest extends EntityTestCase {
                     .setRepoId("1-COM")
                     .build())
             .createVKey();
-    VKey<ContactResource> contact1Key =
+    contact1Key =
         persistResource(
                 new ContactResource.Builder()
                     .setContactId("contact_id1")
                     .setRepoId("2-COM")
                     .build())
             .createVKey();
-    VKey<ContactResource> contact2Key =
+    contact2Key =
         persistResource(
                 new ContactResource.Builder()
                     .setContactId("contact_id2")
@@ -947,4 +948,61 @@ public class DomainBaseTest extends EntityTestCase {
       this.billingEventOneTime = billingEventOneTime;
     }
   }
+
+  @Test
+  void testContactFields() {
+    VKey<ContactResource> contact3Key =
+        persistResource(
+                new ContactResource.Builder()
+                    .setContactId("contact_id3")
+                    .setRepoId("4-COM")
+                    .build())
+            .createVKey();
+    VKey<ContactResource> contact4Key =
+        persistResource(
+                new ContactResource.Builder()
+                    .setContactId("contact_id4")
+                    .setRepoId("5-COM")
+                    .build())
+            .createVKey();
+
+    // Set all of the contacts.
+    domain.setContactFields(
+        ImmutableSet.of(
+            DesignatedContact.create(Type.REGISTRANT, contact1Key),
+            DesignatedContact.create(Type.ADMIN, contact2Key),
+            DesignatedContact.create(Type.BILLING, contact3Key),
+            DesignatedContact.create(Type.TECH, contact4Key)),
+        true);
+    assertThat(domain.getRegistrant()).isEqualTo(contact1Key);
+    assertThat(domain.getAdminContact()).isEqualTo(contact2Key);
+    assertThat(domain.getBillingContact()).isEqualTo(contact3Key);
+    assertThat(domain.getTechContact()).isEqualTo(contact4Key);
+
+    // Make sure everything gets nulled out.
+    domain.setContactFields(ImmutableSet.of(), true);
+    assertThat(domain.getRegistrant()).isNull();
+    assertThat(domain.getAdminContact()).isNull();
+    assertThat(domain.getBillingContact()).isNull();
+    assertThat(domain.getTechContact()).isNull();
+
+    // Make sure that changes don't affect the registrant unless requested.
+    domain.setContactFields(
+        ImmutableSet.of(
+            DesignatedContact.create(Type.REGISTRANT, contact1Key),
+            DesignatedContact.create(Type.ADMIN, contact2Key),
+            DesignatedContact.create(Type.BILLING, contact3Key),
+            DesignatedContact.create(Type.TECH, contact4Key)),
+        false);
+    assertThat(domain.getRegistrant()).isNull();
+    assertThat(domain.getAdminContact()).isEqualTo(contact2Key);
+    assertThat(domain.getBillingContact()).isEqualTo(contact3Key);
+    assertThat(domain.getTechContact()).isEqualTo(contact4Key);
+    domain = domain.asBuilder().setRegistrant(contact1Key).build();
+    domain.setContactFields(ImmutableSet.of(), false);
+    assertThat(domain.getRegistrant()).isEqualTo(contact1Key);
+    assertThat(domain.getAdminContact()).isNull();
+    assertThat(domain.getBillingContact()).isNull();
+    assertThat(domain.getTechContact()).isNull();
+  }
 }

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -21,16 +21,15 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.testing.DatabaseHelper.insertInDb;
 import static google.registry.testing.LogsSubject.assertAboutLogs;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.junit.Assert.assertThrows;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.testing.TestLogHandler;
 import com.google.common.truth.Truth8;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
@@ -63,6 +62,7 @@ import org.junitpioneer.jupiter.RetryingTest;
 public class ReplicateToDatastoreActionTest {
 
   private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ"));
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   @RegisterExtension
   public final AppEngineExtension appEngine =
@@ -203,41 +203,38 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @RetryingTest(4)
-  void testMissingTransactions() {
-    assumeTrue(ReplayExtension.replayTestsEnabled());
-
-    // Write a transaction (should have a transaction id of 1).
+  void testNoTransactionIdUpdate() {
+    // Create an object.
     TestObject foo = TestObject.create("foo");
     insertInDb(foo);
 
-    // Force the last transaction id back to -1 so that we look for transaction 0.
-    ofyTm().transact(() -> ofyTm().insert(new LastSqlTransaction(-1)));
+    // Fail during the transaction to delete it.
+    try {
+      jpaTm()
+          .transact(
+              () -> {
+                jpaTm().delete(foo.key());
+                // Explicitly save the transaction entity to force the id update.
+                jpaTm().insert(new TransactionEntity(new byte[] {1, 2, 3}));
+                throw new RuntimeException("fail!!!");
+              });
+    } catch (Exception e) {
+      logger.atInfo().log("Got expected exception.");
+    }
 
+    TestObject bar = TestObject.create("bar");
+    insertInDb(bar);
+
+    // Make sure we have only the expected transaction ids.
     List<TransactionEntity> txns = action.getTransactionBatchAtSnapshot();
-    assertThat(txns).hasSize(1);
-    assertThat(assertThrows(IllegalStateException.class, () -> applyTransaction(txns.get(0))))
-        .hasMessageThat()
-        .isEqualTo("Missing transaction: last txn id = -1, next available txn = 1");
-  }
+    assertThat(txns).hasSize(2);
+    for (TransactionEntity txn : txns) {
+      assertThat(txn.getId()).isNotEqualTo(2);
+      applyTransaction(txn);
+    }
 
-  @Test
-  void testMissingTransactions_fullTask() {
-    assumeTrue(ReplayExtension.replayTestsEnabled());
-
-    // Write a transaction (should have a transaction id of 1).
-    TestObject foo = TestObject.create("foo");
-    insertInDb(foo);
-
-    // Force the last transaction id back to -1 so that we look for transaction 0.
-    ofyTm().transact(() -> ofyTm().insert(new LastSqlTransaction(-1)));
-    action.run();
-    assertAboutLogs()
-        .that(logHandler)
-        .hasSevereLogWithCause(
-            new IllegalStateException(
-                "Missing transaction: last txn id = -1, next available txn = 1"));
-    assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
-    assertThat(response.getPayload()).isEqualTo("Errored out replaying files.");
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(foo.key()))).isEqualTo(foo);
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(bar.key()))).isEqualTo(bar);
   }
 
   @Test

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -14,7 +14,6 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@@ -29,20 +28,13 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
-import com.google.common.base.Ascii;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSource;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.common.Cursor;
@@ -57,20 +49,21 @@ import google.registry.testing.FakeClock;
 import google.registry.testing.FakeKeyringModule;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.Retrier;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
-import java.util.Map;
 import java.util.Optional;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
 
 /** Unit tests for {@link RdeReportAction}. */
 @DualDatabaseTest
@@ -89,20 +82,21 @@ public class RdeReportActionTest {
 
   private final FakeResponse response = new FakeResponse();
   private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
-  private final URLFetchService urlFetchService = mock(URLFetchService.class);
-  private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
-  private final HTTPResponse httpResponse = mock(HTTPResponse.class);
   private final PGPPublicKey encryptKey =
       new FakeKeyringModule().get().getRdeStagingEncryptionKey();
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
   private final BlobId reportFile =
       BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
 
   private RdeReportAction createAction() {
     RdeReporter reporter = new RdeReporter();
     reporter.reportUrlPrefix = "https://rde-report.example";
-    reporter.urlFetchService = urlFetchService;
     reporter.password = "foo";
+    reporter.urlConnectionService = urlConnectionService;
     reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
     RdeReportAction action = new RdeReportAction();
     action.gcsUtils = gcsUtils;
@@ -127,6 +121,7 @@ public class RdeReportActionTest {
         Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
     gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
   }
 
   @TestOfyAndSql
@@ -142,24 +137,22 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
-    assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
+    verify(httpUrlConnection).setRequestMethod("PUT");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -167,9 +160,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_withPrefix() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     RdeReportAction action = createAction();
     action.prefix = Optional.of("job-name/");
     gcsUtils.delete(reportFile);
@@ -182,16 +174,14 @@ public class RdeReportActionTest {
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
-    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
-    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
-    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
-    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
-    assertThat(headers)
-        .containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
+    verify(httpUrlConnection).setRequestMethod("PUT");
+    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
+    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
+    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
+    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
+    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -204,9 +194,8 @@ public class RdeReportActionTest {
     PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
     gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
   }
@@ -239,9 +228,8 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
-    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST);
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream());
     InternalServerErrorException thrown =
         assertThrows(
             InternalServerErrorException.class,
@@ -252,18 +240,17 @@ public class RdeReportActionTest {
   @TestOfyAndSql
   void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
     class ExpectedThrownException extends RuntimeException {}
-    when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
+    when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException());
     assertThrows(
         ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
   }
 
   @TestOfyAndSql
   void testRunWithLock_socketTimeout_doesRetry() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
-    when(urlFetchService.fetch(request.capture()))
+    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
+    when(httpUrlConnection.getResponseCode())
         .thenThrow(new SocketTimeoutException())
-        .thenReturn(httpResponse);
+        .thenReturn(SC_OK);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@@ -274,14 +261,6 @@ public class RdeReportActionTest {
     return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime();
   }
 
-  private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
-    ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
-    for (HTTPHeader header : headers) {
-      builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
-    }
-    return builder.build();
-  }
-
   private static XjcRdeReportReport parseReport(byte[] data) {
     try {
       return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));

core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java

@@ -1,4 +1,4 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.util;
+package google.registry.request;
 
 import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.MediaType.CSV_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.util.UrlFetchUtils.setPayloadMultipart;
+import static google.registry.request.UrlConnectionUtils.setPayloadMultipart;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
@@ -27,22 +27,19 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import google.registry.testing.AppEngineExtension;
+import java.io.ByteArrayOutputStream;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
+import javax.net.ssl.HttpsURLConnection;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockito.ArgumentCaptor;
 
-/** Unit tests for {@link UrlFetchUtils}. */
-class UrlFetchUtilsTest {
-
-  @RegisterExtension final AppEngineExtension appEngine = AppEngineExtension.builder().build();
+/** Tests for {@link UrlConnectionUtils}. */
+public class UrlConnectionUtilsTest {
 
   private final Random random = mock(Random.class);
 
@@ -58,25 +55,28 @@ class UrlFetchUtilsTest {
   }
 
   @Test
-  void testSetPayloadMultipart() {
-    HTTPRequest request = mock(HTTPRequest.class);
+  void testSetPayloadMultipart() throws Exception {
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
+    ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+    when(connection.getOutputStream()).thenReturn(connectionOutputStream);
     setPayloadMultipart(
-        request,
+        connection,
         "lol",
         "cat",
         CSV_UTF_8,
         "The nice people at the store say hello. ヘ(◕。◕ヘ)",
         random);
-    ArgumentCaptor<HTTPHeader> headerCaptor = ArgumentCaptor.forClass(HTTPHeader.class);
-    verify(request, times(2)).addHeader(headerCaptor.capture());
-    List<HTTPHeader> addedHeaders = headerCaptor.getAllValues();
-    assertThat(addedHeaders.get(0).getName()).isEqualTo(CONTENT_TYPE);
-    assertThat(addedHeaders.get(0).getValue())
-        .isEqualTo(
-            "multipart/form-data; "
-                + "boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"");
-    assertThat(addedHeaders.get(1).getName()).isEqualTo(CONTENT_LENGTH);
-    assertThat(addedHeaders.get(1).getValue()).isEqualTo("294");
+    ArgumentCaptor<String> keyCaptor = ArgumentCaptor.forClass(String.class);
+    ArgumentCaptor<String> valueCaptor = ArgumentCaptor.forClass(String.class);
+    verify(connection, times(2)).setRequestProperty(keyCaptor.capture(), valueCaptor.capture());
+    List<String> addedKeys = keyCaptor.getAllValues();
+    assertThat(addedKeys).containsExactly(CONTENT_TYPE, CONTENT_LENGTH);
+    List<String> addedValues = valueCaptor.getAllValues();
+    assertThat(addedValues)
+        .containsExactly(
+            "multipart/form-data;"
+                + " boundary=\"------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"",
+            "294");
     String payload =
         "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r\n"
             + "Content-Disposition: form-data; name=\"lol\"; filename=\"cat\"\r\n"
@@ -84,18 +84,20 @@ class UrlFetchUtilsTest {
             + "\r\n"
             + "The nice people at the store say hello. ヘ(◕。◕ヘ)\r\n"
             + "--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r\n";
-    verify(request).setPayload(payload.getBytes(UTF_8));
-    verifyNoMoreInteractions(request);
+    verify(connection).setDoOutput(true);
+    verify(connection).getOutputStream();
+    assertThat(connectionOutputStream.toByteArray()).isEqualTo(payload.getBytes(UTF_8));
+    verifyNoMoreInteractions(connection);
   }
 
   @Test
   void testSetPayloadMultipart_boundaryInPayload() {
-    HTTPRequest request = mock(HTTPRequest.class);
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
     String payload = "I screamed------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHHH";
     IllegalStateException thrown =
         assertThrows(
             IllegalStateException.class,
-            () -> setPayloadMultipart(request, "lol", "cat", CSV_UTF_8, payload, random));
+            () -> setPayloadMultipart(connection, "lol", "cat", CSV_UTF_8, payload, random));
     assertThat(thrown)
         .hasMessageThat()
         .contains(

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -472,8 +472,10 @@ public class DatabaseHelper {
 
   public static void allowRegistrarAccess(String registrarId, String tld) {
     Registrar registrar = loadRegistrar(registrarId);
-    persistResource(
-        registrar.asBuilder().setAllowedTlds(union(registrar.getAllowedTlds(), tld)).build());
+    if (!registrar.getAllowedTlds().contains(tld)) {
+      persistResource(
+          registrar.asBuilder().setAllowedTlds(union(registrar.getAllowedTlds(), tld)).build());
+    }
   }
 
   public static void disallowRegistrarAccess(String registrarId, String tld) {

core/src/test/java/google/registry/testing/FakeURLFetchService.java

@@ -1,46 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.collect.ImmutableList;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * A fake {@link URLFetchService} that serves constructed {@link HTTPResponse} objects from
- * a simple {@link Map} ({@link URL} to {@link HTTPResponse}) lookup.
- */
-public class FakeURLFetchService extends ForwardingURLFetchService {
-
-  private Map<URL, HTTPResponse> backingMap;
-
-  public FakeURLFetchService(Map<URL, HTTPResponse> backingMap) {
-    this.backingMap = backingMap;
-  }
-
-  @Override
-  public HTTPResponse fetch(HTTPRequest request) {
-    URL requestURL = request.getURL();
-    if (backingMap.containsKey(requestURL)) {
-      return backingMap.get(requestURL);
-    } else {
-      return new HTTPResponse(HttpURLConnection.HTTP_NOT_FOUND, null, null, ImmutableList.of());
-    }
-  }
-}

core/src/test/java/google/registry/testing/FakeUrlConnectionService.java

@@ -0,0 +1,46 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.testing;
+
+import static org.mockito.Mockito.when;
+
+import google.registry.request.UrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+/** A fake {@link UrlConnectionService} with a mocked HTTP connection for testing. */
+public class FakeUrlConnectionService implements UrlConnectionService {
+
+  private final HttpURLConnection mockConnection;
+  private final List<URL> connectedUrls;
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection) {
+    this(mockConnection, new ArrayList<>());
+  }
+
+  public FakeUrlConnectionService(HttpURLConnection mockConnection, List<URL> connectedUrls) {
+    this.mockConnection = mockConnection;
+    this.connectedUrls = connectedUrls;
+  }
+
+  @Override
+  public HttpURLConnection createConnection(URL url) {
+    connectedUrls.add(url);
+    when(mockConnection.getURL()).thenReturn(url);
+    return mockConnection;
+  }
+}

core/src/test/java/google/registry/testing/ForwardingURLFetchService.java

@@ -1,49 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.testing;
-
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
-import com.google.common.util.concurrent.Futures;
-import java.io.IOException;
-import java.net.URL;
-import java.util.concurrent.Future;
-
-/**
- * An implementation of the {@link URLFetchService} interface that forwards all requests through
- * a synchronous fetch call.
- */
-public abstract class ForwardingURLFetchService implements URLFetchService {
-
-  @Override
-  public HTTPResponse fetch(URL url) throws IOException {
-    return fetch(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(URL url) {
-    return fetchAsync(new HTTPRequest(url)); // Defaults to HTTPMethod.GET
-  }
-
-  @Override
-  public Future<HTTPResponse> fetchAsync(HTTPRequest request) {
-    try {
-      return Futures.immediateFuture(fetch(request));
-    } catch (Exception e) {
-      return Futures.immediateFailedFuture(e);
-    }
-  }
-}

core/src/test/java/google/registry/tmch/NordnUploadActionTest.java

@@ -19,21 +19,22 @@ import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.FORM_DATA;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistDomainAndEnqueueLordn;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
-import static java.nio.charset.StandardCharsets.US_ASCII;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.startsWith;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
@@ -43,10 +44,6 @@ import com.google.appengine.api.taskqueue.TaskHandle;
 import com.google.appengine.api.taskqueue.TaskOptions;
 import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.VerifyException;
 import com.google.common.collect.ImmutableList;
@@ -57,11 +54,15 @@ import google.registry.model.tld.Registry;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.util.Retrier;
 import google.registry.util.TaskQueueUtils;
-import google.registry.util.UrlFetchException;
+import google.registry.util.UrlConnectionException;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
 import java.util.List;
@@ -69,17 +70,9 @@ import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
-import org.mockito.junit.jupiter.MockitoSettings;
-import org.mockito.quality.Strictness;
 
 /** Unit tests for {@link NordnUploadAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnUploadActionTest {
 
   private static final String CLAIMS_CSV =
@@ -101,29 +94,30 @@ class NordnUploadActionTest {
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
-
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
+  
   private final FakeClock clock = new FakeClock(DateTime.parse("2010-05-01T10:11:12Z"));
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnUploadAction action = new NordnUploadAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
     inject.setStaticField(Ofy.class, "clock", clock);
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
-    when(httpResponse.getContent()).thenReturn("Success".getBytes(US_ASCII));
-    when(httpResponse.getResponseCode()).thenReturn(SC_ACCEPTED);
-    when(httpResponse.getHeadersUncombined())
-        .thenReturn(ImmutableList.of(new HTTPHeader(LOCATION, "http://trololol")));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream("Success".getBytes(UTF_8)));
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_ACCEPTED);
+    when(httpUrlConnection.getHeaderField(LOCATION)).thenReturn("http://trololol");
+    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
     persistResource(loadRegistrar("TheRegistrar").asBuilder().setIanaIdentifier(99999L).build());
     createTld("tld");
     persistResource(Registry.get("tld").asBuilder().setLordnUsername("lolcat").build());
     action.clock = clock;
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.phase = "claims";
     action.taskQueueUtils = new TaskQueueUtils(new Retrier(new FakeSleeper(clock), 3));
@@ -133,7 +127,6 @@ class NordnUploadActionTest {
     action.retrier = new Retrier(new FakeSleeper(clock), 3);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv() {
     List<TaskHandle> tasks =
@@ -145,7 +138,6 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_dedupesDuplicates() {
     List<TaskHandle> tasks =
@@ -158,14 +150,12 @@ class NordnUploadActionTest {
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_doesntFailOnEmptyTasks() {
     assertThat(NordnUploadAction.convertTasksToCsv(ImmutableList.of(), clock.nowUtc(), "col1,col2"))
         .isEqualTo("1,2010-05-01T10:11:12.000Z,0\ncol1,col2\n");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_throwsNpeOnNullTasks() {
     assertThrows(
@@ -173,7 +163,6 @@ class NordnUploadActionTest {
         () -> NordnUploadAction.convertTasksToCsv(null, clock.nowUtc(), "header"));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @SuppressWarnings("unchecked")
   @Test
   void test_loadAllTasks_retryLogic_thirdTrysTheCharm() {
@@ -186,7 +175,6 @@ class NordnUploadActionTest {
     assertThat(action.loadAllTasks(queue, "tld")).containsExactly(task);
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_loadAllTasks_retryLogic_allFailures() {
     Queue queue = mock(Queue.class);
@@ -201,47 +189,47 @@ class NordnUploadActionTest {
   void testRun_claimsMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/claims"));
   }
 
   @Test
   void testRun_sunriseMode_appendsTldAndClaimsToRequestUrl() throws Exception {
     persistSunriseModeDomain();
     action.run();
-    assertThat(getCapturedHttpRequest().getURL())
-        .isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/LORDN/tld/sunrise"));
   }
 
   @Test
   void testRun_usesMultipartContentType() throws Exception {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), CONTENT_TYPE).get())
-        .startsWith("multipart/form-data; boundary=");
+    verify(httpUrlConnection)
+        .setRequestProperty(eq(CONTENT_TYPE), startsWith("multipart/form-data; boundary="));
+    verify(httpUrlConnection).setRequestMethod("POST");
   }
 
   @Test
-  void testRun_hasPassword_setsAuthorizationHeader() throws Exception {
+  void testRun_hasPassword_setsAuthorizationHeader() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+    verify(httpUrlConnection)
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testRun_noPassword_doesntSendAuthorizationHeader() throws Exception {
+  void testRun_noPassword_doesntSendAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     persistClaimsModeDomain();
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
-  void testRun_claimsMode_payloadMatchesClaimsCsv() throws Exception {
+  void testRun_claimsMode_payloadMatchesClaimsCsv() {
     persistClaimsModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(CLAIMS_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(CLAIMS_CSV);
   }
 
   @Test
@@ -257,19 +245,19 @@ class NordnUploadActionTest {
   }
 
   @Test
-  void testRun_sunriseMode_payloadMatchesSunriseCsv() throws Exception {
+  void testRun_sunriseMode_payloadMatchesSunriseCsv() {
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
   void test_noResponseContent_stillWorksNormally() throws Exception {
     // Returning null only affects logging.
-    when(httpResponse.getContent()).thenReturn(null);
+    when(httpUrlConnection.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[] {}));
     persistSunriseModeDomain();
     action.run();
-    assertThat(new String(getCapturedHttpRequest().getPayload(), UTF_8)).contains(SUNRISE_CSV);
+    assertThat(new String(connectionOutputStream.toByteArray(), UTF_8)).contains(SUNRISE_CSV);
   }
 
   @Test
@@ -284,7 +272,6 @@ class NordnUploadActionTest {
             .header(CONTENT_TYPE, FORM_DATA.toString()));
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void testFailure_nullRegistryUser() {
     persistClaimsModeDomain();
@@ -293,17 +280,11 @@ class NordnUploadActionTest {
     assertThat(thrown).hasMessageThat().contains("lordnUsername is not set for tld.");
   }
 
-  @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
-  void testFetchFailure() {
+  void testFetchFailure() throws Exception {
     persistClaimsModeDomain();
-    when(httpResponse.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
-    assertThrows(UrlFetchException.class, action::run);
-  }
-
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
+    assertThrows(UrlConnectionException.class, action::run);
   }
 
   private void persistClaimsModeDomain() {

core/src/test/java/google/registry/tmch/NordnVerifyActionTest.java

@@ -16,39 +16,34 @@ package google.registry.tmch;
 
 import static com.google.common.net.HttpHeaders.AUTHORIZATION;
 import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.util.UrlFetchUtils.getHeaderFirst;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static javax.servlet.http.HttpServletResponse.SC_UNAUTHORIZED;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.model.tld.Registry;
 import google.registry.request.HttpException.ConflictException;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeResponse;
+import google.registry.testing.FakeUrlConnectionService;
+import java.io.ByteArrayInputStream;
+import java.net.HttpURLConnection;
 import java.net.URL;
 import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Unit tests for {@link NordnVerifyAction}. */
-@ExtendWith(MockitoExtension.class)
 class NordnVerifyActionTest {
 
   private static final String LOG_ACCEPTED =
@@ -81,84 +76,83 @@ class NordnVerifyActionTest {
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
-  @Mock private URLFetchService fetchService;
-  @Mock private HTTPResponse httpResponse;
-  @Captor private ArgumentCaptor<HTTPRequest> httpRequestCaptor;
-
   private final FakeResponse response = new FakeResponse();
   private final LordnRequestInitializer lordnRequestInitializer =
       new LordnRequestInitializer(Optional.of("attack"));
   private final NordnVerifyAction action = new NordnVerifyAction();
 
+  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  private final FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection);
+
   @BeforeEach
   void beforeEach() throws Exception {
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
-    when(httpResponse.getContent()).thenReturn(LOG_ACCEPTED.getBytes(UTF_8));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
     createTld("gtld");
     persistResource(Registry.get("gtld").asBuilder().setLordnUsername("lolcat").build());
     action.tld = "gtld";
-    action.fetchService = fetchService;
+    action.urlConnectionService = urlConnectionService;
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ACCEPTED.getBytes(UTF_8)));
     action.lordnRequestInitializer = lordnRequestInitializer;
     action.response = response;
     action.url = new URL("http://127.0.0.1/blobio");
   }
 
-  private HTTPRequest getCapturedHttpRequest() throws Exception {
-    verify(fetchService).fetch(httpRequestCaptor.capture());
-    return httpRequestCaptor.getAllValues().get(0);
-  }
-
   @Test
   void testSuccess_sendHttpRequest_urlIsCorrect() throws Exception {
     action.run();
-    assertThat(getCapturedHttpRequest().getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
+    assertThat(httpUrlConnection.getURL()).isEqualTo(new URL("http://127.0.0.1/blobio"));
   }
 
   @Test
-  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() throws Exception {
+  void testSuccess_hasLordnPassword_sendsAuthorizationHeader() {
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION))
-        .hasValue("Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
+    verify(httpUrlConnection)
+        .setRequestProperty(
+            AUTHORIZATION, "Basic bG9sY2F0OmF0dGFjaw=="); // echo -n lolcat:attack | base64
   }
 
   @Test
-  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() throws Exception {
+  void testSuccess_noLordnPassword_doesntSetAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
     action.run();
-    assertThat(getHeaderFirst(getCapturedHttpRequest(), AUTHORIZATION)).isEmpty();
+    verify(httpUrlConnection, times(0)).setRequestProperty(eq(AUTHORIZATION), anyString());
   }
 
   @Test
   void successVerifyRejected() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_REJECTED.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_REJECTED.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.getStatus()).isEqualTo(LordnLog.Status.REJECTED);
   }
 
   @Test
   void successVerifyWarnings() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_WARNINGS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_WARNINGS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
   void successVerifyErrors() throws Exception {
-    when(httpResponse.getContent()).thenReturn(LOG_ERRORS.getBytes(UTF_8));
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(LOG_ERRORS.getBytes(UTF_8)));
     LordnLog lastLog = action.verify();
     assertThat(lastLog.hasWarnings()).isTrue();
   }
 
   @Test
-  void failureVerifyUnauthorized() {
-    when(httpResponse.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
+  void failureVerifyUnauthorized() throws Exception {
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_UNAUTHORIZED);
     assertThrows(Exception.class, action::run);
   }
 
   @Test
-  void failureVerifyNotReady() {
-    when(httpResponse.getResponseCode()).thenReturn(SC_NO_CONTENT);
+  void failureVerifyNotReady() throws Exception {
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_NO_CONTENT);
     ConflictException thrown = assertThrows(ConflictException.class, action::run);
     assertThat(thrown).hasMessageThat().contains("Not ready");
   }

core/src/test/java/google/registry/tmch/TmchActionTestCase.java

@@ -15,21 +15,19 @@
 package google.registry.tmch;
 
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-import com.google.appengine.api.urlfetch.URLFetchService;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.FakeClock;
+import google.registry.testing.FakeUrlConnectionService;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.extension.RegisterExtension;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
 /** Common code for unit tests of classes that extend {@link Marksdb}. */
@@ -46,19 +44,19 @@ abstract class TmchActionTestCase {
   @RegisterExtension
   public final BouncyCastleProviderExtension bouncy = new BouncyCastleProviderExtension();
 
-  @Mock URLFetchService fetchService;
-  @Mock HTTPResponse httpResponse;
-  @Captor ArgumentCaptor<HTTPRequest> httpRequest;
-
   final FakeClock clock = new FakeClock();
   final Marksdb marksdb = new Marksdb();
 
+  protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
+  protected final ArrayList<URL> connectedUrls = new ArrayList<>();
+  protected FakeUrlConnectionService urlConnectionService =
+      new FakeUrlConnectionService(httpUrlConnection, connectedUrls);
+
   @BeforeEach
   public void beforeEachTmchActionTestCase() throws Exception {
-    marksdb.fetchService = fetchService;
     marksdb.tmchMarksdbUrl = MARKSDB_URL;
     marksdb.marksdbPublicKey = TmchData.loadPublicKey(TmchTestData.loadBytes("pubkey"));
-    when(fetchService.fetch(any(HTTPRequest.class))).thenReturn(httpResponse);
-    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    marksdb.urlConnectionService = urlConnectionService;
+    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
   }
 }

core/src/test/java/google/registry/tmch/TmchCrlActionTest.java

@@ -22,6 +22,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.config.RegistryConfig.ConfigModule.TmchCaMode;
+import java.io.ByteArrayInputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.security.SignatureException;
@@ -44,19 +45,22 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testSuccess() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent()).thenReturn(
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     newTmchCrlAction(TmchCaMode.PRODUCTION).run();
-    verify(httpResponse).getContent();
-    verify(fetchService).fetch(httpRequest.capture());
-    assertThat(httpRequest.getValue().getURL().toString()).isEqualTo("http://sloth.lol/tmch.crl");
+    verify(httpUrlConnection).getInputStream();
+    assertThat(connectedUrls).containsExactly(new URL("http://sloth.lol/tmch.crl"));
   }
 
   @Test
   void testFailure_crlTooOld() throws Exception {
     clock.setTo(DateTime.parse("2020-01-01TZ"));
-    when(httpResponse.getContent())
-        .thenReturn(loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                loadBytes(TmchCrlActionTest.class, "icann-tmch-pilot-old.crl").read()));
     TmchCrlAction action = newTmchCrlAction(TmchCaMode.PILOT);
     Exception e = assertThrows(Exception.class, action::run);
     assertThat(e).hasCauseThat().isInstanceOf(CRLException.class);
@@ -69,8 +73,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotSignedByRoot() throws Exception {
     clock.setTo(DateTime.parse("2013-07-24TZ"));
-    when(httpResponse.getContent())
-        .thenReturn(readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(SignatureException.class);
     assertThat(e).hasCauseThat().hasMessageThat().isEqualTo("Signature does not match.");
@@ -79,8 +85,10 @@ class TmchCrlActionTest extends TmchActionTestCase {
   @Test
   void testFailure_crlNotYetValid() throws Exception {
     clock.setTo(DateTime.parse("1984-01-01TZ"));
-    when(httpResponse.getContent()).thenReturn(
-        readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(
+            new ByteArrayInputStream(
+                readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()));
     Exception e = assertThrows(Exception.class, newTmchCrlAction(TmchCaMode.PILOT)::run);
     assertThat(e).hasCauseThat().isInstanceOf(CertificateNotYetValidException.class);
   }

core/src/test/java/google/registry/tmch/TmchDnlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static org.mockito.Mockito.times;
@@ -22,6 +23,8 @@ import static org.mockito.Mockito.when;
 
 import google.registry.model.tmch.ClaimsList;
 import google.registry.model.tmch.ClaimsListDao;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -39,15 +42,13 @@ class TmchDnlActionTest extends TmchActionTestCase {
   @Test
   void testDnl() throws Exception {
     assertThat(ClaimsListDao.get().getClaimKey("xn----7sbejwbn3axu3d")).isEmpty();
-    when(httpResponse.getContent())
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.csv").read())
-        .thenReturn(TmchTestData.loadBytes("dnl-latest.sig").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.csv").read()))
+        .thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl-latest.sig").read()));
     newTmchDnlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.csv");
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/dnl/dnl-latest.sig");
+    verify(httpUrlConnection, times(2)).getInputStream();
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
+        .containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig");
 
     // Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
     ClaimsList claimsList = ClaimsListDao.get();

core/src/test/java/google/registry/tmch/TmchSmdrlActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.tmch;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.tmch.TmchTestData.loadBytes;
 import static org.mockito.Mockito.times;
@@ -21,6 +22,8 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import google.registry.model.smd.SignedMarkRevocationList;
+import java.io.ByteArrayInputStream;
+import java.net.URL;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
@@ -42,15 +45,14 @@ class TmchSmdrlActionTest extends TmchActionTestCase {
     SignedMarkRevocationList smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isFalse();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();
-    when(httpResponse.getContent())
-        .thenReturn(loadBytes("smdrl-latest.csv").read())
-        .thenReturn(loadBytes("smdrl-latest.sig").read());
+    when(httpUrlConnection.getInputStream())
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.csv").read()))
+        .thenReturn(new ByteArrayInputStream(loadBytes("smdrl-latest.sig").read()));
     newTmchSmdrlAction().run();
-    verify(fetchService, times(2)).fetch(httpRequest.capture());
-    assertThat(httpRequest.getAllValues().get(0).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.csv");
-    assertThat(httpRequest.getAllValues().get(1).getURL().toString())
-        .isEqualTo(MARKSDB_URL + "/smdrl/smdrl-latest.sig");
+    verify(httpUrlConnection, times(2)).getInputStream();
+    assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
+        .containsExactly(
+            MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig");
     smdrl = SignedMarkRevocationList.get();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65535", now)).isTrue();
     assertThat(smdrl.isSmdRevoked("0000001681375789102250-65536", now)).isFalse();

core/src/test/java/google/registry/tools/DomainLockUtilsTest.java

@@ -26,17 +26,16 @@ import static google.registry.testing.DatabaseHelper.persistActiveHost;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getRegistryLockByRevisionId;
 import static google.registry.testing.SqlHelper.getRegistryLockByVerificationCode;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
+import static google.registry.testing.SqlHelper.saveRegistryLock;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardHours;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.batch.RelockDomainAction;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -47,12 +46,13 @@ import google.registry.model.host.HostResource;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.SqlHelper;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.UserInfo;
 import google.registry.util.AppEngineServiceUtils;
@@ -62,8 +62,11 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.junit.Assert;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
 
 /** Unit tests for {@link google.registry.tools.DomainLockUtils}. */
 @DualDatabaseTest
@@ -74,6 +77,7 @@ public final class DomainLockUtilsTest {
 
   private final FakeClock clock = new FakeClock(DateTime.now(DateTimeZone.UTC));
   private DomainLockUtils domainLockUtils;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @RegisterExtension
   public final AppEngineExtension appEngineExtension =
@@ -98,8 +102,7 @@ public final class DomainLockUtilsTest {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                appEngineServiceUtils, clock, standardSeconds(90)));
+            cloudTasksHelper.getTestCloudTasksUtils());
   }
 
   @TestOfyAndSql
@@ -265,19 +268,17 @@ public final class DomainLockUtilsTest {
         domainLockUtils.saveNewRegistryUnlockRequest(
             DOMAIN_NAME, "TheRegistrar", false, Optional.of(standardHours(6)));
     domainLockUtils.verifyAndApplyUnlock(lock.getVerificationCode(), false);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(RelockDomainAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .param(
                 RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
                 String.valueOf(lock.getRevisionId()))
             .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, "0")
-            .etaDelta(
-                standardHours(6).minus(standardSeconds(30)),
-                standardDays(6).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(lock.getRelockDuration().get())));
   }
 
   @TestOfyAndSql
@@ -477,6 +478,59 @@ public final class DomainLockUtilsTest {
     assertNoDomainChanges();
   }
 
+  @TestOfyAndSql
+  void testEnqueueRelock() {
+    RegistryLock lock =
+        saveRegistryLock(
+            new RegistryLock.Builder()
+                .setLockCompletionTime(clock.nowUtc())
+                .setUnlockRequestTime(clock.nowUtc())
+                .setUnlockCompletionTime(clock.nowUtc())
+                .isSuperuser(false)
+                .setDomainName("example.tld")
+                .setRepoId("repoId")
+                .setRelockDuration(standardHours(6))
+                .setRegistrarId("TheRegistrar")
+                .setRegistrarPocId("someone@example.com")
+                .setVerificationCode("hi")
+                .build());
+    domainLockUtils.enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
+    cloudTasksHelper.assertTasksEnqueued(
+        QUEUE_ASYNC_ACTIONS,
+        new CloudTasksHelper.TaskMatcher()
+            .url(RelockDomainAction.PATH)
+            .method(HttpMethod.POST)
+            .service("backend")
+            .param(
+                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
+                String.valueOf(lock.getRevisionId()))
+            .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, "0")
+            .scheduleTime(clock.nowUtc().plus(lock.getRelockDuration().get())));
+  }
+
+  @MockitoSettings(strictness = Strictness.LENIENT)
+  @TestOfyAndSql
+  void testFailure_enqueueRelock_noDuration() {
+    RegistryLock lockWithoutDuration =
+        saveRegistryLock(
+            new RegistryLock.Builder()
+                .isSuperuser(false)
+                .setDomainName("example.tld")
+                .setRepoId("repoId")
+                .setRegistrarId("TheRegistrar")
+                .setRegistrarPocId("someone@example.com")
+                .setVerificationCode("hi")
+                .build());
+    assertThat(
+            Assert.assertThrows(
+                IllegalArgumentException.class,
+                () -> domainLockUtils.enqueueDomainRelock(lockWithoutDuration)))
+        .hasMessageThat()
+        .isEqualTo(
+            String.format(
+                "Lock with ID %s not configured for relock", lockWithoutDuration.getRevisionId()));
+  }
+
   private void verifyProperlyLockedDomain(boolean isAdmin) {
     assertThat(loadByEntity(domain).getStatusValues())
         .containsAtLeastElementsIn(REGISTRY_LOCK_STATUSES);

core/src/test/java/google/registry/tools/LockDomainCommandTest.java

@@ -24,19 +24,16 @@ import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.mock;
 
 import com.google.common.collect.ImmutableList;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DeterministicStringGenerator;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.StringGenerator.Alphabets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
-import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -52,8 +49,7 @@ class LockDomainCommandTest extends CommandTestCase<LockDomainCommand> {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                mock(AppEngineServiceUtils.class), fakeClock, Duration.ZERO));
+            new CloudTasksHelper(fakeClock).getTestCloudTasksUtils());
   }
 
   @Test

core/src/test/java/google/registry/tools/UnlockDomainCommandTest.java

@@ -25,21 +25,18 @@ import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.mock;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DeterministicStringGenerator;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.StringGenerator.Alphabets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
-import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -55,8 +52,7 @@ class UnlockDomainCommandTest extends CommandTestCase<UnlockDomainCommand> {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                mock(AppEngineServiceUtils.class), fakeClock, Duration.ZERO));
+            new CloudTasksHelper(fakeClock).getTestCloudTasksUtils());
   }
 
   private DomainBase persistLockedDomain(String domainName, String registrarId) {

core/src/test/java/google/registry/ui/server/registrar/RegistryLockPostActionTest.java

@@ -26,7 +26,6 @@ import static google.registry.testing.SqlHelper.getRegistryLockByVerificationCod
 import static google.registry.testing.SqlHelper.saveRegistryLock;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static google.registry.ui.server.registrar.RegistryLockGetActionTest.userFromRegistrarContact;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
@@ -35,7 +34,6 @@ import com.google.appengine.api.users.User;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.RegistryLock;
 import google.registry.request.JsonActionRunner;
@@ -47,10 +45,10 @@ import google.registry.request.auth.AuthenticatedRegistrarAccessor;
 import google.registry.request.auth.AuthenticatedRegistrarAccessor.Role;
 import google.registry.request.auth.UserAuthInfo;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.FakeClock;
 import google.registry.tools.DomainLockUtils;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import google.registry.util.StringGenerator.Alphabets;
@@ -469,8 +467,7 @@ final class RegistryLockPostActionTest {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                mock(AppEngineServiceUtils.class), clock, Duration.ZERO));
+            new CloudTasksHelper(clock).getTestCloudTasksUtils());
     return new RegistryLockPostAction(
         mockRequest,
         jsonActionRunner,

core/src/test/java/google/registry/ui/server/registrar/RegistryLockVerifyActionTest.java

@@ -33,7 +33,6 @@ import com.google.appengine.api.users.User;
 import com.google.appengine.api.users.UserService;
 import com.google.appengine.api.users.UserServiceFactory;
 import com.google.common.collect.ImmutableMap;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.DomainBase;
@@ -47,13 +46,13 @@ import google.registry.request.auth.AuthResult;
 import google.registry.request.auth.UserAuthInfo;
 import google.registry.security.XsrfTokenManager;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.UserInfo;
 import google.registry.tools.DomainLockUtils;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.StringGenerator;
 import google.registry.util.StringGenerator.Alphabets;
 import javax.servlet.http.HttpServletRequest;
@@ -86,6 +85,7 @@ final class RegistryLockVerifyActionTest {
   private DomainBase domain;
   private AuthResult authResult;
   private RegistryLockVerifyAction action;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(fakeClock);
 
   @BeforeEach
   void beforeEach() {
@@ -329,10 +329,7 @@ final class RegistryLockVerifyActionTest {
     RegistryLockVerifyAction action =
         new RegistryLockVerifyAction(
             new DomainLockUtils(
-                stringGenerator,
-                "adminreg",
-                AsyncTaskEnqueuerTest.createForTesting(
-                    mock(AppEngineServiceUtils.class), fakeClock, Duration.ZERO)),
+                stringGenerator, "adminreg", cloudTasksHelper.getTestCloudTasksUtils()),
             lockVerificationCode,
             isLock);
     authResult = AuthResult.create(AuthLevel.USER, UserAuthInfo.create(user, false));

core/src/test/java/google/registry/webdriver/TestServerExtension.java

@@ -33,6 +33,7 @@ import google.registry.testing.UserInfo;
 import java.net.URL;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
+import java.time.Duration;
 import java.util.Optional;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
@@ -53,6 +54,8 @@ import org.junit.jupiter.api.extension.ExtensionContext;
  */
 public final class TestServerExtension implements BeforeEachCallback, AfterEachCallback {
 
+  private static final Duration SERVER_STATUS_POLLING_INTERVAL = Duration.ofSeconds(5);
+
   private final ImmutableList<Fixture> fixtures;
   private final AppEngineExtension appEngineExtension;
   private final BlockingQueue<FutureTask<?>> jobs = new LinkedBlockingDeque<>();
@@ -107,8 +110,11 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
     serverThread = new Thread(server);
     synchronized (this) {
       serverThread.start();
-      while (!server.isRunning) {
-        this.wait();
+      while (server.serverStatus.equals(ServerStatus.NOT_STARTED)) {
+        this.wait(SERVER_STATUS_POLLING_INTERVAL.toMillis());
+      }
+      if (server.serverStatus.equals(ServerStatus.FAILED)) {
+        throw new RuntimeException("TestServer failed to start. See log for details.");
       }
     }
   }
@@ -163,6 +169,12 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
     return job.get();
   }
 
+  enum ServerStatus {
+    NOT_STARTED,
+    RUNNING,
+    FAILED
+  }
+
   private final class Server implements Runnable {
 
     private ExtensionContext context;
@@ -171,7 +183,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
       this.context = context;
     }
 
-    private volatile boolean isRunning = false;
+    private volatile ServerStatus serverStatus = ServerStatus.NOT_STARTED;
 
     @Override
     public void run() {
@@ -185,6 +197,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
           appEngineExtension.afterEach(context);
         }
       } catch (Throwable e) {
+        serverStatus = ServerStatus.FAILED;
         throw new RuntimeException(e);
       }
     }
@@ -196,7 +209,7 @@ public final class TestServerExtension implements BeforeEachCallback, AfterEachC
       testServer.start();
       System.out.printf("TestServerExtension is listening on: %s\n", testServer.getUrl("/"));
       synchronized (TestServerExtension.this) {
-        isRunning = true;
+        serverStatus = ServerStatus.RUNNING;
         TestServerExtension.this.notify();
       }
       try {

release/cloudbuild-nomulus.yaml

@@ -97,10 +97,8 @@ steps:
       google/registry/beam/invoicing_pipeline_metadata.json \
       google.registry.beam.rde.RdePipeline \
       google/registry/beam/rde_pipeline_metadata.json \
-      google.registry.beam.comparedb.ValidateDatastorePipeline \
-      google/registry/beam/validate_datastore_pipeline_metadata.json \
-      google.registry.beam.comparedb.ValidateSqlPipeline \
-      google/registry/beam/validate_sql_pipeline_metadata.json
+      google.registry.beam.comparedb.ValidateDatabasePipeline \
+      google/registry/beam/validate_database_pipeline_metadata.json
 # Tentatively build and publish Cloud SQL schema jar here, before schema release
 # process is finalized. Also publish nomulus:core jars that are needed for
 # server/schema compatibility tests.

util/src/main/java/google/registry/util/UrlConnectionException.java

@@ -0,0 +1,71 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.util;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.io.ByteStreams;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+
+/**
+ * Used when HTTP requests return a bad response, with troubleshooting info.
+ *
+ * <p>This class displays lots of helpful troubleshooting information.
+ */
+public class UrlConnectionException extends RuntimeException {
+
+  private final HttpURLConnection connection;
+
+  public UrlConnectionException(String message, HttpURLConnection connection) {
+    super(message);
+    this.connection = connection;
+  }
+
+  @Override
+  public String getMessage() {
+    byte[] resultContent;
+    int responseCode;
+    try {
+      resultContent = ByteStreams.toByteArray(connection.getInputStream());
+      responseCode = connection.getResponseCode();
+    } catch (IOException e) {
+      resultContent = new byte[] {};
+      responseCode = 0;
+    }
+    StringBuilder result =
+        new StringBuilder(2048 + resultContent.length)
+            .append(
+                String.format(
+                    "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\n",
+                    getClass().getSimpleName(),
+                    super.getMessage(),
+                    responseCode,
+                    connection.getURL().toString()));
+    connection
+        .getRequestProperties()
+        .forEach(
+            (key, value) -> {
+              result.append(key);
+              result.append(": ");
+              result.append(value);
+              result.append('\n');
+            });
+    result.append(">>>\n");
+    result.append(new String(resultContent, UTF_8));
+    result.append("\n<<<");
+    return result.toString();
+  }
+}

util/src/main/java/google/registry/util/UrlFetchException.java

@@ -1,63 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.util;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.google.appengine.api.urlfetch.HTTPHeader;
-import com.google.appengine.api.urlfetch.HTTPRequest;
-import com.google.appengine.api.urlfetch.HTTPResponse;
-
-/**
- * Exception for when App Engine HTTP requests return a bad response.
- *
- * <p>This class displays lots of helpful troubleshooting information.
- */
-public class UrlFetchException extends RuntimeException {
-
-  private final HTTPRequest req;
-  private final HTTPResponse rsp;
-
-  public UrlFetchException(String message, HTTPRequest req, HTTPResponse rsp) {
-    super(message);
-    this.req = checkNotNull(req, "req");
-    this.rsp = checkNotNull(rsp, "rsp");
-  }
-
-  @Override
-  public String getMessage() {
-    StringBuilder res =
-        new StringBuilder(2048 + rsp.getContent().length)
-            .append(
-                String.format(
-                    "%s: %s (HTTP Status %d)\nX-Fetch-URL: %s\nX-Final-URL: %s\n",
-                    getClass().getSimpleName(),
-                    super.getMessage(),
-                    rsp.getResponseCode(),
-                    req.getURL().toString(),
-                    rsp.getFinalUrl()));
-    for (HTTPHeader header : rsp.getHeadersUncombined()) {
-      res.append(header.getName());
-      res.append(": ");
-      res.append(header.getValue());
-      res.append('\n');
-    }
-    res.append(">>>\n");
-    res.append(new String(rsp.getContent(), UTF_8));
-    res.append("\n<<<");
-    return res.toString();
-  }
-}