Save db discrepancies to GCS (#1527)

* Save db discrepancies to GCS

common/src/main/java/google/registry/util/Sleeper.java

@@ -41,4 +41,20 @@ public interface Sleeper {
    * @see com.google.common.util.concurrent.Uninterruptibles#sleepUninterruptibly
    */
   void sleepUninterruptibly(ReadableDuration duration);
+
+  /**
+   * Puts the current thread to interruptible sleep.
+   *
+   * <p>This is a convenience method for {@link #sleep} that properly converts an {@link
+   * InterruptedException} to a {@link RuntimeException}.
+   */
+  default void sleepInterruptibly(ReadableDuration duration) {
+    try {
+      sleep(duration);
+    } catch (InterruptedException e) {
+      // Restore current thread's interrupted state.
+      Thread.currentThread().interrupt();
+      throw new RuntimeException("Interrupted.", e);
+    }
+  }
 }

core/build.gradle

@@ -319,6 +319,7 @@ dependencies {
   testCompile deps['com.google.appengine:appengine-testing']
   testCompile deps['com.google.guava:guava-testlib']
   testCompile deps['com.google.monitoring-client:contrib']
+  testCompile deps['com.google.protobuf:protobuf-java-util']
   testCompile deps['com.google.truth:truth']
   testCompile deps['com.google.truth.extensions:truth-java8-extension']
   testCompile deps['org.checkerframework:checker-qual']
@@ -706,10 +707,7 @@ createToolTask(
     'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
 
 createToolTask(
-    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
-
-createToolTask(
-    'validateDatastorePipeline', 'google.registry.beam.comparedb.ValidateDatastorePipeline')
+    'validateDatabasePipeline', 'google.registry.beam.comparedb.ValidateDatabasePipeline')
 
 
 createToolTask(
@@ -796,15 +794,10 @@ if (environment == 'alpha') {
               mainClass: 'google.registry.beam.rde.RdePipeline',
               metaData : 'google/registry/beam/rde_pipeline_metadata.json'
           ],
-      validateDatastore  :
+      validateDatabase  :
           [
-              mainClass: 'google.registry.beam.comparedb.ValidateDatastorePipeline',
-              metaData: 'google/registry/beam/validate_datastore_pipeline_metadata.json'
-          ],
-      validateSql  :
-          [
-              mainClass: 'google.registry.beam.comparedb.ValidateSqlPipeline',
-              metaData: 'google/registry/beam/validate_sql_pipeline_metadata.json'
+              mainClass: 'google.registry.beam.comparedb.ValidateDatabasePipeline',
+              metaData: 'google/registry/beam/validate_database_pipeline_metadata.json'
           ],
   ]
   project.tasks.create("stageBeamPipelines") {

core/src/main/java/google/registry/backup/BackupModule.java

@@ -21,6 +21,7 @@ import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_
 import static google.registry.backup.RestoreCommitLogsAction.BUCKET_OVERRIDE_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.FROM_TIME_PARAM;
 import static google.registry.backup.RestoreCommitLogsAction.TO_TIME_PARAM;
+import static google.registry.backup.SyncDatastoreToSqlSnapshotAction.SQL_SNAPSHOT_ID_PARAM;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredDatetimeParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
@@ -98,6 +99,12 @@ public final class BackupModule {
     return extractRequiredDatetimeParameter(req, TO_TIME_PARAM);
   }
 
+  @Provides
+  @Parameter(SQL_SNAPSHOT_ID_PARAM)
+  static String provideSqlSnapshotId(HttpServletRequest req) {
+    return extractRequiredParameter(req, SQL_SNAPSHOT_ID_PARAM);
+  }
+
   @Provides
   @Backups
   static ListeningExecutorService provideListeningExecutorService() {

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -30,6 +30,7 @@ import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -64,33 +65,47 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Override
   public void run() {
+    createCheckPointAndStartAsyncExport();
+  }
+
+  /**
+   * Creates a {@link CommitLogCheckpoint} and initiates an asynchronous export task.
+   *
+   * @return the {@code CommitLogCheckpoint} to be exported
+   */
+  public Optional<CommitLogCheckpoint> createCheckPointAndStartAsyncExport() {
     final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
     logger.atInfo().log(
         "Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
-    ofyTm()
-        .transact(
-            () -> {
-              DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
-              if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
-                logger.atInfo().log(
-                    "Newer checkpoint already written at time: %s", lastWrittenTime);
-                return;
-              }
-              auditedOfy()
-                  .saveIgnoringReadOnlyWithoutBackup()
-                  .entities(
-                      checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
-              // Enqueue a diff task between previous and current checkpoints.
-              cloudTasksUtils.enqueue(
-                  QUEUE_NAME,
-                  CloudTasksUtils.createPostTask(
-                      ExportCommitLogDiffAction.PATH,
-                      Service.BACKEND.toString(),
-                      ImmutableMultimap.of(
-                          LOWER_CHECKPOINT_TIME_PARAM,
-                          lastWrittenTime.toString(),
-                          UPPER_CHECKPOINT_TIME_PARAM,
-                          checkpoint.getCheckpointTime().toString())));
-            });
+    boolean isCheckPointPersisted =
+        ofyTm()
+            .transact(
+                () -> {
+                  DateTime lastWrittenTime =
+                      CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
+                  if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {
+                    logger.atInfo().log(
+                        "Newer checkpoint already written at time: %s", lastWrittenTime);
+                    return false;
+                  }
+                  auditedOfy()
+                      .saveIgnoringReadOnlyWithoutBackup()
+                      .entities(
+                          checkpoint,
+                          CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
+                  // Enqueue a diff task between previous and current checkpoints.
+                  cloudTasksUtils.enqueue(
+                      QUEUE_NAME,
+                      cloudTasksUtils.createPostTask(
+                          ExportCommitLogDiffAction.PATH,
+                          Service.BACKEND.toString(),
+                          ImmutableMultimap.of(
+                              LOWER_CHECKPOINT_TIME_PARAM,
+                              lastWrittenTime.toString(),
+                              UPPER_CHECKPOINT_TIME_PARAM,
+                              checkpoint.getCheckpointTime().toString())));
+                  return true;
+                });
+    return isCheckPointPersisted ? Optional.of(checkpoint) : Optional.empty();
   }
 }

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -76,7 +76,10 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private static final Duration LEASE_LENGTH = standardHours(1);
+  public static final String REPLAY_TO_SQL_LOCK_NAME =
+      ReplayCommitLogsToSqlAction.class.getSimpleName();
+
+  public static final Duration REPLAY_TO_SQL_LOCK_LEASE_LENGTH = standardHours(1);
   // Stop / pause where we are if we've been replaying for more than five minutes to avoid GAE
   // request timeouts
   private static final Duration REPLAY_TIMEOUT_DURATION = Duration.standardMinutes(5);
@@ -115,7 +118,11 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
     }
     Optional<Lock> lock =
         Lock.acquireSql(
-            this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
+            REPLAY_TO_SQL_LOCK_NAME,
+            null,
+            REPLAY_TO_SQL_LOCK_LEASE_LENGTH,
+            requestStatusChecker,
+            false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
       logger.atSevere().log(message);

core/src/main/java/google/registry/backup/SyncDatastoreToSqlSnapshotAction.java

@@ -0,0 +1,172 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.ofy.CommitLogCheckpoint;
+import google.registry.model.replay.ReplicateToDatastoreAction;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.Parameter;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Sleeper;
+import java.util.Optional;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * Synchronizes Datastore to a given SQL snapshot when SQL is the primary database.
+ *
+ * <p>The caller takes the responsibility for:
+ *
+ * <ul>
+ *   <li>verifying the current migration stage
+ *   <li>acquiring the {@link ReplicateToDatastoreAction#REPLICATE_TO_DATASTORE_LOCK_NAME
+ *       replication lock}, and
+ *   <li>while holding the lock, creating an SQL snapshot and invoking this action with the snapshot
+ *       id
+ * </ul>
+ *
+ * The caller may release the replication lock upon receiving the response from this action. Please
+ * refer to {@link google.registry.tools.ValidateDatastoreCommand} for more information on usage.
+ *
+ * <p>This action plays SQL transactions up to the user-specified snapshot, creates a new CommitLog
+ * checkpoint, and exports all CommitLogs to GCS up to this checkpoint. The timestamp of this
+ * checkpoint can be used to recreate a Datastore snapshot that is equivalent to the given SQL
+ * snapshot. If this action succeeds, the checkpoint timestamp is included in the response (the
+ * format of which is defined by {@link #SUCCESS_RESPONSE_TEMPLATE}).
+ */
+@Action(
+    service = Service.BACKEND,
+    path = SyncDatastoreToSqlSnapshotAction.PATH,
+    method = Action.Method.POST,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+@DeleteAfterMigration
+public class SyncDatastoreToSqlSnapshotAction implements Runnable {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  public static final String PATH = "/_dr/task/syncDatastoreToSqlSnapshot";
+
+  public static final String SUCCESS_RESPONSE_TEMPLATE =
+      "Datastore is up-to-date with provided SQL snapshot (%s). CommitLog timestamp is (%s).";
+
+  static final String SQL_SNAPSHOT_ID_PARAM = "sqlSnapshotId";
+
+  private static final int COMMITLOGS_PRESENCE_CHECK_ATTEMPTS = 10;
+  private static final Duration COMMITLOGS_PRESENCE_CHECK_DELAY = Duration.standardSeconds(6);
+
+  private final Response response;
+  private final Sleeper sleeper;
+
+  @Config("commitLogGcsBucket")
+  private final String gcsBucket;
+
+  private final GcsDiffFileLister gcsDiffFileLister;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+  private final CommitLogCheckpointAction commitLogCheckpointAction;
+  private final String sqlSnapshotId;
+
+  @Inject
+  SyncDatastoreToSqlSnapshotAction(
+      Response response,
+      Sleeper sleeper,
+      @Config("commitLogGcsBucket") String gcsBucket,
+      GcsDiffFileLister gcsDiffFileLister,
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder,
+      CommitLogCheckpointAction commitLogCheckpointAction,
+      @Parameter(SQL_SNAPSHOT_ID_PARAM) String sqlSnapshotId) {
+    this.response = response;
+    this.sleeper = sleeper;
+    this.gcsBucket = gcsBucket;
+    this.gcsDiffFileLister = gcsDiffFileLister;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+    this.commitLogCheckpointAction = commitLogCheckpointAction;
+    this.sqlSnapshotId = sqlSnapshotId;
+  }
+
+  @Override
+  public void run() {
+    logger.atInfo().log("Datastore validation invoked. SqlSnapshotId is %s.", sqlSnapshotId);
+
+    try {
+      CommitLogCheckpoint checkpoint = ensureDatabasesComparable(sqlSnapshotId);
+      response.setStatus(SC_OK);
+      response.setPayload(
+          String.format(SUCCESS_RESPONSE_TEMPLATE, sqlSnapshotId, checkpoint.getCheckpointTime()));
+      return;
+    } catch (Exception e) {
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(e.getMessage());
+    }
+  }
+
+  private CommitLogCheckpoint ensureDatabasesComparable(String sqlSnapshotId) {
+    // Replicate SQL transaction to Datastore, up to when this snapshot is taken.
+    int playbacks = ReplicateToDatastoreAction.replayAllTransactions(Optional.of(sqlSnapshotId));
+    logger.atInfo().log("Played %s SQL transactions.", playbacks);
+
+    Optional<CommitLogCheckpoint> checkpoint = exportCommitLogs();
+    if (!checkpoint.isPresent()) {
+      throw new RuntimeException("Cannot create CommitLog checkpoint");
+    }
+    logger.atInfo().log(
+        "CommitLog checkpoint created at %s.", checkpoint.get().getCheckpointTime());
+    verifyCommitLogsPersisted(checkpoint.get());
+    return checkpoint.get();
+  }
+
+  private Optional<CommitLogCheckpoint> exportCommitLogs() {
+    // Trigger an async CommitLog export to GCS. Will check file availability later.
+    // Although we can add support to synchronous execution, it can disrupt the export cadence
+    // when the system is busy
+    Optional<CommitLogCheckpoint> checkpoint =
+        commitLogCheckpointAction.createCheckPointAndStartAsyncExport();
+
+    // Failure to create checkpoint most likely caused by race with cron-triggered checkpointing.
+    // Retry once.
+    if (!checkpoint.isPresent()) {
+      commitLogCheckpointAction.createCheckPointAndStartAsyncExport();
+    }
+    return checkpoint;
+  }
+
+  private void verifyCommitLogsPersisted(CommitLogCheckpoint checkpoint) {
+    DateTime exportStartTime =
+        datastoreSnapshotFinder
+            .getSnapshotInfo(checkpoint.getCheckpointTime().toInstant())
+            .exportInterval()
+            .getStart();
+    logger.atInfo().log("Found Datastore export at %s", exportStartTime);
+    for (int attempts = 0; attempts < COMMITLOGS_PRESENCE_CHECK_ATTEMPTS; attempts++) {
+      try {
+        gcsDiffFileLister.listDiffFiles(gcsBucket, exportStartTime, checkpoint.getCheckpointTime());
+        return;
+      } catch (IllegalStateException e) {
+        // Gap in commitlog files. Fall through to sleep and retry.
+        logger.atInfo().log("Commitlog files not yet found on GCS.");
+      }
+      sleeper.sleepInterruptibly(COMMITLOGS_PRESENCE_CHECK_DELAY);
+    }
+    throw new RuntimeException("Cannot find all commitlog files.");
+  }
+}

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
@@ -152,32 +151,6 @@ public final class AsyncTaskEnqueuer {
             .param(PARAM_REQUESTED_TIME, now.toString()));
   }
 
-  /**
-   * Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked.
-   *
-   * <p>Note: the relockDuration must be present on the lock object.
-   */
-  public void enqueueDomainRelock(RegistryLock lock) {
-    checkArgument(
-        lock.getRelockDuration().isPresent(),
-        "Lock with ID %s not configured for relock",
-        lock.getRevisionId());
-    enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
-  }
-
-  /** Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked. */
-  void enqueueDomainRelock(Duration countdown, long lockRevisionId, int previousAttempts) {
-    String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
-    addTaskToQueueWithRetry(
-        asyncActionsPushQueue,
-        TaskOptions.Builder.withUrl(RelockDomainAction.PATH)
-            .method(Method.POST)
-            .header("Host", backendHostname)
-            .param(RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM, String.valueOf(lockRevisionId))
-            .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, String.valueOf(previousAttempts))
-            .countdownMillis(countdown.getMillis()));
-  }
-
   /**
    * Adds a task to a queue with retrying, to avoid aborting the entire flow over a transient issue
    * enqueuing a task.

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -88,7 +88,6 @@ public class RelockDomainAction implements Runnable {
   private final SendEmailService sendEmailService;
   private final DomainLockUtils domainLockUtils;
   private final Response response;
-  private final AsyncTaskEnqueuer asyncTaskEnqueuer;
 
   @Inject
   public RelockDomainAction(
@@ -99,8 +98,7 @@ public class RelockDomainAction implements Runnable {
       @Config("supportEmail") String supportEmail,
       SendEmailService sendEmailService,
       DomainLockUtils domainLockUtils,
-      Response response,
-      AsyncTaskEnqueuer asyncTaskEnqueuer) {
+      Response response) {
     this.oldUnlockRevisionId = oldUnlockRevisionId;
     this.previousAttempts = previousAttempts;
     this.alertRecipientAddress = alertRecipientAddress;
@@ -109,7 +107,6 @@ public class RelockDomainAction implements Runnable {
     this.sendEmailService = sendEmailService;
     this.domainLockUtils = domainLockUtils;
     this.response = response;
-    this.asyncTaskEnqueuer = asyncTaskEnqueuer;
   }
 
   @Override
@@ -245,8 +242,7 @@ public class RelockDomainAction implements Runnable {
       }
     }
     Duration timeBeforeRetry = previousAttempts < ATTEMPTS_BEFORE_SLOWDOWN ? TEN_MINUTES : ONE_HOUR;
-    asyncTaskEnqueuer.enqueueDomainRelock(
-        timeBeforeRetry, oldUnlockRevisionId, previousAttempts + 1);
+    domainLockUtils.enqueueDomainRelock(timeBeforeRetry, oldUnlockRevisionId, previousAttempts + 1);
   }
 
   private void sendSuccessEmail(RegistryLock oldLock) {

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -66,7 +66,7 @@ public class LatestDatastoreSnapshotFinder {
      * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
      */
     Optional<String> metaFilePathOptional =
-        findNewestExportMetadataFileBeforeTime(bucketName, exportEndTimeUpperBound, 2);
+        findNewestExportMetadataFileBeforeTime(bucketName, exportEndTimeUpperBound, 5);
     if (!metaFilePathOptional.isPresent()) {
       throw new NoSuchElementException("No exports found over the past 2 days.");
     }
@@ -125,12 +125,12 @@ public class LatestDatastoreSnapshotFinder {
 
   /** Holds information about a Datastore snapshot. */
   @AutoValue
-  abstract static class DatastoreSnapshotInfo {
-    abstract String exportDir();
+  public abstract static class DatastoreSnapshotInfo {
+    public abstract String exportDir();
 
-    abstract String commitLogDir();
+    public abstract String commitLogDir();
 
-    abstract Interval exportInterval();
+    public abstract Interval exportInterval();
 
     static DatastoreSnapshotInfo create(
         String exportDir, String commitLogDir, Interval exportOperationInterval) {

core/src/main/java/google/registry/beam/comparedb/ValidateDatabasePipeline.java

@@ -0,0 +1,206 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.flogger.FluentLogger;
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.util.SystemClock;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.TextIO;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * Validates the asynchronous data replication process between Datastore and Cloud SQL.
+ *
+ * <p>This pipeline is to be launched by {@link google.registry.tools.ValidateDatastoreCommand} or
+ * {@link google.registry.tools.ValidateSqlCommand}.
+ */
+@DeleteAfterMigration
+public class ValidateDatabasePipeline {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final Duration COMMITLOG_START_TIME_MARGIN = Duration.standardMinutes(10);
+
+  private final ValidateDatabasePipelineOptions options;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+
+  public ValidateDatabasePipeline(
+      ValidateDatabasePipelineOptions options,
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
+    this.options = options;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    DateTime latestCommitLogTime = DateTime.parse(options.getLatestCommitLogTimestamp());
+    DatastoreSnapshotInfo mostRecentExport =
+        datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
+
+    logger.atInfo().log(
+        "Comparing datastore export at %s and commitlog timestamp %s.",
+        mostRecentExport.exportDir(), latestCommitLogTime);
+
+    Optional<String> outputPath =
+        Optional.ofNullable(options.getDiffOutputGcsBucket())
+            .map(
+                bucket ->
+                    String.format(
+                        "gs://%s/validate_database/%s/diffs.txt",
+                        bucket, new SystemClock().nowUtc()));
+    outputPath.ifPresent(path -> logger.atInfo().log("Discrepancies will be logged to %s", path));
+
+    setupPipeline(
+        pipeline,
+        Optional.ofNullable(options.getSqlSnapshotId()),
+        mostRecentExport,
+        latestCommitLogTime,
+        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse),
+        outputPath);
+
+    pipeline.run();
+  }
+
+  static void setupPipeline(
+      Pipeline pipeline,
+      Optional<String> sqlSnapshotId,
+      DatastoreSnapshotInfo mostRecentExport,
+      DateTime latestCommitLogTime,
+      Optional<DateTime> compareStartTime,
+      Optional<String> diffOutputPath) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minus(COMMITLOG_START_TIME_MARGIN),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS,
+            compareStartTime);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId, compareStartTime);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    PCollectionList<String> diffLogs = PCollectionList.empty(pipeline);
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      diffLogs =
+          diffLogs.and(
+              PCollectionList.of(datastoreSnapshot.get(tag))
+                  .and(cloudSqlSnapshot.get(tag))
+                  .apply(
+                      "Combine from both snapshots: " + clazz.getSimpleName(),
+                      Flatten.pCollections())
+                  .apply(
+                      "Assign primary key to merged " + clazz.getSimpleName(),
+                      WithKeys.of(ValidateDatabasePipeline::getPrimaryKeyString)
+                          .withKeyType(strings()))
+                  .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+                  .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity())));
+    }
+    if (diffOutputPath.isPresent()) {
+      diffLogs
+          .apply("Gather diff logs", Flatten.pCollections())
+          .apply(
+              "Output diffs",
+              TextIO.write()
+                  .to(diffOutputPath.get())
+                  /**
+                   * Output to a single file for ease of use since diffs should be few. If this
+                   * assumption turns out not to be false, user should abort the pipeline and
+                   * investigate why.
+                   */
+                  .withoutSharding()
+                  .withDelimiter((Strings.repeat("-", 80) + "\n").toCharArray()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  public static void main(String[] args) {
+    ValidateDatabasePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args)
+            .withValidation()
+            .as(ValidateDatabasePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Set up JPA in the pipeline harness (the locally executed part of the main() method). Reuse
+    // code in RegistryPipelineWorkerInitializer, which only applies to pipeline worker VMs.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+
+    new ValidateDatabasePipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateDatabasePipelineOptions.java

@@ -14,14 +14,15 @@
 
 package google.registry.beam.comparedb;
 
+import google.registry.beam.common.RegistryPipelineOptions;
 import google.registry.model.annotations.DeleteAfterMigration;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.Validation;
 
-/** BEAM pipeline options for {@link ValidateDatastorePipelineOptions}. */
+/** BEAM pipeline options for {@link ValidateDatabasePipeline}. */
 @DeleteAfterMigration
-public interface ValidateDatastorePipelineOptions extends ValidateSqlPipelineOptions {
+public interface ValidateDatabasePipelineOptions extends RegistryPipelineOptions {
 
   @Description(
       "The id of the SQL snapshot to be compared with Datastore. "
@@ -36,4 +37,19 @@ public interface ValidateDatastorePipelineOptions extends ValidateSqlPipelineOpt
   String getLatestCommitLogTimestamp();
 
   void setLatestCommitLogTimestamp(String commitLogEndTimestamp);
+
+  @Description(
+      "For history entries and EPP resources, only those modified strictly after this time are "
+          + "included in comparison. Value is in ISO8601 format. "
+          + "Other entity types are not affected.")
+  @Nullable
+  String getComparisonStartTimestamp();
+
+  void setComparisonStartTimestamp(String comparisonStartTimestamp);
+
+  @Description("The GCS bucket where discrepancies found during comparison should be logged.")
+  @Nullable
+  String getDiffOutputGcsBucket();
+
+  void setDiffOutputGcsBucket(String gcsBucket);
 }

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipeline.java

@@ -1,83 +0,0 @@
-// Copyright 2022 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.comparedb;
-
-import google.registry.beam.common.RegistryPipelineOptions;
-import google.registry.beam.common.RegistryPipelineWorkerInitializer;
-import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
-import google.registry.model.annotations.DeleteAfterMigration;
-import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
-import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
-import java.util.Optional;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.joda.time.DateTime;
-
-/**
- * Validates the asynchronous data replication process from Cloud SQL (primary) to Datastore
- * (secondary).
- *
- * <p>This pipeline simply compares the snapshots provided by an invoker, which is responsible for
- * obtaining two consistent snapshots for the same point of time.
- */
-// TODO(weiminyu): Implement the invoker action in a followup PR.
-@DeleteAfterMigration
-public class ValidateDatastorePipeline {
-
-  private final ValidateDatastorePipelineOptions options;
-  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
-
-  public ValidateDatastorePipeline(
-      ValidateDatastorePipelineOptions options,
-      LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
-    this.options = options;
-    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
-  }
-
-  void run(Pipeline pipeline) {
-    DateTime latestCommitLogTime = DateTime.parse(options.getLatestCommitLogTimestamp());
-    DatastoreSnapshotInfo mostRecentExport =
-        datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
-
-    ValidateSqlPipeline.setupPipeline(
-        pipeline,
-        Optional.ofNullable(options.getSqlSnapshotId()),
-        mostRecentExport,
-        latestCommitLogTime,
-        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
-
-    pipeline.run();
-  }
-
-  public static void main(String[] args) {
-    ValidateDatastorePipelineOptions options =
-        PipelineOptionsFactory.fromArgs(args)
-            .withValidation()
-            .as(ValidateDatastorePipelineOptions.class);
-    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
-
-    // Defensively set important options.
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
-    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
-
-    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
-    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
-
-    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
-        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
-            .datastoreSnapshotInfoFinder();
-    new ValidateDatastorePipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
-  }
-}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -1,264 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.comparedb;
-
-import static com.google.common.base.Verify.verify;
-import static org.apache.beam.sdk.values.TypeDescriptors.strings;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
-import com.google.common.flogger.FluentLogger;
-import google.registry.beam.common.DatabaseSnapshot;
-import google.registry.beam.common.RegistryPipelineWorkerInitializer;
-import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
-import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
-import google.registry.model.annotations.DeleteAfterMigration;
-import google.registry.model.common.DatabaseMigrationStateSchedule;
-import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
-import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.DomainHistory;
-import google.registry.model.replay.SqlEntity;
-import google.registry.model.replay.SqlReplayCheckpoint;
-import google.registry.model.server.Lock;
-import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
-import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
-import google.registry.persistence.transaction.TransactionManagerFactory;
-import google.registry.util.RequestStatusChecker;
-import google.registry.util.SystemClock;
-import java.io.Serializable;
-import java.util.Optional;
-import org.apache.beam.sdk.Pipeline;
-import org.apache.beam.sdk.PipelineResult.State;
-import org.apache.beam.sdk.coders.SerializableCoder;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.transforms.Flatten;
-import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.ParDo;
-import org.apache.beam.sdk.transforms.WithKeys;
-import org.apache.beam.sdk.values.PCollectionList;
-import org.apache.beam.sdk.values.PCollectionTuple;
-import org.apache.beam.sdk.values.TupleTag;
-import org.joda.time.DateTime;
-import org.joda.time.Duration;
-
-/**
- * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
- * (secondary storage).
- */
-@DeleteAfterMigration
-public class ValidateSqlPipeline {
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  /** Specifies the extra CommitLogs to load before the start of a Database export. */
-  private static final Duration COMMITLOG_START_TIME_MARGIN = Duration.standardMinutes(10);
-
-  /**
-   * Name of the lock used by the commitlog replay process.
-   *
-   * <p>See {@link google.registry.backup.ReplayCommitLogsToSqlAction} for more information.
-   */
-  private static final String COMMITLOG_REPLAY_LOCK_NAME = "ReplayCommitLogsToSqlAction";
-
-  private static final Duration REPLAY_LOCK_LEASE_LENGTH = Duration.standardHours(1);
-  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_TIMEOUT =
-      java.time.Duration.ofMinutes(6);
-  private static final java.time.Duration REPLAY_LOCK_ACQUIRE_DELAY =
-      java.time.Duration.ofSeconds(30);
-
-  private final ValidateSqlPipelineOptions options;
-  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
-
-  public ValidateSqlPipeline(
-      ValidateSqlPipelineOptions options, LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
-    this.options = options;
-    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
-  }
-
-  @VisibleForTesting
-  void run(Pipeline pipeline) {
-    Optional<Lock> lock = acquireCommitLogReplayLock();
-    if (lock.isPresent()) {
-      logger.atInfo().log("Acquired CommitLog Replay lock.");
-    } else {
-      throw new RuntimeException("Failed to acquire CommitLog Replay lock.");
-    }
-
-    try {
-      DateTime latestCommitLogTime =
-          TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
-      DatastoreSnapshotInfo mostRecentExport =
-          datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
-      Preconditions.checkState(
-          latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
-          "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
-      try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
-        // Eagerly release the commitlog replay lock so that replay can resume.
-        lock.ifPresent(Lock::releaseSql);
-        lock = Optional.empty();
-
-        logger.atInfo().log(
-            "Starting comparison with export at %s and latestCommitLogTime at %s",
-            mostRecentExport.exportDir(), latestCommitLogTime);
-
-        setupPipeline(
-            pipeline,
-            Optional.of(databaseSnapshot.getSnapshotId()),
-            mostRecentExport,
-            latestCommitLogTime,
-            Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
-        State state = pipeline.run().waitUntilFinish();
-        if (!State.DONE.equals(state)) {
-          throw new IllegalStateException("Unexpected pipeline state: " + state);
-        }
-      }
-    } finally {
-      lock.ifPresent(Lock::releaseSql);
-    }
-  }
-
-  static void setupPipeline(
-      Pipeline pipeline,
-      Optional<String> sqlSnapshotId,
-      DatastoreSnapshotInfo mostRecentExport,
-      DateTime latestCommitLogTime,
-      Optional<DateTime> compareStartTime) {
-    pipeline
-        .getCoderRegistry()
-        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
-
-    PCollectionTuple datastoreSnapshot =
-        DatastoreSnapshots.loadDatastoreSnapshotByKind(
-            pipeline,
-            mostRecentExport.exportDir(),
-            mostRecentExport.commitLogDir(),
-            mostRecentExport.exportInterval().getStart().minus(COMMITLOG_START_TIME_MARGIN),
-            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
-            // this parameter is exclusive.
-            latestCommitLogTime.plusMillis(1),
-            DatastoreSnapshots.ALL_DATASTORE_KINDS,
-            compareStartTime);
-
-    PCollectionTuple cloudSqlSnapshot =
-        SqlSnapshots.loadCloudSqlSnapshotByType(
-            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId, compareStartTime);
-
-    verify(
-        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
-        "Expecting the same set of types in both snapshots.");
-
-    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
-      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
-      verify(
-          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
-      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
-      PCollectionList.of(datastoreSnapshot.get(tag))
-          .and(cloudSqlSnapshot.get(tag))
-          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
-          .apply(
-              "Assign primary key to merged " + clazz.getSimpleName(),
-              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
-          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
-          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
-    }
-  }
-
-  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
-    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
-    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
-    // DomainHistory. See BulkQueryEntities.java for more information.
-    if (sqlEntity instanceof DomainBase) {
-      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
-    }
-    if (sqlEntity instanceof DomainHistory) {
-      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
-    }
-    return sqlEntity.getPrimaryKeyString();
-  }
-
-  private static Optional<Lock> acquireCommitLogReplayLock() {
-    Stopwatch stopwatch = Stopwatch.createStarted();
-    while (stopwatch.elapsed().minus(REPLAY_LOCK_ACQUIRE_TIMEOUT).isNegative()) {
-      Optional<Lock> lock = tryAcquireCommitLogReplayLock();
-      if (lock.isPresent()) {
-        return lock;
-      }
-      logger.atInfo().log("Failed to acquired CommitLog Replay lock. Will retry...");
-      try {
-        Thread.sleep(REPLAY_LOCK_ACQUIRE_DELAY.toMillis());
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        throw new RuntimeException("Interrupted.");
-      }
-    }
-    return Optional.empty();
-  }
-
-  private static Optional<Lock> tryAcquireCommitLogReplayLock() {
-    return Lock.acquireSql(
-        COMMITLOG_REPLAY_LOCK_NAME,
-        null,
-        REPLAY_LOCK_LEASE_LENGTH,
-        getLockingRequestStatusChecker(),
-        false);
-  }
-
-  /**
-   * Returns a fake implementation of {@link RequestStatusChecker} that is required for lock
-   * acquisition. The default implementation is AppEngine-specific and is unusable on GCE.
-   */
-  private static RequestStatusChecker getLockingRequestStatusChecker() {
-    return new RequestStatusChecker() {
-      @Override
-      public String getLogId() {
-        return "ValidateSqlPipeline";
-      }
-
-      LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
-          DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
-              .datastoreSnapshotInfoFinder();
-
-      @Override
-      public boolean isRunning(String requestLogId) {
-        return true;
-      }
-    };
-  }
-
-  public static void main(String[] args) {
-    ValidateSqlPipelineOptions options =
-        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
-
-    // Defensively set important options.
-    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
-    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
-
-    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
-    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
-
-    MigrationState state =
-        DatabaseMigrationStateSchedule.getValueAtTime(new SystemClock().nowUtc());
-    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
-      throw new IllegalStateException("This pipeline is not designed for migration phase " + state);
-    }
-
-    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
-        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
-            .datastoreSnapshotInfoFinder();
-
-    new ValidateSqlPipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
-  }
-}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -1,34 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.beam.comparedb;
-
-import google.registry.beam.common.RegistryPipelineOptions;
-import google.registry.model.annotations.DeleteAfterMigration;
-import javax.annotation.Nullable;
-import org.apache.beam.sdk.options.Description;
-
-/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
-@DeleteAfterMigration
-public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {
-
-  @Description(
-      "For history entries and EPP resources, only those modified strictly after this time are "
-          + "included in comparison. Value is in ISO8601 format. "
-          + "Other entity types are not affected.")
-  @Nullable
-  String getComparisonStartTimestamp();
-
-  void setComparisonStartTimestamp(String comparisonStartTimestamp);
-}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -20,10 +20,8 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.flogger.FluentLogger;
 import google.registry.beam.initsql.Transforms;
 import google.registry.config.RegistryEnvironment;
-import google.registry.model.BackupGroupRoot;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.DeleteAfterMigration;
@@ -38,6 +36,7 @@ import google.registry.model.poll.PollMessage;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.util.DiffUtils;
 import java.lang.reflect.Field;
 import java.math.BigInteger;
 import java.util.HashMap;
@@ -52,10 +51,9 @@ import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.TupleTag;
 
-/** Helpers for use by {@link ValidateSqlPipeline}. */
+/** Helpers for use by {@link ValidateDatabasePipeline}. */
 @DeleteAfterMigration
 final class ValidateSqlUtils {
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   private ValidateSqlUtils() {}
 
@@ -67,7 +65,7 @@ final class ValidateSqlUtils {
    * Query template for finding the median value of the {@code history_revision_id} column in one of
    * the History tables.
    *
-   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * <p>The {@link ValidateDatabasePipeline} uses this query to parallelize the query to some of the
    * history tables. Although the {@code repo_id} column is the leading column in the primary keys
    * of these tables, in practice and with production data, division by {@code history_revision_id}
    * works slightly faster for unknown reasons.
@@ -99,15 +97,13 @@ final class ValidateSqlUtils {
     return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
   }
 
-  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, String> {
     private final HashMap<String, Counter> totalCounters = new HashMap<>();
     private final HashMap<String, Counter> missingCounters = new HashMap<>();
     private final HashMap<String, Counter> unequalCounters = new HashMap<>();
     private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
     private final HashMap<String, Counter> duplicateEntityCounters = new HashMap<>();
 
-    private volatile boolean logPrinted = false;
-
     private String getCounterKey(Class<?> clazz) {
       return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
     }
@@ -125,39 +121,36 @@ final class ValidateSqlUtils {
           counterKey, Metrics.counter("CompareDB", "Duplicate Entities:" + counterKey));
     }
 
+    String duplicateEntityLog(String key, ImmutableList<SqlEntity> entities) {
+      return String.format("%s: %d entities.", key, entities.size());
+    }
+
+    String unmatchedEntityLog(String key, SqlEntity entry) {
+      // For a PollMessage only found in Datastore, key is not enough to query for it.
+      return String.format("Missing in one DB:\n%s", entry instanceof PollMessage ? entry : key);
+    }
+
     /**
      * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
      * This will be removed when we start exporting such entities to GCS.
      */
-    void logDiff(String key, Object entry0, Object entry1) {
-      if (logPrinted) {
-        return;
-      }
-      logPrinted = true;
+    String unEqualEntityLog(String key, SqlEntity entry0, SqlEntity entry1) {
       Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
       Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
-      StringBuilder sb = new StringBuilder();
-      fields0.forEach(
-          (field, value) -> {
-            if (fields1.containsKey(field)) {
-              if (!Objects.equals(value, fields1.get(field))) {
-                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
-              }
-            } else {
-              sb.append(field + "Not found in entity 2\n");
-            }
-          });
-      fields1.forEach(
-          (field, value) -> {
-            if (!fields0.containsKey(field)) {
-              sb.append(field + "Not found in entity 1\n");
-            }
-          });
-      logger.atWarning().log(key + "  " + sb.toString());
+      return key + "  " + DiffUtils.prettyPrintEntityDeepDiff(fields0, fields1);
+    }
+
+    String badEntitiesLog(String key, SqlEntity entry0, SqlEntity entry1) {
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      return String.format(
+          "Failed to parse one or both entities for key %s:\n%s\n",
+          key, DiffUtils.prettyPrintEntityDeepDiff(fields0, fields1));
     }
 
     @ProcessElement
-    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+    public void processElement(
+        @Element KV<String, Iterable<SqlEntity>> kv, OutputReceiver<String> out) {
       ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
 
       verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
@@ -170,6 +163,7 @@ final class ValidateSqlUtils {
         // Duplicates may happen with Cursors if imported across projects. Its key in Datastore, the
         // id field, encodes the project name and is not fixed by the importing job.
         duplicateEntityCounters.get(counterKey).inc();
+        out.output(duplicateEntityLog(kv.getKey(), entities) + "\n");
         return;
       }
 
@@ -178,31 +172,31 @@ final class ValidateSqlUtils {
           return;
         }
         missingCounters.get(counterKey).inc();
-        // Temporary debugging help. See logDiff() above.
-        if (!logPrinted) {
-          logPrinted = true;
-          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
-        }
+        out.output(unmatchedEntityLog(kv.getKey(), entities.get(0)) + "\n");
+        return;
+      }
+      SqlEntity entity0 = entities.get(0);
+      SqlEntity entity1 = entities.get(1);
+
+      if (isSpecialCaseProberEntity(entity0) && isSpecialCaseProberEntity(entity1)) {
+        // Ignore prober-related data: their deletions are not propagated from Datastore to SQL.
+        // When code reaches here, in most cases it involves one soft deleted entity in Datastore
+        // and an SQL entity with its pre-deletion status.
         return;
       }
-      SqlEntity entity0;
-      SqlEntity entity1;
 
       try {
-        entity0 = normalizeEntity(entities.get(0));
-        entity1 = normalizeEntity(entities.get(1));
+        entity0 = normalizeEntity(entity0);
+        entity1 = normalizeEntity(entity1);
       } catch (Exception e) {
-        // Temporary debugging help. See logDiff() above.
-        if (!logPrinted) {
-          logPrinted = true;
-          badEntityCounters.get(counterKey).inc();
-        }
+        badEntityCounters.get(counterKey).inc();
+        out.output(badEntitiesLog(kv.getKey(), entity0, entity1));
         return;
       }
 
       if (!Objects.equals(entity0, entity1)) {
         unequalCounters.get(counterKey).inc();
-        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+        out.output(unEqualEntityLog(kv.getKey(), entities.get(0), entities.get(1)));
       }
     }
   }
@@ -227,15 +221,6 @@ final class ValidateSqlUtils {
    */
   static SqlEntity normalizeEppResource(SqlEntity eppResource) {
     try {
-      if (isSpecialCaseProberEntity(eppResource)) {
-        // Clearing some timestamps. See isSpecialCaseProberEntity() for reasons.
-        Field lastUpdateTime = BackupGroupRoot.class.getDeclaredField("updateTimestamp");
-        lastUpdateTime.setAccessible(true);
-        lastUpdateTime.set(eppResource, null);
-        Field deletionTime = EppResource.class.getDeclaredField("deletionTime");
-        deletionTime.setAccessible(true);
-        deletionTime.set(eppResource, null);
-      }
       Field authField =
           eppResource instanceof DomainContent
               ? DomainContent.class.getDeclaredField("authInfo")

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -304,7 +304,7 @@ public class RdeIO {
                 if (key.mode() == RdeMode.FULL) {
                   cloudTasksUtils.enqueue(
                       RDE_UPLOAD_QUEUE,
-                      CloudTasksUtils.createPostTask(
+                      cloudTasksUtils.createPostTask(
                           RdeUploadAction.PATH,
                           Service.BACKEND.getServiceId(),
                           ImmutableMultimap.of(
@@ -315,7 +315,7 @@ public class RdeIO {
                 } else {
                   cloudTasksUtils.enqueue(
                       BRDA_QUEUE,
-                      CloudTasksUtils.createPostTask(
+                      cloudTasksUtils.createPostTask(
                           BrdaCopyAction.PATH,
                           Service.BACKEND.getServiceId(),
                           ImmutableMultimap.of(

core/src/main/java/google/registry/config/CloudTasksUtilsModule.java

@@ -21,6 +21,7 @@ import dagger.Module;
 import dagger.Provides;
 import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.CloudTasksUtils.GcpCloudTasksClient;
 import google.registry.util.CloudTasksUtils.SerializableCloudTasksClient;
@@ -46,8 +47,9 @@ public abstract class CloudTasksUtilsModule {
       @Config("projectId") String projectId,
       @Config("locationId") String locationId,
       SerializableCloudTasksClient client,
-      Retrier retrier) {
-    return new CloudTasksUtils(retrier, projectId, locationId, client);
+      Retrier retrier,
+      Clock clock) {
+    return new CloudTasksUtils(retrier, clock, projectId, locationId, client);
   }
 
   // Provides a supplier instead of using a Dagger @Provider because the latter is not serializable.

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -20,7 +20,6 @@ import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -35,7 +34,6 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  @Inject Clock clock;
   @Inject CloudTasksUtils cloudTasksUtils;
 
   @Inject @Parameter("endpoint") String endpoint;
@@ -43,18 +41,15 @@ public final class CommitLogFanoutAction implements Runnable {
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
-
-
   @Override
   public void run() {
     for (int bucketId : CommitLogBucket.getBucketIds()) {
       cloudTasksUtils.enqueue(
           queue,
-          CloudTasksUtils.createPostTask(
+          cloudTasksUtils.createPostTaskWithJitter(
               endpoint,
               Service.BACKEND.toString(),
               ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
-              clock,
               jitterSeconds));
     }
   }

core/src/main/java/google/registry/cron/TldFanoutAction.java

@@ -45,7 +45,6 @@ import google.registry.request.ParameterMap;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
 import java.util.stream.Stream;
@@ -98,7 +97,6 @@ public final class TldFanoutAction implements Runnable {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  @Inject Clock clock;
   @Inject CloudTasksUtils cloudTasksUtils;
   @Inject Response response;
   @Inject @Parameter(ENDPOINT_PARAM) String endpoint;
@@ -159,7 +157,7 @@ public final class TldFanoutAction implements Runnable {
       params = ArrayListMultimap.create(params);
       params.put(RequestParameters.PARAM_TLD, tld);
     }
-    return CloudTasksUtils.createPostTask(
-        endpoint, Service.BACKEND.toString(), params, clock, jitterSeconds);
+    return cloudTasksUtils.createPostTaskWithJitter(
+        endpoint, Service.BACKEND.toString(), params, jitterSeconds);
   }
 }

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -422,6 +422,12 @@ have been in the database for a certain period of time. -->
     <url-pattern>/_dr/task/createSyntheticHistoryEntries</url-pattern>
   </servlet-mapping>
 
+  <!-- Action to sync Datastore to a snapshot of the primary SQL database. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/syncDatastoreToSqlSnapshot</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -253,16 +253,6 @@
     <target>backend</target>
   </cron>
 
-  <cron>
-    <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
-    <description>
-      This job clears out data from probers and runs once a week.
-    </description>
-    <schedule>every monday 14:00</schedule>
-    <timezone>UTC</timezone>
-    <target>backend</target>
-  </cron>
-
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
     <description>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -168,15 +168,6 @@
     <target>backend</target>
   </cron>
 
-  <cron>
-    <url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
-    <description>
-      This job runs an action that sends emails to partners if their certificates are expiring soon.
-    </description>
-    <schedule>every day 04:30</schedule>
-    <target>backend</target>
-  </cron>
-
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>
@@ -191,16 +182,6 @@
     <target>backend</target>
   </cron>
 
-  <cron>
-    <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/deleteProberData&runInEmpty]]></url>
-    <description>
-      This job clears out data from probers and runs once a week.
-    </description>
-    <schedule>every monday 14:00</schedule>
-    <timezone>UTC</timezone>
-    <target>backend</target>
-  </cron>
-
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/exportReservedTerms&forEachRealTld]]></url>
     <description>

core/src/main/java/google/registry/export/sheet/SyncRegistrarsSheetAction.java

@@ -14,8 +14,6 @@
 
 package google.registry.export.sheet;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
@@ -23,7 +21,6 @@ import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action;
@@ -100,7 +97,7 @@ public class SyncRegistrarsSheetAction implements Runnable {
   }
 
   public static final String PATH = "/_dr/task/syncRegistrarsSheet";
-  private static final String QUEUE = "sheet";
+  public static final String QUEUE = "sheet";
   private static final String LOCK_NAME = "Synchronize registrars sheet";
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -144,11 +141,4 @@ public class SyncRegistrarsSheetAction implements Runnable {
       Result.LOCKED.send(response, null);
     }
   }
-
-  /**
-   * Enqueues a sync registrar sheet task targeting the App Engine service specified by hostname.
-   */
-  public static void enqueueRegistrarSheetSync(String hostname) {
-    getQueue(QUEUE).add(withUrl(PATH).method(Method.GET).header("Host", hostname));
-  }
 }

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -1112,7 +1112,16 @@ public class DomainFlowUtils {
         // Only cancel fields which are cancelable
         if (cancelableFields.contains(record.getReportField())) {
           int cancelledAmount = -1 * record.getReportAmount();
-          recordsBuilder.add(record.asBuilder().setReportAmount(cancelledAmount).build());
+          // NB: It's necessary to create a new DomainTransactionRecord from scratch so that we
+          // don't retain the ID of the previous record to cancel. If we keep the ID, Hibernate
+          // will remove that record from the DB entirely as the record will be re-parented on
+          // this DomainHistory being created now.
+          recordsBuilder.add(
+              DomainTransactionRecord.create(
+                  record.getTld(),
+                  record.getReportingTime(),
+                  record.getReportField(),
+                  cancelledAmount));
         }
       }
     }

core/src/main/java/google/registry/loadtest/LoadTestAction.java

@@ -14,8 +14,6 @@
 
 package google.registry.loadtest;
 
-import static com.google.appengine.api.taskqueue.QueueConstants.maxTasksPerAdd;
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Lists.partition;
@@ -24,16 +22,20 @@ import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.util.Arrays.asList;
 import static org.joda.time.DateTimeZone.UTC;
 
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.cloud.tasks.v2.Task;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Iterators;
 import com.google.common.flogger.FluentLogger;
+import com.google.protobuf.Timestamp;
 import google.registry.config.RegistryEnvironment;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
 import google.registry.security.XsrfTokenManager;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
+import java.time.Instant;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
@@ -62,6 +64,7 @@ public class LoadTestAction implements Runnable {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   private static final int NUM_QUEUES = 10;
+  private static final int MAX_TASKS_PER_LOAD = 100;
   private static final int ARBITRARY_VALID_HOST_LENGTH = 40;
   private static final int MAX_CONTACT_LENGTH = 13;
   private static final int MAX_DOMAIN_LABEL_LENGTH = 63;
@@ -146,7 +149,7 @@ public class LoadTestAction implements Runnable {
   @Parameter("hostInfos")
   int hostInfosPerSecond;
 
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
   private final String xmlContactCreateTmpl;
   private final String xmlContactCreateFail;
@@ -208,7 +211,7 @@ public class LoadTestAction implements Runnable {
     ImmutableList<String> contactNames = contactNamesBuilder.build();
     ImmutableList<String> hostPrefixes = hostPrefixesBuilder.build();
 
-    ImmutableList.Builder<TaskOptions> tasks = new ImmutableList.Builder<>();
+    ImmutableList.Builder<Task> tasks = new ImmutableList.Builder<>();
     for (int offsetSeconds = 0; offsetSeconds < runSeconds; offsetSeconds++) {
       DateTime startSecond = initialStartSecond.plusSeconds(offsetSeconds);
       // The first "failed" creates might actually succeed if the object doesn't already exist, but
@@ -254,7 +257,7 @@ public class LoadTestAction implements Runnable {
                   .collect(toImmutableList()),
               startSecond));
     }
-    ImmutableList<TaskOptions> taskOptions = tasks.build();
+    ImmutableList<Task> taskOptions = tasks.build();
     enqueue(taskOptions);
     logger.atInfo().log("Added %d total load test tasks.", taskOptions.size());
   }
@@ -322,28 +325,51 @@ public class LoadTestAction implements Runnable {
     return name.toString();
   }
 
-  private List<TaskOptions> createTasks(List<String> xmls, DateTime start) {
-    ImmutableList.Builder<TaskOptions> tasks = new ImmutableList.Builder<>();
+  private List<Task> createTasks(List<String> xmls, DateTime start) {
+    ImmutableList.Builder<Task> tasks = new ImmutableList.Builder<>();
     for (int i = 0; i < xmls.size(); i++) {
       // Space tasks evenly within across a second.
-      int offsetMillis = (int) (1000.0 / xmls.size() * i);
+      Instant scheduleTime =
+          Instant.ofEpochMilli(start.plusMillis((int) (1000.0 / xmls.size() * i)).getMillis());
       tasks.add(
-          TaskOptions.Builder.withUrl("/_dr/epptool")
-              .etaMillis(start.getMillis() + offsetMillis)
-              .header(X_CSRF_TOKEN, xsrfToken)
-              .param("clientId", registrarId)
-              .param("superuser", Boolean.FALSE.toString())
-              .param("dryRun", Boolean.FALSE.toString())
-              .param("xml", xmls.get(i)));
+          Task.newBuilder()
+              .setAppEngineHttpRequest(
+                  cloudTasksUtils
+                      .createPostTask(
+                          "/_dr/epptool",
+                          Service.TOOLS.toString(),
+                          ImmutableMultimap.of(
+                              "clientId",
+                              registrarId,
+                              "superuser",
+                              Boolean.FALSE.toString(),
+                              "dryRun",
+                              Boolean.FALSE.toString(),
+                              "xml",
+                              xmls.get(i)))
+                      .toBuilder()
+                      .getAppEngineHttpRequest()
+                      .toBuilder()
+                      // instead of adding the X_CSRF_TOKEN to params, this remains as part of
+                      // headers because of the existing setup for authentication in {@link
+                      // google.registry.request.auth.LegacyAuthenticationMechanism}
+                      .putHeaders(X_CSRF_TOKEN, xsrfToken)
+                      .build())
+              .setScheduleTime(
+                  Timestamp.newBuilder()
+                      .setSeconds(scheduleTime.getEpochSecond())
+                      .setNanos(scheduleTime.getNano())
+                      .build())
+              .build());
     }
     return tasks.build();
   }
 
-  private void enqueue(List<TaskOptions> tasks) {
-    List<List<TaskOptions>> chunks = partition(tasks, maxTasksPerAdd());
+  private void enqueue(List<Task> tasks) {
+    List<List<Task>> chunks = partition(tasks, MAX_TASKS_PER_LOAD);
     // Farm out tasks to multiple queues to work around queue qps quotas.
     for (int i = 0; i < chunks.size(); i++) {
-      taskQueueUtils.enqueue(getQueue("load" + (i % NUM_QUEUES)), chunks.get(i));
+      cloudTasksUtils.enqueue("load" + (i % NUM_QUEUES), chunks.get(i));
     }
   }
 }

core/src/main/java/google/registry/model/BackupGroupRoot.java

@@ -60,4 +60,25 @@ public abstract class BackupGroupRoot extends ImmutableObject implements UnsafeS
   protected void copyUpdateTimestamp(BackupGroupRoot other) {
     this.updateTimestamp = PreconditionsUtils.checkArgumentNotNull(other, "other").updateTimestamp;
   }
+
+  /**
+   * Resets the {@link #updateTimestamp} to force Hibernate to persist it.
+   *
+   * <p>This method is for use in setters in derived builders that do not result in the derived
+   * object being persisted.
+   */
+  protected void resetUpdateTimestamp() {
+    this.updateTimestamp = UpdateAutoTimestamp.create(null);
+  }
+
+  /**
+   * Sets the {@link #updateTimestamp}.
+   *
+   * <p>This method is for use in the few places where we need to restore the update timestamp after
+   * mutating a collection in order to force the new timestamp to be persisted when it ordinarily
+   * wouldn't.
+   */
+  protected void setUpdateTimestamp(UpdateAutoTimestamp timestamp) {
+    updateTimestamp = timestamp;
+  }
 }

core/src/main/java/google/registry/model/EppResource.java

@@ -362,6 +362,16 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
       return thisCastToDerived();
     }
 
+    /**
+     * Set the update timestamp.
+     *
+     * <p>This is provided at EppResource since BackupGroupRoot doesn't have a Builder.
+     */
+    public B setUpdateTimestamp(UpdateAutoTimestamp updateTimestamp) {
+      getInstance().setUpdateTimestamp(updateTimestamp);
+      return thisCastToDerived();
+    }
+
     /** Build the resource, nullifying empty strings and sets and setting defaults. */
     @Override
     public T build() {

core/src/main/java/google/registry/model/bulkquery/BulkQueryEntities.java

@@ -74,6 +74,8 @@ public class BulkQueryEntities {
     builder.setGracePeriods(gracePeriods);
     builder.setDsData(delegationSignerData);
     builder.setNameservers(nsHosts);
+    // Restore the original update timestamp (this gets cleared when we set nameservers or DS data).
+    builder.setUpdateTimestamp(domainBaseLite.getUpdateTimestamp());
     return builder.build();
   }
 
@@ -100,6 +102,9 @@ public class BulkQueryEntities {
                   dsDataHistories.stream()
                       .map(DelegationSignerData::create)
                       .collect(toImmutableSet()))
+              // Restore the original update timestamp (this gets cleared when we set nameservers or
+              // DS data).
+              .setUpdateTimestamp(domainHistoryLite.domainContent.getUpdateTimestamp())
               .build();
       builder.setDomain(newDomainContent);
     }

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -64,7 +64,7 @@ public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSer
 
   // Store ContactBase instead of ContactResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects
-  @Nullable ContactBase contactBase;
+  @DoNotCompare @Nullable ContactBase contactBase;
 
   @Id
   @Access(AccessType.PROPERTY)

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -779,6 +779,10 @@ public class DomainContent extends EppResource
    */
   void setContactFields(Set<DesignatedContact> contacts, boolean includeRegistrant) {
     // Set the individual contact fields.
+    billingContact = techContact = adminContact = null;
+    if (includeRegistrant) {
+      registrantContact = null;
+    }
     for (DesignatedContact contact : contacts) {
       switch (contact.getType()) {
         case BILLING:
@@ -895,6 +899,7 @@ public class DomainContent extends EppResource
 
     public B setDsData(ImmutableSet<DelegationSignerData> dsData) {
       getInstance().dsData = dsData;
+      getInstance().resetUpdateTimestamp();
       return thisCastToDerived();
     }
 
@@ -918,11 +923,13 @@ public class DomainContent extends EppResource
 
     public B setNameservers(VKey<HostResource> nameserver) {
       getInstance().nsHosts = ImmutableSet.of(nameserver);
+      getInstance().resetUpdateTimestamp();
       return thisCastToDerived();
     }
 
     public B setNameservers(ImmutableSet<VKey<HostResource>> nameservers) {
       getInstance().nsHosts = forceEmptyToNull(nameservers);
+      getInstance().resetUpdateTimestamp();
       return thisCastToDerived();
     }
 
@@ -1032,17 +1039,20 @@ public class DomainContent extends EppResource
 
     public B setGracePeriods(ImmutableSet<GracePeriod> gracePeriods) {
       getInstance().gracePeriods = gracePeriods;
+      getInstance().resetUpdateTimestamp();
       return thisCastToDerived();
     }
 
     public B addGracePeriod(GracePeriod gracePeriod) {
       getInstance().gracePeriods = union(getInstance().getGracePeriods(), gracePeriod);
+      getInstance().resetUpdateTimestamp();
       return thisCastToDerived();
     }
 
     public B removeGracePeriod(GracePeriod gracePeriod) {
       getInstance().gracePeriods =
           CollectionUtils.difference(getInstance().getGracePeriods(), gracePeriod);
+      getInstance().resetUpdateTimestamp();
       return thisCastToDerived();
     }
 

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -86,7 +86,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
 
   // Store DomainContent instead of DomainBase so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects
-  @Nullable DomainContent domainContent;
+  @DoNotCompare @Nullable DomainContent domainContent;
 
   @Id
   @Access(AccessType.PROPERTY)
@@ -105,6 +105,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   // We could have reused domainContent.nsHosts here, but Hibernate throws a weird exception after
   // we change to use a composite primary key.
   // TODO(b/166776754): Investigate if we can reuse domainContent.nsHosts for storing host keys.
+  @DoNotCompare
   @ElementCollection
   @JoinTable(
       name = "DomainHistoryHost",
@@ -118,6 +119,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   @Column(name = "host_repo_id")
   Set<VKey<HostResource>> nsHosts;
 
+  @DoNotCompare
   @OneToMany(
       cascade = {CascadeType.ALL},
       fetch = FetchType.EAGER,
@@ -137,6 +139,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   // HashSet rather than ImmutableSet so that Hibernate can fill them out lazily on request
   Set<DomainDsDataHistory> dsDataHistories = new HashSet<>();
 
+  @DoNotCompare
   @OneToMany(
       cascade = {CascadeType.ALL},
       fetch = FetchType.EAGER,

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -66,7 +66,7 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial
 
   // Store HostBase instead of HostResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects
-  @Nullable HostBase hostBase;
+  @DoNotCompare @Nullable HostBase hostBase;
 
   @Id
   @Access(AccessType.PROPERTY)

core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java

@@ -41,7 +41,7 @@ import org.joda.time.DateTime;
 
 /** Wrapper for {@link Supplier} that associates a time with each attempt. */
 @DeleteAfterMigration
-class CommitLoggedWork<R> implements Runnable {
+public class CommitLoggedWork<R> implements Runnable {
 
   private final Supplier<R> work;
   private final Clock clock;

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -46,6 +46,7 @@ import static google.registry.util.X509Utils.loadCertificate;
 import static java.util.Comparator.comparing;
 import static java.util.function.Predicate.isEqual;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
@@ -991,6 +992,16 @@ public class Registrar extends ImmutableObject
       return this;
     }
 
+    /**
+     * This lets tests set the update timestamp in cases where setting fields resets the timestamp
+     * and breaks the verification that an object has not been updated since it was copied.
+     */
+    @VisibleForTesting
+    public Builder setLastUpdateTime(DateTime timestamp) {
+      getInstance().lastUpdateTime = UpdateAutoTimestamp.create(timestamp);
+      return this;
+    }
+
     /** Build the registrar, nullifying empty fields. */
     @Override
     public Registrar build() {

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -59,6 +59,10 @@ public class ReplicateToDatastoreAction implements Runnable {
   public static final String PATH = "/_dr/cron/replicateToDatastore";
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
+  /** Name of the lock that ensures sequential execution of replays. */
+  public static final String REPLICATE_TO_DATASTORE_LOCK_NAME =
+      ReplicateToDatastoreAction.class.getSimpleName();
+
   /**
    * Number of transactions to fetch from SQL. The rationale for 200 is that we're processing these
    * every minute and our production instance currently does about 2 mutations per second, so this
@@ -66,7 +70,7 @@ public class ReplicateToDatastoreAction implements Runnable {
    */
   public static final int BATCH_SIZE = 200;
 
-  private static final Duration LEASE_LENGTH = standardHours(1);
+  public static final Duration REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH = standardHours(1);
 
   private final Clock clock;
   private final RequestStatusChecker requestStatusChecker;
@@ -81,21 +85,26 @@ public class ReplicateToDatastoreAction implements Runnable {
   }
 
   @VisibleForTesting
-  public List<TransactionEntity> getTransactionBatch() {
+  public List<TransactionEntity> getTransactionBatchAtSnapshot() {
+    return getTransactionBatchAtSnapshot(Optional.empty());
+  }
+
+  static List<TransactionEntity> getTransactionBatchAtSnapshot(Optional<String> snapshotId) {
     // Get the next batch of transactions that we haven't replicated.
     LastSqlTransaction lastSqlTxnBeforeBatch = ofyTm().transact(LastSqlTransaction::load);
     try {
       return jpaTm()
           .transactWithoutBackup(
-              () ->
-                  jpaTm()
-                      .query(
-                          "SELECT txn FROM TransactionEntity txn WHERE id >"
-                              + " :lastId ORDER BY id",
-                          TransactionEntity.class)
-                      .setParameter("lastId", lastSqlTxnBeforeBatch.getTransactionId())
-                      .setMaxResults(BATCH_SIZE)
-                      .getResultList());
+              () -> {
+                snapshotId.ifPresent(jpaTm()::setDatabaseSnapshot);
+                return jpaTm()
+                    .query(
+                        "SELECT txn FROM TransactionEntity txn WHERE id >" + " :lastId ORDER BY id",
+                        TransactionEntity.class)
+                    .setParameter("lastId", lastSqlTxnBeforeBatch.getTransactionId())
+                    .setMaxResults(BATCH_SIZE)
+                    .getResultList();
+              });
     } catch (NoResultException e) {
       return ImmutableList.of();
     }
@@ -108,7 +117,7 @@ public class ReplicateToDatastoreAction implements Runnable {
    * <p>Throws an exception if a fatal error occurred and the batch should be aborted
    */
   @VisibleForTesting
-  public void applyTransaction(TransactionEntity txnEntity) {
+  public static void applyTransaction(TransactionEntity txnEntity) {
     logger.atInfo().log("Applying a single transaction Cloud SQL -> Cloud Datastore.");
     try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
         UpdateAutoTimestamp.disableAutoUpdate()) {
@@ -174,7 +183,11 @@ public class ReplicateToDatastoreAction implements Runnable {
     }
     Optional<Lock> lock =
         Lock.acquireSql(
-            this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
+            REPLICATE_TO_DATASTORE_LOCK_NAME,
+            null,
+            REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH,
+            requestStatusChecker,
+            false);
     if (!lock.isPresent()) {
       String message = "Can't acquire ReplicateToDatastoreAction lock, aborting.";
       logger.atSevere().log(message);
@@ -203,10 +216,14 @@ public class ReplicateToDatastoreAction implements Runnable {
   }
 
   private int replayAllTransactions() {
+    return replayAllTransactions(Optional.empty());
+  }
+
+  public static int replayAllTransactions(Optional<String> snapshotId) {
     int numTransactionsReplayed = 0;
     List<TransactionEntity> transactionBatch;
     do {
-      transactionBatch = getTransactionBatch();
+      transactionBatch = getTransactionBatchAtSnapshot(snapshotId);
       for (TransactionEntity transaction : transactionBatch) {
         applyTransaction(transaction);
         numTransactionsReplayed++;

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -324,7 +324,7 @@ public class HistoryEntry extends ImmutableObject
     // Note: how we wish to treat this Hibernate setter depends on the current state of the object
     // and what's passed in. The key principle is that we wish to maintain the link between parent
     // and child objects, meaning that we should keep around whichever of the two sets (the
-    // parameter vs the class variable and clear/populate that as appropriate.
+    // parameter vs the class variable) and clear/populate that as appropriate.
     //
     // If the class variable is a PersistentSet and we overwrite it here, Hibernate will throw
     // an exception "A collection with cascade=”all-delete-orphan” was no longer referenced by the
@@ -539,7 +539,7 @@ public class HistoryEntry extends ImmutableObject
 
     public B setDomainTransactionRecords(
         ImmutableSet<DomainTransactionRecord> domainTransactionRecords) {
-      getInstance().domainTransactionRecords = domainTransactionRecords;
+      getInstance().setDomainTransactionRecords(domainTransactionRecords);
       return thisCastToDerived();
     }
   }

core/src/main/java/google/registry/model/translators/CreateAutoTimestampTranslatorFactory.java

@@ -14,12 +14,10 @@
 
 package google.registry.model.translators;
 
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.joda.time.DateTimeZone.UTC;
 
 import google.registry.model.CreateAutoTimestamp;
-import google.registry.persistence.transaction.Transaction;
 import java.util.Date;
 import org.joda.time.DateTime;
 
@@ -47,13 +45,13 @@ public class CreateAutoTimestampTranslatorFactory
       /** Save a timestamp, setting it to the current time if it did not have a previous value. */
       @Override
       public Date saveValue(CreateAutoTimestamp pojoValue) {
-
-        // Don't do this if we're in the course of transaction serialization.
-        if (Transaction.inSerializationMode()) {
-          return pojoValue.getTimestamp() == null ? null : pojoValue.getTimestamp().toDate();
-        }
-
-        return firstNonNull(pojoValue.getTimestamp(), ofyTm().getTransactionTime()).toDate();
+        // Note that we use the current transaction manager -- we need to do this under JPA when we
+        // serialize the entity from a Transaction object, but we need to use the JPA transaction
+        // manager in that case.
+        return (pojoValue.getTimestamp() == null
+                ? tm().getTransactionTime()
+                : pojoValue.getTimestamp())
+            .toDate();
       }
     };
   }

core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.translators;
 
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static org.joda.time.DateTimeZone.UTC;
 
@@ -48,9 +50,16 @@ public class UpdateAutoTimestampTranslatorFactory
       @Override
       public Date saveValue(UpdateAutoTimestamp pojoValue) {
 
-        // Don't do this if we're in the course of transaction serialization.
+        // If we're in the course of Transaction serialization, we have to use the transaction time
+        // here and the JPA transaction manager which is what will ultimately be saved during the
+        // commit.
+        // Note that this branch doesn't respect "auto update disabled", as this state is
+        // specifically to address replay, so we add a runtime check for this.
         if (Transaction.inSerializationMode()) {
-          return pojoValue.getTimestamp() == null ? null : pojoValue.getTimestamp().toDate();
+          checkState(
+              UpdateAutoTimestamp.autoUpdateEnabled(),
+              "Auto-update disabled during transaction serialization.");
+          return jpaTm().getTransactionTime().toDate();
         }
 
         return UpdateAutoTimestamp.autoUpdateEnabled()

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -21,6 +21,7 @@ import google.registry.backup.CommitLogCheckpointAction;
 import google.registry.backup.DeleteOldCommitLogsAction;
 import google.registry.backup.ExportCommitLogDiffAction;
 import google.registry.backup.ReplayCommitLogsToSqlAction;
+import google.registry.backup.SyncDatastoreToSqlSnapshotAction;
 import google.registry.batch.BatchModule;
 import google.registry.batch.DeleteContactsAndHostsAction;
 import google.registry.batch.DeleteExpiredDomainsAction;
@@ -199,6 +200,8 @@ interface BackendRequestComponent {
 
   SendExpiringCertificateNotificationEmailAction sendExpiringCertificateNotificationEmailAction();
 
+  SyncDatastoreToSqlSnapshotAction syncDatastoreToSqlSnapshotAction();
+
   SyncGroupMembersAction syncGroupMembersAction();
 
   SyncRegistrarsSheetAction syncRegistrarsSheetAction();

core/src/main/java/google/registry/module/frontend/FrontendComponent.java

@@ -17,6 +17,7 @@ package google.registry.module.frontend;
 import com.google.monitoring.metrics.MetricReporter;
 import dagger.Component;
 import dagger.Lazy;
+import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.flows.ServerTridProviderModule;
@@ -45,10 +46,12 @@ import javax.inject.Singleton;
 @Component(
     modules = {
       AuthModule.class,
+      CloudTasksUtilsModule.class,
       ConfigModule.class,
       ConsoleConfigModule.class,
       CredentialModule.class,
       CustomLogicFactoryModule.class,
+      CloudTasksUtilsModule.class,
       DirectoryModule.class,
       DummyKeyringModule.class,
       FrontendRequestComponentModule.class,

core/src/main/java/google/registry/module/tools/ToolsComponent.java

@@ -17,6 +17,7 @@ package google.registry.module.tools;
 import com.google.monitoring.metrics.MetricReporter;
 import dagger.Component;
 import dagger.Lazy;
+import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.export.DriveModule;
@@ -49,6 +50,7 @@ import javax.inject.Singleton;
       ConfigModule.class,
       CredentialModule.class,
       CustomLogicFactoryModule.class,
+      CloudTasksUtilsModule.class,
       DatastoreServiceModule.class,
       DirectoryModule.class,
       DummyKeyringModule.class,

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
+import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
@@ -604,6 +605,13 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
       managedEntity = getEntityManager().merge(entity);
     }
     getEntityManager().remove(managedEntity);
+
+    // We check shouldReplicate() in TransactionInfo.addDelete(), but we have to check it here as
+    // well prior to attempting to create a datastore key because a non-replicated entity may not
+    // have one.
+    if (shouldReplicate(entity.getClass())) {
+      transactionInfo.get().addDelete(VKey.from(Key.create(entity)));
+    }
     return managedEntity;
   }
 
@@ -827,6 +835,12 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     replaySqlToDatastoreOverrideForTest.set(Optional.empty());
   }
 
+  /** Returns true if the entity class should be replicated from SQL to datastore. */
+  private static boolean shouldReplicate(Class<?> entityClass) {
+    return !NonReplicatedEntity.class.isAssignableFrom(entityClass)
+        && !SqlOnlyEntity.class.isAssignableFrom(entityClass);
+  }
+
   private static class TransactionInfo {
     ReadOnlyCheckingEntityManager entityManager;
     boolean inTransaction = false;
@@ -883,12 +897,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
       }
     }
 
-    /** Returns true if the entity class should be replicated from SQL to datastore. */
-    private boolean shouldReplicate(Class<?> entityClass) {
-      return !NonReplicatedEntity.class.isAssignableFrom(entityClass)
-          && !SqlOnlyEntity.class.isAssignableFrom(entityClass);
-    }
-
     private void recordTransaction() {
       if (contentsBuilder != null) {
         Transaction persistedTxn = contentsBuilder.build();

core/src/main/java/google/registry/rde/RdeStagingReducer.java

@@ -233,14 +233,14 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
               if (mode == RdeMode.FULL) {
                 cloudTasksUtils.enqueue(
                     "rde-upload",
-                    CloudTasksUtils.createPostTask(
+                    cloudTasksUtils.createPostTask(
                         RdeUploadAction.PATH,
                         Service.BACKEND.toString(),
                         ImmutableMultimap.of(RequestParameters.PARAM_TLD, tld)));
               } else {
                 cloudTasksUtils.enqueue(
                     "brda",
-                    CloudTasksUtils.createPostTask(
+                    cloudTasksUtils.createPostTask(
                         BrdaCopyAction.PATH,
                         Service.BACKEND.toString(),
                         ImmutableMultimap.of(

core/src/main/java/google/registry/rde/RdeUploadAction.java

@@ -134,7 +134,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
     }
     cloudTasksUtils.enqueue(
         RDE_REPORT_QUEUE,
-        CloudTasksUtils.createPostTask(
+        cloudTasksUtils.createPostTask(
             RdeReportAction.PATH, Service.BACKEND.getServiceId(), params));
   }
 

core/src/main/java/google/registry/reporting/ReportingModule.java

@@ -40,6 +40,8 @@ public class ReportingModule {
 
   public static final String BEAM_QUEUE = "beam-reporting";
 
+  /** The amount of time expected for the Dataflow jobs to complete. */
+  public static final int ENQUEUE_DELAY_MINUTES = 10;
   /**
    * The request parameter name used by reporting actions that takes a year/month parameter, which
    * defaults to the last month.

core/src/main/java/google/registry/reporting/ReportingUtils.java

@@ -1,38 +0,0 @@
-// Copyright 2018 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.reporting;
-
-import com.google.appengine.api.taskqueue.QueueFactory;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import java.util.Map;
-import org.joda.time.Duration;
-import org.joda.time.YearMonth;
-
-/** Static methods common to various reporting tasks. */
-public class ReportingUtils {
-
-  private static final int ENQUEUE_DELAY_MINUTES = 10;
-
-  /** Enqueues a task that takes a Beam jobId and the {@link YearMonth} as parameters. */
-  public static void enqueueBeamReportingTask(String path, Map<String, String> parameters) {
-    TaskOptions publishTask =
-        TaskOptions.Builder.withUrl(path)
-            .method(TaskOptions.Method.POST)
-            // Dataflow jobs tend to take about 10 minutes to complete.
-            .countdownMillis(Duration.standardMinutes(ENQUEUE_DELAY_MINUTES).getMillis());
-    parameters.forEach(publishTask::param);
-    QueueFactory.getQueue(ReportingModule.BEAM_QUEUE).add(publishTask);
-  }
-}

core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java

@@ -17,7 +17,6 @@ package google.registry.reporting.billing;
 import static google.registry.beam.BeamUtils.createJobName;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase.CLOUD_SQL;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.reporting.ReportingUtils.enqueueBeamReportingTask;
 import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -27,6 +26,7 @@ import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
@@ -35,14 +35,16 @@ import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDataba
 import google.registry.persistence.PersistenceModule;
 import google.registry.reporting.ReportingModule;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.io.IOException;
-import java.util.Map;
 import javax.inject.Inject;
+import org.joda.time.Duration;
 import org.joda.time.YearMonth;
 
 /**
@@ -76,6 +78,7 @@ public class GenerateInvoicesAction implements Runnable {
   private final Response response;
   private final Dataflow dataflow;
   private final PrimaryDatabase database;
+  private final CloudTasksUtils cloudTasksUtils;
 
   @Inject
   GenerateInvoicesAction(
@@ -88,6 +91,7 @@ public class GenerateInvoicesAction implements Runnable {
       @Parameter(RequestParameters.PARAM_DATABASE) PrimaryDatabase database,
       YearMonth yearMonth,
       BillingEmailUtils emailUtils,
+      CloudTasksUtils cloudTasksUtils,
       Clock clock,
       Response response,
       Dataflow dataflow) {
@@ -105,6 +109,7 @@ public class GenerateInvoicesAction implements Runnable {
     this.database = database;
     this.yearMonth = yearMonth;
     this.emailUtils = emailUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
     this.clock = clock;
     this.response = response;
     this.dataflow = dataflow;
@@ -144,13 +149,17 @@ public class GenerateInvoicesAction implements Runnable {
       logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
       String jobId = launchResponse.getJob().getId();
       if (shouldPublish) {
-        Map<String, String> beamTaskParameters =
-            ImmutableMap.of(
-                ReportingModule.PARAM_JOB_ID,
-                jobId,
-                ReportingModule.PARAM_YEAR_MONTH,
-                yearMonth.toString());
-        enqueueBeamReportingTask(PublishInvoicesAction.PATH, beamTaskParameters);
+        cloudTasksUtils.enqueue(
+            ReportingModule.BEAM_QUEUE,
+            cloudTasksUtils.createPostTaskWithDelay(
+                PublishInvoicesAction.PATH,
+                Service.BACKEND.toString(),
+                ImmutableMultimap.of(
+                    ReportingModule.PARAM_JOB_ID,
+                    jobId,
+                    ReportingModule.PARAM_YEAR_MONTH,
+                    yearMonth.toString()),
+                Duration.standardMinutes(ReportingModule.ENQUEUE_DELAY_MINUTES)));
       }
       response.setStatus(SC_OK);
       response.setPayload(String.format("Launched invoicing pipeline: %s", jobId));

core/src/main/java/google/registry/reporting/billing/PublishInvoicesAction.java

@@ -125,7 +125,7 @@ public class PublishInvoicesAction implements Runnable {
   private void enqueueCopyDetailReportsTask() {
     cloudTasksUtils.enqueue(
         BillingModule.CRON_QUEUE,
-        CloudTasksUtils.createPostTask(
+        cloudTasksUtils.createPostTask(
             CopyDetailReportsAction.PATH,
             Service.BACKEND.toString(),
             ImmutableMultimap.of(PARAM_YEAR_MONTH, yearMonth.toString())));

core/src/main/java/google/registry/reporting/icann/IcannReportingStagingAction.java

@@ -21,9 +21,6 @@ import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.appengine.api.taskqueue.QueueFactory;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -33,9 +30,11 @@ import google.registry.bigquery.BigqueryJobFailureException;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.reporting.icann.IcannReportingModule.ReportType;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.Retrier;
 import google.registry.util.SendEmailService;
@@ -86,6 +85,7 @@ public final class IcannReportingStagingAction implements Runnable {
   @Inject @Config("gSuiteOutgoingEmailAddress") InternetAddress sender;
   @Inject @Config("alertRecipientEmailAddress") InternetAddress recipient;
   @Inject SendEmailService emailService;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
   @Inject IcannReportingStagingAction() {}
 
@@ -119,11 +119,13 @@ public final class IcannReportingStagingAction implements Runnable {
             response.setPayload("Completed staging action.");
 
             logger.atInfo().log("Enqueueing report upload.");
-            TaskOptions uploadTask =
-                TaskOptions.Builder.withUrl(IcannReportingUploadAction.PATH)
-                    .method(Method.POST)
-                    .countdownMillis(Duration.standardMinutes(2).getMillis());
-            QueueFactory.getQueue(CRON_QUEUE).add(uploadTask);
+            cloudTasksUtils.enqueue(
+                CRON_QUEUE,
+                cloudTasksUtils.createPostTaskWithDelay(
+                    IcannReportingUploadAction.PATH,
+                    Service.BACKEND.toString(),
+                    null,
+                    Duration.standardMinutes(2)));
             return null;
           },
           BigqueryJobFailureException.class);

core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java

@@ -16,7 +16,6 @@ package google.registry.reporting.spec11;
 
 import static google.registry.beam.BeamUtils.createJobName;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.reporting.ReportingUtils.enqueueBeamReportingTask;
 import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -26,6 +25,7 @@ import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
@@ -34,14 +34,16 @@ import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
 import google.registry.reporting.ReportingModule;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.io.IOException;
-import java.util.Map;
 import javax.inject.Inject;
+import org.joda.time.Duration;
 import org.joda.time.LocalDate;
 
 /**
@@ -73,6 +75,7 @@ public class GenerateSpec11ReportAction implements Runnable {
   private final Dataflow dataflow;
   private final PrimaryDatabase database;
   private final boolean sendEmail;
+  private final CloudTasksUtils cloudTasksUtils;
 
   @Inject
   GenerateSpec11ReportAction(
@@ -86,7 +89,8 @@ public class GenerateSpec11ReportAction implements Runnable {
       @Parameter(ReportingModule.SEND_EMAIL) boolean sendEmail,
       Clock clock,
       Response response,
-      Dataflow dataflow) {
+      Dataflow dataflow,
+      CloudTasksUtils cloudTasksUtils) {
     this.projectId = projectId;
     this.jobRegion = jobRegion;
     this.stagingBucketUrl = stagingBucketUrl;
@@ -101,6 +105,7 @@ public class GenerateSpec11ReportAction implements Runnable {
     this.response = response;
     this.dataflow = dataflow;
     this.sendEmail = sendEmail;
+    this.cloudTasksUtils = cloudTasksUtils;
   }
 
   @Override
@@ -136,11 +141,18 @@ public class GenerateSpec11ReportAction implements Runnable {
               .execute();
       logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
       String jobId = launchResponse.getJob().getId();
-      Map<String, String> beamTaskParameters =
-          ImmutableMap.of(
-              ReportingModule.PARAM_JOB_ID, jobId, ReportingModule.PARAM_DATE, date.toString());
       if (sendEmail) {
-        enqueueBeamReportingTask(PublishSpec11ReportAction.PATH, beamTaskParameters);
+        cloudTasksUtils.enqueue(
+            ReportingModule.BEAM_QUEUE,
+            cloudTasksUtils.createPostTaskWithDelay(
+                PublishSpec11ReportAction.PATH,
+                Service.BACKEND.toString(),
+                ImmutableMultimap.of(
+                    ReportingModule.PARAM_JOB_ID,
+                    jobId,
+                    ReportingModule.PARAM_DATE,
+                    date.toString()),
+                Duration.standardMinutes(ReportingModule.ENQUEUE_DELAY_MINUTES)));
       }
       response.setStatus(SC_OK);
       response.setPayload(String.format("Launched Spec11 pipeline: %s", jobId));

core/src/main/java/google/registry/tmch/NordnUploadAction.java

@@ -39,8 +39,11 @@ import com.google.appengine.api.urlfetch.HTTPRequest;
 import com.google.appengine.api.urlfetch.HTTPResponse;
 import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.apphosting.api.DeadlineExceededException;
+import com.google.common.base.Joiner;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.request.Action;
@@ -125,15 +128,19 @@ public final class NordnUploadAction implements Runnable {
    * delimited String.
    */
   static String convertTasksToCsv(List<TaskHandle> tasks, DateTime now, String columns) {
-    String header = String.format("1,%s,%d\n%s\n", now, tasks.size(), columns);
-    StringBuilder csv = new StringBuilder(header);
+    // Use a Set for deduping purposes so we can be idempotent in case tasks happened to be
+    // enqueued multiple times for a given domain create.
+    ImmutableSortedSet.Builder<String> builder =
+        new ImmutableSortedSet.Builder<>(Ordering.natural());
     for (TaskHandle task : checkNotNull(tasks)) {
       String payload = new String(task.getPayload(), UTF_8);
       if (!Strings.isNullOrEmpty(payload)) {
-        csv.append(payload).append("\n");
+        builder.add(payload + '\n');
       }
     }
-    return csv.toString();
+    ImmutableSortedSet<String> csvLines = builder.build();
+    String header = String.format("1,%s,%d\n%s\n", now, csvLines.size(), columns);
+    return header + Joiner.on("").join(csvLines);
   }
 
   /** Leases and returns all tasks from the queue with the specified tag tld, in batches. */
@@ -168,6 +175,11 @@ public final class NordnUploadAction implements Runnable {
                 : LordnTaskUtils.QUEUE_CLAIMS);
     String columns = phase.equals(PARAM_LORDN_PHASE_SUNRISE) ? COLUMNS_SUNRISE : COLUMNS_CLAIMS;
     List<TaskHandle> tasks = loadAllTasks(queue, tld);
+    // Note: This upload/task deletion isn't done atomically (it's not clear how one would do so
+    // anyway). As a result, it is possible that the upload might succeed yet the deletion of
+    // enqueued tasks might fail. If so, this would result in the same lines being uploaded to NORDN
+    // across mulitple uploads. This is probably OK; all that we really cannot have is a missing
+    // line.
     if (!tasks.isEmpty()) {
       String csvData = convertTasksToCsv(tasks, now, columns);
       uploadCsvToLordn(String.format("/LORDN/%s/%s", tld, phase), csvData);

core/src/main/java/google/registry/tools/DomainLockUtils.java

@@ -15,14 +15,16 @@
 package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
 import static google.registry.model.EppResourceUtils.loadByForeignKeyCached;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
-import google.registry.batch.AsyncTaskEnqueuer;
+import google.registry.batch.RelockDomainAction;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -32,6 +34,8 @@ import google.registry.model.domain.RegistryLock;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
 import google.registry.model.tld.RegistryLockDao;
+import google.registry.request.Action.Service;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.StringGenerator;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -53,16 +57,16 @@ public final class DomainLockUtils {
 
   private final StringGenerator stringGenerator;
   private final String registryAdminRegistrarId;
-  private final AsyncTaskEnqueuer asyncTaskEnqueuer;
+  private CloudTasksUtils cloudTasksUtils;
 
   @Inject
   public DomainLockUtils(
       @Named("base58StringGenerator") StringGenerator stringGenerator,
       @Config("registryAdminClientId") String registryAdminRegistrarId,
-      AsyncTaskEnqueuer asyncTaskEnqueuer) {
+      CloudTasksUtils cloudTasksUtils) {
     this.stringGenerator = stringGenerator;
     this.registryAdminRegistrarId = registryAdminRegistrarId;
-    this.asyncTaskEnqueuer = asyncTaskEnqueuer;
+    this.cloudTasksUtils = cloudTasksUtils;
   }
 
   /**
@@ -203,10 +207,38 @@ public final class DomainLockUtils {
 
   private void submitRelockIfNecessary(RegistryLock lock) {
     if (lock.getRelockDuration().isPresent()) {
-      asyncTaskEnqueuer.enqueueDomainRelock(lock);
+      enqueueDomainRelock(lock);
     }
   }
 
+  /**
+   * Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked.
+   *
+   * <p>Note: the relockDuration must be present on the lock object.
+   */
+  public void enqueueDomainRelock(RegistryLock lock) {
+    checkArgument(
+        lock.getRelockDuration().isPresent(),
+        "Lock with ID %s not configured for relock",
+        lock.getRevisionId());
+    enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
+  }
+
+  /** Enqueues a task to asynchronously re-lock a registry-locked domain after it was unlocked. */
+  public void enqueueDomainRelock(Duration countdown, long lockRevisionId, int previousAttempts) {
+    cloudTasksUtils.enqueue(
+        QUEUE_ASYNC_ACTIONS,
+        cloudTasksUtils.createPostTaskWithDelay(
+            RelockDomainAction.PATH,
+            Service.BACKEND.toString(),
+            ImmutableMultimap.of(
+                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
+                String.valueOf(lockRevisionId),
+                RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM,
+                String.valueOf(previousAttempts)),
+            countdown));
+  }
+
   private void setAsRelock(RegistryLock newLock) {
     jpaTm()
         .transact(

core/src/main/java/google/registry/tools/EncryptEscrowDepositCommand.java

@@ -18,6 +18,7 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
+import google.registry.model.rde.RdeMode;
 import google.registry.tools.params.PathParameter;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -46,11 +47,20 @@ class EncryptEscrowDepositCommand implements CommandWithRemoteApi {
       validateWith = PathParameter.OutputDirectory.class)
   private Path outdir = Paths.get(".");
 
-  @Inject
-  EscrowDepositEncryptor encryptor;
+  @Parameter(
+      names = {"-m", "--mode"},
+      description = "Specify the escrow mode, FULL for RDE and THIN for BRDA.")
+  private RdeMode mode = RdeMode.FULL;
+
+  @Parameter(
+      names = {"-r", "--revision"},
+      description = "Specify the revision.")
+  private int revision = 0;
+
+  @Inject EscrowDepositEncryptor encryptor;
 
   @Override
   public final void run() throws Exception {
-    encryptor.encrypt(canonicalizeDomainName(tld), input, outdir);
+    encryptor.encrypt(mode, canonicalizeDomainName(tld), revision, input, outdir);
   }
 }

core/src/main/java/google/registry/tools/EscrowDepositEncryptor.java

@@ -18,6 +18,7 @@ import static google.registry.model.rde.RdeMode.FULL;
 
 import com.google.common.io.ByteStreams;
 import google.registry.keyring.api.KeyModule.Key;
+import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.rde.RdeUtil;
 import google.registry.rde.RydeEncoder;
@@ -42,26 +43,44 @@ final class EscrowDepositEncryptor {
 
   @Inject @Key("rdeSigningKey") Provider<PGPKeyPair> rdeSigningKey;
   @Inject @Key("rdeReceiverKey") Provider<PGPPublicKey> rdeReceiverKey;
+
+  @Inject
+  @Key("brdaSigningKey")
+  Provider<PGPKeyPair> brdaSigningKey;
+
+  @Inject
+  @Key("brdaReceiverKey")
+  Provider<PGPPublicKey> brdaReceiverKey;
+
   @Inject EscrowDepositEncryptor() {}
 
   /** Creates a {@code .ryde} and {@code .sig} file, provided an XML deposit file. */
-  void encrypt(String tld, Path xmlFile, Path outdir)
+  void encrypt(RdeMode mode, String tld, Integer revision, Path xmlFile, Path outdir)
       throws IOException, XmlException {
     try (InputStream xmlFileInput = Files.newInputStream(xmlFile);
         BufferedInputStream xmlInput = new BufferedInputStream(xmlFileInput, PEEK_BUFFER_SIZE)) {
       DateTime watermark = RdeUtil.peekWatermark(xmlInput);
-      String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, 0);
+      String name = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
       Path rydePath = outdir.resolve(name + ".ryde");
       Path sigPath = outdir.resolve(name + ".sig");
       Path pubPath = outdir.resolve(tld + ".pub");
-      PGPKeyPair signingKey = rdeSigningKey.get();
+      PGPKeyPair signingKey;
+      PGPPublicKey receiverKey;
+      if (mode == FULL) {
+        signingKey = rdeSigningKey.get();
+        receiverKey = rdeReceiverKey.get();
+      } else {
+        signingKey = brdaSigningKey.get();
+        receiverKey = brdaReceiverKey.get();
+      }
       try (OutputStream rydeOutput = Files.newOutputStream(rydePath);
           OutputStream sigOutput = Files.newOutputStream(sigPath);
-          RydeEncoder rydeEncoder = new RydeEncoder.Builder()
-              .setRydeOutput(rydeOutput, rdeReceiverKey.get())
-              .setSignatureOutput(sigOutput, signingKey)
-              .setFileMetadata(name, Files.size(xmlFile), watermark)
-              .build()) {
+          RydeEncoder rydeEncoder =
+              new RydeEncoder.Builder()
+                  .setRydeOutput(rydeOutput, receiverKey)
+                  .setSignatureOutput(sigOutput, signingKey)
+                  .setFileMetadata(name, Files.size(xmlFile), watermark)
+                  .build()) {
         ByteStreams.copy(xmlInput, rydeEncoder);
       }
       try (OutputStream pubOutput = Files.newOutputStream(pubPath);

core/src/main/java/google/registry/tools/GenerateEscrowDepositCommand.java

@@ -133,7 +133,7 @@ final class GenerateEscrowDepositCommand implements CommandWithRemoteApi {
     }
     cloudTasksUtils.enqueue(
         RDE_REPORT_QUEUE,
-        CloudTasksUtils.createPostTask(
+        cloudTasksUtils.createPostTask(
             RdeStagingAction.PATH, Service.BACKEND.toString(), paramsBuilder.build()));
   }
 

core/src/main/java/google/registry/tools/LoadTestCommand.java

@@ -27,9 +27,9 @@ import google.registry.model.tld.Registries;
 class LoadTestCommand extends ConfirmingCommand
     implements CommandWithConnection, CommandWithRemoteApi {
 
-  // This is a mostly arbitrary value, roughly an hour and a quarter.  It served as a generous
+  // This is a mostly arbitrary value, roughly two and a half hours.  It served as a generous
   // timespan for initial backup/restore testing, but has no other special significance.
-  private static final int DEFAULT_RUN_SECONDS = 4600;
+  private static final int DEFAULT_RUN_SECONDS = 9200;
 
   @Parameter(
       names = {"--tld"},

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -123,8 +123,10 @@ public final class RegistryTool {
           .put("update_server_locks", UpdateServerLocksCommand.class)
           .put("update_tld", UpdateTldCommand.class)
           .put("upload_claims_list", UploadClaimsListCommand.class)
+          .put("validate_datastore", ValidateDatastoreCommand.class)
           .put("validate_escrow_deposit", ValidateEscrowDepositCommand.class)
           .put("validate_login_credentials", ValidateLoginCredentialsCommand.class)
+          .put("validate_sql", ValidateSqlCommand.class)
           .put("verify_ote", VerifyOteCommand.class)
           .put("whois_query", WhoisQueryCommand.class)
           .build();

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -76,6 +76,7 @@ import javax.inject.Singleton;
       LocalCredentialModule.class,
       PersistenceModule.class,
       RdeModule.class,
+      RegistryToolDataflowModule.class,
       RequestFactoryModule.class,
       SecretManagerModule.class,
       URLFetchServiceModule.class,
@@ -170,10 +171,14 @@ interface RegistryToolComponent {
 
   void inject(UpdateTldCommand command);
 
+  void inject(ValidateDatastoreCommand command);
+
   void inject(ValidateEscrowDepositCommand command);
 
   void inject(ValidateLoginCredentialsCommand command);
 
+  void inject(ValidateSqlCommand command);
+
   void inject(WhoisQueryCommand command);
 
   AppEngineConnection appEngineConnection();

core/src/main/java/google/registry/tools/RegistryToolDataflowModule.java

@@ -0,0 +1,39 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.google.api.services.dataflow.Dataflow;
+import dagger.Module;
+import dagger.Provides;
+import google.registry.config.CredentialModule.LocalCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.GoogleCredentialsBundle;
+
+/** Provides a {@link Dataflow} API client for use in {@link RegistryTool}. */
+@Module
+public class RegistryToolDataflowModule {
+
+  @Provides
+  static Dataflow provideDataflow(
+      @LocalCredential GoogleCredentialsBundle credentialsBundle,
+      @Config("projectId") String projectId) {
+    return new Dataflow.Builder(
+            credentialsBundle.getHttpTransport(),
+            credentialsBundle.getJsonFactory(),
+            credentialsBundle.getHttpRequestInitializer())
+        .setApplicationName(String.format("%s nomulus", projectId))
+        .build();
+  }
+}

core/src/main/java/google/registry/tools/ValidateDatabaseMigrationCommand.java

@@ -0,0 +1,232 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.beam.BeamUtils.createJobName;
+
+import com.beust.jcommander.Parameter;
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.Job;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.tools.params.DateTimeParameter;
+import google.registry.util.Clock;
+import google.registry.util.RequestStatusChecker;
+import google.registry.util.Sleeper;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.UUID;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/** Shared setup for commands that validate the data replication between Datastore and Cloud SQL. */
+abstract class ValidateDatabaseMigrationCommand
+    implements CommandWithConnection, CommandWithRemoteApi {
+
+  private static final String PIPELINE_NAME = "validate_database_pipeline";
+
+  private static final String MANUAL_PIPELINE_LAUNCH_COMMAND_TEMPLATE =
+      "gcloud dataflow flex-template run "
+          + "\"%s-${USER}-$(date +%%Y%%m%%dt%%H%%M%%S)\" "
+          + "--template-file-gcs-location %s "
+          + "--project %s "
+          + "--region=%s "
+          + "--worker-machine-type=n2-standard-8 --num-workers=8 "
+          + "--parameters registryEnvironment=%s "
+          + "--parameters sqlSnapshotId=%s "
+          + "--parameters latestCommitLogTimestamp=%s "
+          + "--parameters diffOutputGcsBucket=%s ";
+
+  // States indicating a job is not finished yet.
+  static final ImmutableSet<String> DATAFLOW_JOB_RUNNING_STATES =
+      ImmutableSet.of(
+          "JOB_STATE_UNKNOWN",
+          "JOB_STATE_RUNNING",
+          "JOB_STATE_STOPPED",
+          "JOB_STATE_PENDING",
+          "JOB_STATE_QUEUED");
+
+  static final Duration JOB_POLLING_INTERVAL = Duration.standardSeconds(60);
+
+  @Parameter(
+      names = {"-m", "--manual"},
+      description =
+          "If true, let user launch the comparison pipeline manually out of band. "
+              + "Command will wait for user key-press to exit after syncing Datastore.")
+  boolean manualLaunchPipeline;
+
+  @Parameter(
+      names = {"-r", "--release"},
+      description = "The release tag of the BEAM pipeline to run. It defaults to 'live'.")
+  String release = "live";
+
+  @Parameter(
+      names = {"-c", "--comparisonStartTimestamp"},
+      description =
+          "When comparing History and Epp Resource entities, ignore those that have not"
+              + " changed since this time.",
+      converter = DateTimeParameter.class)
+  DateTime comparisonStartTimestamp;
+
+  @Parameter(
+      names = {"-o", "--outputBucket"},
+      description =
+          "The GCS bucket where data discrepancies are logged. "
+              + "It defaults to ${projectId}-beam")
+  String outputBucket;
+
+  @Inject Clock clock;
+  @Inject Dataflow dataflow;
+
+  @Inject
+  @Config("defaultJobRegion")
+  String jobRegion;
+
+  @Inject
+  @Config("beamStagingBucketUrl")
+  String stagingBucketUrl;
+
+  @Inject
+  @Config("projectId")
+  String projectId;
+
+  @Inject Sleeper sleeper;
+
+  AppEngineConnection connection;
+
+  @Override
+  public void setConnection(AppEngineConnection connection) {
+    this.connection = connection;
+  }
+
+  String getDataflowJobStatus(String jobId) {
+    try {
+      return dataflow
+          .projects()
+          .locations()
+          .jobs()
+          .get(projectId, jobRegion, jobId)
+          .execute()
+          .getCurrentState();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  void launchPipelineAndWaitUntilFinish(
+      String pipelineName, DatabaseSnapshot snapshot, String latestCommitTimestamp) {
+    Job pipelineJob =
+        launchComparisonPipeline(pipelineName, snapshot.getSnapshotId(), latestCommitTimestamp)
+            .getJob();
+    String jobId = pipelineJob.getId();
+
+    System.out.printf("Launched comparison pipeline %s (%s).\n", pipelineJob.getName(), jobId);
+
+    while (DATAFLOW_JOB_RUNNING_STATES.contains(getDataflowJobStatus(jobId))) {
+      sleeper.sleepInterruptibly(JOB_POLLING_INTERVAL);
+    }
+    System.out.printf(
+        "Pipeline ended with %s state. Please check counters for results.\n",
+        getDataflowJobStatus(jobId));
+  }
+
+  String getOutputBucket() {
+    return Optional.ofNullable(outputBucket).orElse(projectId + "-beam");
+  }
+
+  String getContainerSpecGcsPath() {
+    return String.format(
+        "%s/%s_metadata.json", stagingBucketUrl.replace("live", release), PIPELINE_NAME);
+  }
+
+  String getManualLaunchCommand(
+      String jobName, String snapshotId, String latestCommitLogTimestamp) {
+    String baseCommand =
+        String.format(
+            MANUAL_PIPELINE_LAUNCH_COMMAND_TEMPLATE,
+            jobName,
+            getContainerSpecGcsPath(),
+            projectId,
+            jobRegion,
+            RegistryToolEnvironment.get().name(),
+            snapshotId,
+            latestCommitLogTimestamp,
+            getOutputBucket());
+    if (comparisonStartTimestamp == null) {
+      return baseCommand;
+    }
+    return baseCommand + "--parameters comparisonStartTimestamp=" + comparisonStartTimestamp;
+  }
+
+  LaunchFlexTemplateResponse launchComparisonPipeline(
+      String jobName, String sqlSnapshotId, String latestCommitLogTimestamp) {
+    try {
+      // Hardcode machine type and initial workers to force a quick start.
+      ImmutableMap.Builder<String, String> paramsBuilder =
+          new ImmutableMap.Builder()
+              .put("workerMachineType", "n2-standard-8")
+              .put("numWorkers", "8")
+              .put("sqlSnapshotId", sqlSnapshotId)
+              .put("latestCommitLogTimestamp", latestCommitLogTimestamp)
+              .put("registryEnvironment", RegistryToolEnvironment.get().name())
+              .put("diffOutputGcsBucket", getOutputBucket());
+      if (comparisonStartTimestamp != null) {
+        paramsBuilder.put("comparisonStartTimestamp", comparisonStartTimestamp.toString());
+      }
+      LaunchFlexTemplateParameter parameter =
+          new LaunchFlexTemplateParameter()
+              .setJobName(createJobName(Ascii.toLowerCase(jobName).replace('_', '-'), clock))
+              .setContainerSpecGcsPath(getContainerSpecGcsPath())
+              .setParameters(paramsBuilder.build());
+      return dataflow
+          .projects()
+          .locations()
+          .flexTemplates()
+          .launch(
+              projectId, jobRegion, new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
+          .execute();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * A fake implementation of {@link RequestStatusChecker} for managing SQL-backed locks from
+   * non-AppEngine platforms. This is only required until the Nomulus server is migrated off
+   * AppEngine.
+   */
+  static class FakeRequestStatusChecker implements RequestStatusChecker {
+
+    private final String logId =
+        ValidateDatastoreCommand.class.getSimpleName() + "-" + UUID.randomUUID();
+
+    @Override
+    public String getLogId() {
+      return logId;
+    }
+
+    @Override
+    public boolean isRunning(String requestLogId) {
+      return false;
+    }
+  }
+}

core/src/main/java/google/registry/tools/ValidateDatastoreCommand.java

@@ -0,0 +1,105 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.model.replay.ReplicateToDatastoreAction.REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH;
+import static google.registry.model.replay.ReplicateToDatastoreAction.REPLICATE_TO_DATASTORE_LOCK_NAME;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.net.MediaType;
+import google.registry.backup.SyncDatastoreToSqlSnapshotAction;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.server.Lock;
+import google.registry.request.Action.Service;
+import java.util.Optional;
+
+/**
+ * Validates asynchronously replicated data from the primary Cloud SQL database to Datastore.
+ *
+ * <p>This command suspends the replication process (by acquiring the replication lock), take a
+ * snapshot of the Cloud SQL database, invokes a Nomulus server action to sync Datastore to this
+ * snapshot (See {@link SyncDatastoreToSqlSnapshotAction} for details), and finally launches a BEAM
+ * pipeline to compare Datastore with the given SQL snapshot.
+ *
+ * <p>This command does not lock up the SQL database. Normal processing can proceed.
+ */
+@Parameters(commandDescription = "Validates Datastore with the primary Cloud SQL database.")
+public class ValidateDatastoreCommand extends ValidateDatabaseMigrationCommand {
+
+  private static final Service NOMULUS_SERVICE = Service.BACKEND;
+
+  @Override
+  public void run() throws Exception {
+    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc());
+    if (!state.getReplayDirection().equals(ReplayDirection.SQL_TO_DATASTORE)) {
+      throw new IllegalStateException("Cannot validate Datastore in migration step " + state);
+    }
+    Optional<Lock> lock =
+        Lock.acquireSql(
+            REPLICATE_TO_DATASTORE_LOCK_NAME,
+            null,
+            REPLICATE_TO_DATASTORE_LOCK_LEASE_LENGTH,
+            new FakeRequestStatusChecker(),
+            false);
+    if (!lock.isPresent()) {
+      throw new IllegalStateException("Cannot acquire the async propagation lock.");
+    }
+
+    try {
+      try (DatabaseSnapshot snapshot = DatabaseSnapshot.createSnapshot()) {
+        System.out.printf("Obtained snapshot %s\n", snapshot.getSnapshotId());
+        AppEngineConnection connectionToService = connection.withService(NOMULUS_SERVICE);
+        String response =
+            connectionToService.sendPostRequest(
+                getNomulusEndpoint(snapshot.getSnapshotId()),
+                ImmutableMap.<String, String>of(),
+                MediaType.PLAIN_TEXT_UTF_8,
+                "".getBytes(UTF_8));
+        System.out.println(response);
+
+        lock.ifPresent(Lock::releaseSql);
+        lock = Optional.empty();
+
+        // See SyncDatastoreToSqlSnapshotAction for response format.
+        String latestCommitTimestamp =
+            response.substring(response.lastIndexOf('(') + 1, response.lastIndexOf(')'));
+
+        if (manualLaunchPipeline) {
+          System.out.printf(
+              "To launch the pipeline manually, use the following command:\n%s\n",
+              getManualLaunchCommand(
+                  "validate-datastore", snapshot.getSnapshotId(), latestCommitTimestamp));
+
+          System.out.print("\nEnter any key to continue when the pipeline ends:");
+          System.in.read();
+        } else {
+          launchPipelineAndWaitUntilFinish("validate-datastore", snapshot, latestCommitTimestamp);
+        }
+      }
+    } finally {
+      lock.ifPresent(Lock::releaseSql);
+    }
+  }
+
+  private static String getNomulusEndpoint(String sqlSnapshotId) {
+    return String.format(
+        "%s?sqlSnapshotId=%s", SyncDatastoreToSqlSnapshotAction.PATH, sqlSnapshotId);
+  }
+}

core/src/main/java/google/registry/tools/ValidateSqlCommand.java

@@ -0,0 +1,91 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.backup.ReplayCommitLogsToSqlAction.REPLAY_TO_SQL_LOCK_LEASE_LENGTH;
+import static google.registry.backup.ReplayCommitLogsToSqlAction.REPLAY_TO_SQL_LOCK_NAME;
+
+import com.beust.jcommander.Parameters;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.model.server.Lock;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import java.util.Optional;
+import org.joda.time.DateTime;
+
+/**
+ * Validates asynchronously replicated data from the primary Datastore to Cloud SQL.
+ *
+ * <p>This command suspends the replication process (by acquiring the replication lock), take a
+ * snapshot of the Cloud SQL database, finds the corresponding Datastore snapshot, and finally
+ * launches a BEAM pipeline to compare the two snapshots.
+ *
+ * <p>This command does not lock up either database. Normal processing can proceed.
+ */
+@Parameters(commandDescription = "Validates Cloud SQL with the primary Datastore.")
+public class ValidateSqlCommand extends ValidateDatabaseMigrationCommand {
+
+  @Override
+  public void run() throws Exception {
+    MigrationState state = DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc());
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      throw new IllegalStateException("Cannot validate SQL in migration step " + state);
+    }
+    Optional<Lock> lock =
+        Lock.acquireSql(
+            REPLAY_TO_SQL_LOCK_NAME,
+            null,
+            REPLAY_TO_SQL_LOCK_LEASE_LENGTH,
+            new FakeRequestStatusChecker(),
+            false);
+    if (!lock.isPresent()) {
+      throw new IllegalStateException("Cannot acquire the async propagation lock.");
+    }
+
+    try {
+      DateTime latestCommitLogTime =
+          TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+      try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+        // Eagerly release the commitlog replay lock so that replay can resume.
+        lock.ifPresent(Lock::releaseSql);
+        lock = Optional.empty();
+
+        System.out.printf(
+            "Start comparison with SQL snapshot (%s) and CommitLog timestamp (%s).\n",
+            databaseSnapshot.getSnapshotId(), latestCommitLogTime);
+
+        if (manualLaunchPipeline) {
+          System.out.printf(
+              "To launch the pipeline manually, use the following command:\n%s\n",
+              getManualLaunchCommand(
+                  "validate-sql",
+                  databaseSnapshot.getSnapshotId(),
+                  latestCommitLogTime.toString()));
+
+          System.out.print("\nEnter any key to continue when the pipeline ends:");
+          System.in.read();
+        } else {
+          launchPipelineAndWaitUntilFinish(
+              "validate-sql", databaseSnapshot, latestCommitLogTime.toString());
+        }
+      }
+    } finally {
+      lock.ifPresent(Lock::releaseSql);
+    }
+  }
+}

core/src/main/java/google/registry/ui/server/registrar/RegistrarSettingsAction.java

@@ -19,7 +19,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Sets.difference;
 import static google.registry.config.RegistryEnvironment.PRODUCTION;
-import static google.registry.export.sheet.SyncRegistrarsSheetAction.enqueueRegistrarSheetSync;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.security.JsonResponseHelper.Status.ERROR;
@@ -32,18 +31,21 @@ import com.google.common.base.Strings;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryEnvironment;
+import google.registry.export.sheet.SyncRegistrarsSheetAction;
 import google.registry.flows.certs.CertificateChecker;
 import google.registry.flows.certs.CertificateChecker.InsecureCertificateException;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.registrar.RegistrarContact.Type;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.HttpException.ForbiddenException;
 import google.registry.request.JsonActionRunner;
@@ -58,6 +60,7 @@ import google.registry.ui.forms.FormFieldException;
 import google.registry.ui.server.RegistrarFormFields;
 import google.registry.ui.server.SendEmailUtils;
 import google.registry.util.AppEngineServiceUtils;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.CollectionUtils;
 import google.registry.util.DiffUtils;
 import java.util.HashSet;
@@ -88,6 +91,22 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
   static final String ARGS_PARAM = "args";
   static final String ID_PARAM = "id";
 
+  /**
+   * Allows task enqueueing to be disabled when executing registrar console test cases.
+   *
+   * <p>The existing workflow in UI test cases triggers task enqueueing, which was not an issue with
+   * Task Queue since it's a native App Engine feature simulated by the App Engine SDK's
+   * environment. However, with Cloud Tasks, the server enqueues and fails to deliver to the actual
+   * Cloud Tasks endpoint due to lack of permission.
+   *
+   * <p>One way to allow enqueuing in backend test and avoid enqueuing in UI test is to disable
+   * enqueuing when the test server starts and enable enqueueing once the test server stops. This
+   * can be done by utilizing a ThreadLocal<Boolean> variable isInTestDriver, which is set to false
+   * by default. Enqueuing is allowed only if the value of isInTestDriver is false. It's set to true
+   * in start() and set to false in stop() inside TestDriver.java, a class used in testing.
+   */
+  private static ThreadLocal<Boolean> isInTestDriver = ThreadLocal.withInitial(() -> false);
+
   @Inject JsonActionRunner jsonActionRunner;
   @Inject AppEngineServiceUtils appEngineServiceUtils;
   @Inject RegistrarConsoleMetrics registrarConsoleMetrics;
@@ -95,6 +114,7 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
   @Inject AuthenticatedRegistrarAccessor registrarAccessor;
   @Inject AuthResult authResult;
   @Inject CertificateChecker certificateChecker;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
   @Inject RegistrarSettingsAction() {}
 
@@ -102,6 +122,14 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
     return contact.getPhoneNumber() != null;
   }
 
+  public static void setIsInTestDriverToFalse() {
+    isInTestDriver.set(false);
+  }
+
+  public static void setIsInTestDriverToTrue() {
+    isInTestDriver.set(true);
+  }
+
   @Override
   public void run() {
     jsonActionRunner.run(this);
@@ -170,6 +198,26 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
     }
   }
 
+  @AutoValue
+  abstract static class EmailInfo {
+    abstract Registrar registrar();
+
+    abstract Registrar updatedRegistrar();
+
+    abstract ImmutableSet<RegistrarContact> contacts();
+
+    abstract ImmutableSet<RegistrarContact> updatedContacts();
+
+    static EmailInfo create(
+        Registrar registrar,
+        Registrar updatedRegistrar,
+        ImmutableSet<RegistrarContact> contacts,
+        ImmutableSet<RegistrarContact> updatedContacts) {
+      return new AutoValue_RegistrarSettingsAction_EmailInfo(
+          registrar, updatedRegistrar, contacts, updatedContacts);
+    }
+  }
+
   private RegistrarResult read(String registrarId) {
     return RegistrarResult.create("Success", loadRegistrarUnchecked(registrarId));
   }
@@ -183,72 +231,69 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
   }
 
   private RegistrarResult update(final Map<String, ?> args, String registrarId) {
-    tm().transact(
-            () -> {
-              // We load the registrar here rather than outside of the transaction - to make
-              // sure we have the latest version. This one is loaded inside the transaction, so it's
-              // guaranteed to not change before we update it.
-              Registrar registrar = loadRegistrarUnchecked(registrarId);
-              // Detach the registrar to avoid Hibernate object-updates, since we wish to email
-              // out the diffs between the existing and updated registrar objects
-              if (!tm().isOfy()) {
-                jpaTm().getEntityManager().detach(registrar);
-              }
-              // Verify that the registrar hasn't been changed.
-              // To do that - we find the latest update time (or null if the registrar has been
-              // deleted) and compare to the update time from the args. The update time in the args
-              // comes from the read that gave the UI the data - if it's out of date, then the UI
-              // had out of date data.
-              DateTime latest = registrar.getLastUpdateTime();
-              DateTime latestFromArgs =
-                  RegistrarFormFields.LAST_UPDATE_TIME.extractUntyped(args).get();
-              if (!latestFromArgs.equals(latest)) {
-                logger.atWarning().log(
-                    "Registrar changed since reading the data!"
-                        + " Last updated at %s, but args data last updated at %s.",
-                    latest, latestFromArgs);
-                throw new IllegalStateException(
-                    "Registrar has been changed by someone else. Please reload and retry.");
-              }
-
-              // Keep the current contacts so we can later check that no required contact was
-              // removed, email the changes to the contacts
-              ImmutableSet<RegistrarContact> contacts = registrar.getContacts();
-
-              Registrar updatedRegistrar = registrar;
-              // Do OWNER only updates to the registrar from the request.
-              updatedRegistrar = checkAndUpdateOwnerControlledFields(updatedRegistrar, args);
-              // Do ADMIN only updates to the registrar from the request.
-              updatedRegistrar = checkAndUpdateAdminControlledFields(updatedRegistrar, args);
-
-              // read the contacts from the request.
-              ImmutableSet<RegistrarContact> updatedContacts =
-                  readContacts(registrar, contacts, args);
-
-              // Save the updated contacts
-              if (!updatedContacts.equals(contacts)) {
-                if (!registrarAccessor.hasRoleOnRegistrar(Role.OWNER, registrar.getRegistrarId())) {
-                  throw new ForbiddenException("Only OWNERs can update the contacts");
-                }
-                checkContactRequirements(contacts, updatedContacts);
-                RegistrarContact.updateContacts(updatedRegistrar, updatedContacts);
-                updatedRegistrar =
-                    updatedRegistrar.asBuilder().setContactsRequireSyncing(true).build();
-              }
-
-              // Save the updated registrar
-              if (!updatedRegistrar.equals(registrar)) {
-                tm().put(updatedRegistrar);
-              }
-
-              // Email the updates
-              sendExternalUpdatesIfNecessary(
-                  registrar, contacts, updatedRegistrar, updatedContacts);
-            });
+    // Email the updates
+    sendExternalUpdatesIfNecessary(tm().transact(() -> saveUpdates(args, registrarId)));
     // Reload the result outside of the transaction to get the most recent version
     return RegistrarResult.create("Saved " + registrarId, loadRegistrarUnchecked(registrarId));
   }
 
+  /** Saves the updates and returns info needed for the update email */
+  private EmailInfo saveUpdates(final Map<String, ?> args, String registrarId) {
+    // We load the registrar here rather than outside of the transaction - to make
+    // sure we have the latest version. This one is loaded inside the transaction, so it's
+    // guaranteed to not change before we update it.
+    Registrar registrar = loadRegistrarUnchecked(registrarId);
+    // Detach the registrar to avoid Hibernate object-updates, since we wish to email
+    // out the diffs between the existing and updated registrar objects
+    if (!tm().isOfy()) {
+      jpaTm().getEntityManager().detach(registrar);
+    }
+    // Verify that the registrar hasn't been changed.
+    // To do that - we find the latest update time (or null if the registrar has been
+    // deleted) and compare to the update time from the args. The update time in the args
+    // comes from the read that gave the UI the data - if it's out of date, then the UI
+    // had out of date data.
+    DateTime latest = registrar.getLastUpdateTime();
+    DateTime latestFromArgs = RegistrarFormFields.LAST_UPDATE_TIME.extractUntyped(args).get();
+    if (!latestFromArgs.equals(latest)) {
+      logger.atWarning().log(
+          "Registrar changed since reading the data!"
+              + " Last updated at %s, but args data last updated at %s.",
+          latest, latestFromArgs);
+      throw new IllegalStateException(
+          "Registrar has been changed by someone else. Please reload and retry.");
+    }
+
+    // Keep the current contacts so we can later check that no required contact was
+    // removed, email the changes to the contacts
+    ImmutableSet<RegistrarContact> contacts = registrar.getContacts();
+
+    Registrar updatedRegistrar = registrar;
+    // Do OWNER only updates to the registrar from the request.
+    updatedRegistrar = checkAndUpdateOwnerControlledFields(updatedRegistrar, args);
+    // Do ADMIN only updates to the registrar from the request.
+    updatedRegistrar = checkAndUpdateAdminControlledFields(updatedRegistrar, args);
+
+    // read the contacts from the request.
+    ImmutableSet<RegistrarContact> updatedContacts = readContacts(registrar, contacts, args);
+
+    // Save the updated contacts
+    if (!updatedContacts.equals(contacts)) {
+      if (!registrarAccessor.hasRoleOnRegistrar(Role.OWNER, registrar.getRegistrarId())) {
+        throw new ForbiddenException("Only OWNERs can update the contacts");
+      }
+      checkContactRequirements(contacts, updatedContacts);
+      RegistrarContact.updateContacts(updatedRegistrar, updatedContacts);
+      updatedRegistrar = updatedRegistrar.asBuilder().setContactsRequireSyncing(true).build();
+    }
+
+    // Save the updated registrar
+    if (!updatedRegistrar.equals(registrar)) {
+      tm().put(updatedRegistrar);
+    }
+    return EmailInfo.create(registrar, updatedRegistrar, contacts, updatedContacts);
+  }
+
   private Map<String, Object> expandRegistrarWithContacts(
       Iterable<RegistrarContact> contacts, Registrar registrar) {
     ImmutableSet<Map<String, Object>> expandedContacts =
@@ -408,6 +453,13 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
     Map<?, ?> diffs =
         DiffUtils.deepDiff(
             originalRegistrar.toDiffableFieldMap(), updatedRegistrar.toDiffableFieldMap(), true);
+
+    // It's expected that the update timestamp will be changed, as it gets reset whenever we change
+    // nested collections.  If it's the only change, just return the original registrar.
+    if (diffs.keySet().equals(ImmutableSet.of("lastUpdateTime"))) {
+      return originalRegistrar;
+    }
+
     throw new ForbiddenException(
         String.format("Unauthorized: only %s can change fields %s", allowedRole, diffs.keySet()));
   }
@@ -575,26 +627,30 @@ public class RegistrarSettingsAction implements Runnable, JsonActionRunner.JsonA
    * sends an email with a diff of the changes to the configured notification email address and all
    * contact addresses and enqueues a task to re-sync the registrar sheet.
    */
-  private void sendExternalUpdatesIfNecessary(
-      Registrar existingRegistrar,
-      ImmutableSet<RegistrarContact> existingContacts,
-      Registrar updatedRegistrar,
-      ImmutableSet<RegistrarContact> updatedContacts) {
+  private void sendExternalUpdatesIfNecessary(EmailInfo emailInfo) {
+    ImmutableSet<RegistrarContact> existingContacts = emailInfo.contacts();
     if (!sendEmailUtils.hasRecipients() && existingContacts.isEmpty()) {
       return;
     }
-
+    Registrar existingRegistrar = emailInfo.registrar();
     Map<?, ?> diffs =
         DiffUtils.deepDiff(
             expandRegistrarWithContacts(existingContacts, existingRegistrar),
-            expandRegistrarWithContacts(updatedContacts, updatedRegistrar),
+            expandRegistrarWithContacts(emailInfo.updatedContacts(), emailInfo.updatedRegistrar()),
             true);
     @SuppressWarnings("unchecked")
     Set<String> changedKeys = (Set<String>) diffs.keySet();
     if (CollectionUtils.difference(changedKeys, "lastUpdateTime").isEmpty()) {
       return;
     }
-    enqueueRegistrarSheetSync(appEngineServiceUtils.getCurrentVersionHostname("backend"));
+    if (!isInTestDriver.get()) {
+      // Enqueues a sync registrar sheet task if enqueuing is not triggered by console tests and
+      // there's an update besides the lastUpdateTime
+      cloudTasksUtils.enqueue(
+          SyncRegistrarsSheetAction.QUEUE,
+          cloudTasksUtils.createGetTask(
+              SyncRegistrarsSheetAction.PATH, Service.BACKEND.toString(), ImmutableMultimap.of()));
+    }
     String environment = Ascii.toLowerCase(String.valueOf(RegistryEnvironment.get()));
     sendEmailUtils.sendEmail(
         String.format(

core/src/main/resources/google/registry/beam/validate_database_pipeline_metadata.json

@@ -1,6 +1,6 @@
 {
-  "name": "Validate Datastore with Cloud SQL",
-  "description": "An Apache Beam batch pipeline that compares Datastore with the primary Cloud SQL database.",
+  "name": "Validate Datastore and Cloud SQL",
+  "description": "An Apache Beam batch pipeline that compares the data in Datastore and Cloud SQL database.",
   "parameters": [
     {
       "name": "registryEnvironment",
@@ -24,7 +24,7 @@
       "name": "sqlSnapshotId",
       "label": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
       "helpText": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
-      "is_optional": true
+      "is_optional": false
     },
     {
       "name": "latestCommitLogTimestamp",
@@ -37,6 +37,12 @@
       "label": "Only entities updated at or after this time are included for validation.",
       "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
       "is_optional": true
+    },
+    {
+      "name": "diffOutputGcsBucket",
+      "label": "The GCS bucket where data discrepancies should be output to.",
+      "helpText": "The GCS bucket where data discrepancies should be output to.",
+      "is_optional": true
     }
   ]
 }

core/src/main/resources/google/registry/beam/validate_sql_pipeline_metadata.json

@@ -1,21 +0,0 @@
-{
-  "name": "Validate Cloud SQL with Datastore being primary",
-  "description": "An Apache Beam batch pipeline that compares Cloud SQL with the primary Datastore.",
-  "parameters": [
-    {
-      "name": "registryEnvironment",
-      "label": "The Registry environment.",
-      "helpText": "The Registry environment.",
-      "is_optional": false,
-      "regexes": [
-        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
-      ]
-    },
-    {
-      "name": "comparisonStartTimestamp",
-      "label": "Only entities updated at or after this time are included for validation.",
-      "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
-      "is_optional": true
-    }
-  ]
-}

core/src/test/java/google/registry/batch/AsyncTaskEnqueuerTest.java

@@ -15,7 +15,6 @@
 package google.registry.batch;
 
 import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.common.truth.Truth.assertThat;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
@@ -23,19 +22,16 @@ import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_ACTIONS;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
-import static google.registry.testing.SqlHelper.saveRegistryLock;
 import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
 import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestLogHandlerUtils.assertLogMessage;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardHours;
 import static org.joda.time.Duration.standardSeconds;
-import static org.junit.Assert.assertThrows;
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableSortedSet;
 import google.registry.model.contact.ContactResource;
-import google.registry.model.domain.RegistryLock;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
@@ -142,59 +138,4 @@ public class AsyncTaskEnqueuerTest {
     assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
     assertLogMessage(logHandler, Level.INFO, "Ignoring async re-save");
   }
-
-  @Test
-  void testEnqueueRelock() {
-    RegistryLock lock =
-        saveRegistryLock(
-            new RegistryLock.Builder()
-                .setLockCompletionTime(clock.nowUtc())
-                .setUnlockRequestTime(clock.nowUtc())
-                .setUnlockCompletionTime(clock.nowUtc())
-                .isSuperuser(false)
-                .setDomainName("example.tld")
-                .setRepoId("repoId")
-                .setRelockDuration(standardHours(6))
-                .setRegistrarId("TheRegistrar")
-                .setRegistrarPocId("someone@example.com")
-                .setVerificationCode("hi")
-                .build());
-    asyncTaskEnqueuer.enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
-    assertTasksEnqueued(
-        QUEUE_ASYNC_ACTIONS,
-        new TaskMatcher()
-            .url(RelockDomainAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
-            .param(
-                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
-                String.valueOf(lock.getRevisionId()))
-            .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, "0")
-            .etaDelta(
-                standardHours(6).minus(standardSeconds(30)),
-                standardHours(6).plus(standardSeconds(30))));
-  }
-
-  @MockitoSettings(strictness = Strictness.LENIENT)
-  @Test
-  void testFailure_enqueueRelock_noDuration() {
-    RegistryLock lockWithoutDuration =
-        saveRegistryLock(
-            new RegistryLock.Builder()
-                .isSuperuser(false)
-                .setDomainName("example.tld")
-                .setRepoId("repoId")
-                .setRegistrarId("TheRegistrar")
-                .setRegistrarPocId("someone@example.com")
-                .setVerificationCode("hi")
-                .build());
-    assertThat(
-        assertThrows(
-            IllegalArgumentException.class,
-            () -> asyncTaskEnqueuer.enqueueDomainRelock(lockWithoutDuration)))
-        .hasMessageThat()
-        .isEqualTo(
-            String.format(
-                "Lock with ID %s not configured for relock", lockWithoutDuration.getRevisionId()));
-  }
 }

core/src/test/java/google/registry/batch/ExpandRecurringBillingEventsActionTest.java

@@ -48,7 +48,6 @@ import google.registry.model.common.Cursor;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.Period;
-import google.registry.model.ofy.Ofy;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
@@ -56,7 +55,6 @@ import google.registry.model.tld.Registry;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
-import google.registry.testing.InjectExtension;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.TestOfyOnly;
@@ -78,11 +76,6 @@ public class ExpandRecurringBillingEventsActionTest
   private DateTime currentTestTime = DateTime.parse("1999-01-05T00:00:00Z");
   private final FakeClock clock = new FakeClock(currentTestTime);
 
-  @Order(Order.DEFAULT - 1)
-  @RegisterExtension
-  public final InjectExtension inject =
-      new InjectExtension().withStaticFieldOverride(Ofy.class, "clock", clock);
-
   @Order(Order.DEFAULT - 2)
   @RegisterExtension
   public final ReplayExtension replayExtension = ReplayExtension.createWithDoubleReplay(clock);

core/src/test/java/google/registry/batch/RelockDomainActionTest.java

@@ -28,27 +28,26 @@ import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getMostRecentVerifiedRegistryLockByRepoId;
 import static google.registry.testing.SqlHelper.getRegistryLockByVerificationCode;
 import static google.registry.testing.SqlHelper.saveRegistryLock;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static org.joda.time.Duration.standardSeconds;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.host.HostResource;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.UserInfo;
 import google.registry.tools.DomainLockUtils;
@@ -78,12 +77,12 @@ public class RelockDomainActionTest {
 
   private final FakeResponse response = new FakeResponse();
   private final FakeClock clock = new FakeClock(DateTime.parse("2015-05-18T12:34:56Z"));
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
   private final DomainLockUtils domainLockUtils =
       new DomainLockUtils(
           new DeterministicStringGenerator(Alphabets.BASE_58),
           "adminreg",
-          AsyncTaskEnqueuerTest.createForTesting(
-              mock(AppEngineServiceUtils.class), clock, Duration.ZERO));
+          cloudTasksHelper.getTestCloudTasksUtils());
 
   @RegisterExtension
   public final AppEngineExtension appEngineExtension =
@@ -96,7 +95,6 @@ public class RelockDomainActionTest {
   private DomainBase domain;
   private RegistryLock oldLock;
   @Mock private SendEmailService sendEmailService;
-  private AsyncTaskEnqueuer asyncTaskEnqueuer;
   private RelockDomainAction action;
 
   @BeforeEach
@@ -118,8 +116,6 @@ public class RelockDomainActionTest {
         .when(appEngineServiceUtils.getServiceHostname("backend"))
         .thenReturn("backend.hostname.fake");
 
-    asyncTaskEnqueuer =
-        AsyncTaskEnqueuerTest.createForTesting(appEngineServiceUtils, clock, Duration.ZERO);
     action = createAction(oldLock.getRevisionId());
   }
 
@@ -158,7 +154,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -170,7 +166,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -180,7 +176,7 @@ public class RelockDomainActionTest {
     assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
     assertThat(response.getPayload())
         .isEqualTo("Domain example.tld is already manually re-locked, skipping automated re-lock.");
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -192,7 +188,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -207,7 +203,7 @@ public class RelockDomainActionTest {
     assertThat(response.getPayload())
         .isEqualTo(String.format("Re-lock failed: %s", expectedFailureMessage));
     assertNonTransientFailureEmail(expectedFailureMessage);
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -253,7 +249,7 @@ public class RelockDomainActionTest {
     assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
     assertThat(response.getPayload())
         .isEqualTo("Domain example.tld is already manually re-locked, skipping automated re-lock.");
-    assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
+    cloudTasksHelper.assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
   }
 
   @TestOfyAndSql
@@ -320,17 +316,16 @@ public class RelockDomainActionTest {
   }
 
   private void assertTaskEnqueued(int numAttempts, long oldUnlockRevisionId, Duration duration) {
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(RelockDomainAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
             .param(
                 RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
                 String.valueOf(oldUnlockRevisionId))
             .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, String.valueOf(numAttempts))
-            .etaDelta(duration.minus(standardSeconds(30)), duration.plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(duration)));
   }
 
   private RelockDomainAction createAction(Long oldUnlockRevisionId) throws Exception {
@@ -349,7 +344,6 @@ public class RelockDomainActionTest {
         "support@example.com",
         sendEmailService,
         domainLockUtils,
-        response,
-        asyncTaskEnqueuer);
+        response);
   }
 }

core/src/test/java/google/registry/beam/initsql/DomainBaseUtilTest.java

@@ -195,7 +195,7 @@ public class DomainBaseUtilTest {
     domainTransformedByUtil = domainTransformedByUtil.asBuilder().build();
     assertAboutImmutableObjects()
         .that(domainTransformedByUtil)
-        .isEqualExceptFields(domainTransformedByOfy, "revisions");
+        .isEqualExceptFields(domainTransformedByOfy, "revisions", "updateTimestamp");
   }
 
   @Test
@@ -218,7 +218,7 @@ public class DomainBaseUtilTest {
     domainTransformedByUtil = domainTransformedByUtil.asBuilder().build();
     assertAboutImmutableObjects()
         .that(domainTransformedByUtil)
-        .isEqualExceptFields(domainWithoutFKeys, "revisions");
+        .isEqualExceptFields(domainWithoutFKeys, "revisions", "updateTimestamp");
   }
 
   @Test

core/src/test/java/google/registry/cron/CommitLogFanoutActionTest.java

@@ -33,7 +33,7 @@ class CommitLogFanoutActionTest {
 
   private static final String ENDPOINT = "/the/servlet";
   private static final String QUEUE = "the-queue";
-  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(new FakeClock());
 
   @RegisterExtension
   final AppEngineExtension appEngineExtension =
@@ -58,7 +58,6 @@ class CommitLogFanoutActionTest {
     action.endpoint = ENDPOINT;
     action.queue = QUEUE;
     action.jitterSeconds = Optional.empty();
-    action.clock = new FakeClock();
     action.run();
     List<TaskMatcher> matchers = new ArrayList<>();
     for (int bucketId : CommitLogBucket.getBucketIds()) {

core/src/test/java/google/registry/cron/TldFanoutActionTest.java

@@ -45,7 +45,7 @@ class TldFanoutActionTest {
   private static final String ENDPOINT = "/the/servlet";
   private static final String QUEUE = "the-queue";
   private final FakeResponse response = new FakeResponse();
-  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(new FakeClock());
 
   @RegisterExtension
   final AppEngineExtension appEngine =
@@ -61,7 +61,6 @@ class TldFanoutActionTest {
 
   private void run(ImmutableListMultimap<String, String> params) {
     TldFanoutAction action = new TldFanoutAction();
-    action.clock = new FakeClock();
     action.params = params;
     action.endpoint = ENDPOINT;
     action.queue = QUEUE;

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -347,6 +347,12 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
             createBillingEvent));
     assertDnsTasksEnqueued(getUniqueIdFromCommand());
     assertEppResourceIndexEntityFor(domain);
+
+    replayExtension.expectUpdateFor(domain);
+
+    // Verify that all timestamps are correct after SQL -> DS replay.
+    // Added to confirm that timestamps get updated correctly.
+    replayExtension.enableDomainTimestampChecks();
   }
 
   private void assertNoLordn() throws Exception {
@@ -547,6 +553,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
         .hasValue(HistoryEntry.createVKey(Key.create(historyEntry)));
   }
 
+  // DomainTransactionRecord is not propagated.
   @TestOfyAndSql
   void testSuccess_validAllocationToken_multiUse() throws Exception {
     setEppInput(
@@ -574,6 +581,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
         ImmutableMap.of("DOMAIN", "otherexample.tld", "YEARS", "2"));
     runFlowAssertResponse(
         loadFile("domain_create_response.xml", ImmutableMap.of("DOMAIN", "otherexample.tld")));
+    replayExtension.expectUpdateFor(reloadResourceByForeignKey());
   }
 
   @TestOfyAndSql
@@ -826,7 +834,8 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
   @TestOfyAndSql
   void testSuccess_existedButWasDeleted() throws Exception {
     persistContactsAndHosts();
-    persistDeletedDomain(getUniqueIdFromCommand(), clock.nowUtc().minusDays(1));
+    replayExtension.expectUpdateFor(
+        persistDeletedDomain(getUniqueIdFromCommand(), clock.nowUtc().minusDays(1)));
     clock.advanceOneMilli();
     doSuccessfulTest();
   }

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -202,6 +202,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
             .setDomain(domain)
             .build());
     clock.advanceOneMilli();
+    replayExtension.expectUpdateFor(domain);
     return domain;
   }
 
@@ -212,9 +213,11 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   private void doSuccessfulTest(String expectedXmlFilename) throws Exception {
     assertTransactionalFlow(true);
     runFlowAssertResponse(loadFile(expectedXmlFilename));
+    DomainBase domain = reloadResourceByForeignKey();
+    replayExtension.expectUpdateFor(domain);
     // Check that the domain was updated. These values came from the xml.
     assertAboutDomains()
-        .that(reloadResourceByForeignKey())
+        .that(domain)
         .hasStatusValue(StatusValue.CLIENT_HOLD)
         .and()
         .hasAuthInfoPwd("2BARfoo")
@@ -229,6 +232,10 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     assertNoBillingEvents();
     assertDnsTasksEnqueued("example.tld");
     assertLastHistoryContainsResource(reloadResourceByForeignKey());
+
+    // Verify that all timestamps are correct after SQL -> DS replay.
+    // Added to confirm that timestamps get updated correctly.
+    replayExtension.enableDomainTimestampChecks();
   }
 
   @TestOfyAndSql
@@ -308,6 +315,8 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     }
     persistResource(
         reloadResourceByForeignKey().asBuilder().setNameservers(nameservers.build()).build());
+    // Add a null update here so we don't compare.
+    replayExtension.expectUpdateFor(null);
     clock.advanceOneMilli();
   }
 
@@ -1272,6 +1281,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
+  // Contacts mismatch.
   @TestOfyAndSql
   void testFailure_sameContactAddedAndRemoved() throws Exception {
     setEppInput("domain_update_add_remove_same_contact.xml");

core/src/test/java/google/registry/model/domain/DomainBaseSqlTest.java

@@ -671,4 +671,56 @@ public class DomainBaseSqlTest {
         .that(domain.getTransferData())
         .isEqualExceptFields(thatDomain.getTransferData(), "serverApproveEntities");
   }
+
+  @TestSqlOnly
+  void testUpdateTimeAfterNameserverUpdate() {
+    persistDomain();
+    DomainBase persisted = loadByKey(domain.createVKey());
+    DateTime originalUpdateTime = persisted.getUpdateTimestamp().getTimestamp();
+    fakeClock.advanceOneMilli();
+    DateTime transactionTime =
+        jpaTm()
+            .transact(
+                () -> {
+                  HostResource host2 =
+                      new HostResource.Builder()
+                          .setRepoId("host2")
+                          .setHostName("ns2.example.com")
+                          .setCreationRegistrarId("registrar1")
+                          .setPersistedCurrentSponsorRegistrarId("registrar2")
+                          .build();
+                  insertInDb(host2);
+                  domain = persisted.asBuilder().addNameserver(host2.createVKey()).build();
+                  updateInDb(domain);
+                  return jpaTm().getTransactionTime();
+                });
+    domain = loadByKey(domain.createVKey());
+    assertThat(domain.getUpdateTimestamp().getTimestamp()).isEqualTo(transactionTime);
+    assertThat(domain.getUpdateTimestamp().getTimestamp()).isNotEqualTo(originalUpdateTime);
+  }
+
+  @TestSqlOnly
+  void testUpdateTimeAfterDsDataUpdate() {
+    persistDomain();
+    DomainBase persisted = loadByKey(domain.createVKey());
+    DateTime originalUpdateTime = persisted.getUpdateTimestamp().getTimestamp();
+    fakeClock.advanceOneMilli();
+    DateTime transactionTime =
+        jpaTm()
+            .transact(
+                () -> {
+                  domain =
+                      persisted
+                          .asBuilder()
+                          .setDsData(
+                              ImmutableSet.of(
+                                  DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2})))
+                          .build();
+                  updateInDb(domain);
+                  return jpaTm().getTransactionTime();
+                });
+    domain = loadByKey(domain.createVKey());
+    assertThat(domain.getUpdateTimestamp().getTimestamp()).isEqualTo(transactionTime);
+    assertThat(domain.getUpdateTimestamp().getTimestamp()).isNotEqualTo(originalUpdateTime);
+  }
 }

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -75,6 +75,7 @@ public class DomainBaseTest extends EntityTestCase {
   private VKey<BillingEvent.OneTime> oneTimeBillKey;
   private VKey<BillingEvent.Recurring> recurringBillKey;
   private Key<HistoryEntry> historyEntryKey;
+  private VKey<ContactResource> contact1Key, contact2Key;
 
   @BeforeEach
   void setUp() {
@@ -88,14 +89,14 @@ public class DomainBaseTest extends EntityTestCase {
                     .setRepoId("1-COM")
                     .build())
             .createVKey();
-    VKey<ContactResource> contact1Key =
+    contact1Key =
         persistResource(
                 new ContactResource.Builder()
                     .setContactId("contact_id1")
                     .setRepoId("2-COM")
                     .build())
             .createVKey();
-    VKey<ContactResource> contact2Key =
+    contact2Key =
         persistResource(
                 new ContactResource.Builder()
                     .setContactId("contact_id2")
@@ -947,4 +948,61 @@ public class DomainBaseTest extends EntityTestCase {
       this.billingEventOneTime = billingEventOneTime;
     }
   }
+
+  @Test
+  void testContactFields() {
+    VKey<ContactResource> contact3Key =
+        persistResource(
+                new ContactResource.Builder()
+                    .setContactId("contact_id3")
+                    .setRepoId("4-COM")
+                    .build())
+            .createVKey();
+    VKey<ContactResource> contact4Key =
+        persistResource(
+                new ContactResource.Builder()
+                    .setContactId("contact_id4")
+                    .setRepoId("5-COM")
+                    .build())
+            .createVKey();
+
+    // Set all of the contacts.
+    domain.setContactFields(
+        ImmutableSet.of(
+            DesignatedContact.create(Type.REGISTRANT, contact1Key),
+            DesignatedContact.create(Type.ADMIN, contact2Key),
+            DesignatedContact.create(Type.BILLING, contact3Key),
+            DesignatedContact.create(Type.TECH, contact4Key)),
+        true);
+    assertThat(domain.getRegistrant()).isEqualTo(contact1Key);
+    assertThat(domain.getAdminContact()).isEqualTo(contact2Key);
+    assertThat(domain.getBillingContact()).isEqualTo(contact3Key);
+    assertThat(domain.getTechContact()).isEqualTo(contact4Key);
+
+    // Make sure everything gets nulled out.
+    domain.setContactFields(ImmutableSet.of(), true);
+    assertThat(domain.getRegistrant()).isNull();
+    assertThat(domain.getAdminContact()).isNull();
+    assertThat(domain.getBillingContact()).isNull();
+    assertThat(domain.getTechContact()).isNull();
+
+    // Make sure that changes don't affect the registrant unless requested.
+    domain.setContactFields(
+        ImmutableSet.of(
+            DesignatedContact.create(Type.REGISTRANT, contact1Key),
+            DesignatedContact.create(Type.ADMIN, contact2Key),
+            DesignatedContact.create(Type.BILLING, contact3Key),
+            DesignatedContact.create(Type.TECH, contact4Key)),
+        false);
+    assertThat(domain.getRegistrant()).isNull();
+    assertThat(domain.getAdminContact()).isEqualTo(contact2Key);
+    assertThat(domain.getBillingContact()).isEqualTo(contact3Key);
+    assertThat(domain.getTechContact()).isEqualTo(contact4Key);
+    domain = domain.asBuilder().setRegistrant(contact1Key).build();
+    domain.setContactFields(ImmutableSet.of(), false);
+    assertThat(domain.getRegistrant()).isEqualTo(contact1Key);
+    assertThat(domain.getAdminContact()).isNull();
+    assertThat(domain.getBillingContact()).isNull();
+    assertThat(domain.getTechContact()).isNull();
+  }
 }

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -15,6 +15,7 @@
 package google.registry.model.replay;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.replay.ReplicateToDatastoreAction.applyTransaction;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.testing.DatabaseHelper.insertInDb;
@@ -158,23 +159,23 @@ public class ReplicateToDatastoreActionTest {
 
     // Write a transaction and run just the batch fetch.
     insertInDb(foo);
-    List<TransactionEntity> txns1 = action.getTransactionBatch();
+    List<TransactionEntity> txns1 = action.getTransactionBatchAtSnapshot();
     assertThat(txns1).hasSize(1);
 
     // Write a second transaction and do another batch fetch.
     insertInDb(bar);
-    List<TransactionEntity> txns2 = action.getTransactionBatch();
+    List<TransactionEntity> txns2 = action.getTransactionBatchAtSnapshot();
     assertThat(txns2).hasSize(2);
 
     // Apply the first batch.
-    action.applyTransaction(txns1.get(0));
+    applyTransaction(txns1.get(0));
 
     // Remove the foo record so we can ensure that this transaction doesn't get doublle-played.
     ofyTm().transact(() -> ofyTm().delete(foo.key()));
 
     // Apply the second batch.
     for (TransactionEntity txn : txns2) {
-      action.applyTransaction(txn);
+      applyTransaction(txn);
     }
 
     // Verify that the first transaction didn't get replayed but the second one did.
@@ -212,10 +213,9 @@ public class ReplicateToDatastoreActionTest {
     // Force the last transaction id back to -1 so that we look for transaction 0.
     ofyTm().transact(() -> ofyTm().insert(new LastSqlTransaction(-1)));
 
-    List<TransactionEntity> txns = action.getTransactionBatch();
+    List<TransactionEntity> txns = action.getTransactionBatchAtSnapshot();
     assertThat(txns).hasSize(1);
-    assertThat(
-            assertThrows(IllegalStateException.class, () -> action.applyTransaction(txns.get(0))))
+    assertThat(assertThrows(IllegalStateException.class, () -> applyTransaction(txns.get(0))))
         .hasMessageThat()
         .isEqualTo("Missing transaction: last txn id = -1, next available txn = 1");
   }

core/src/test/java/google/registry/persistence/EntityCallbacksListenerTest.java

@@ -22,6 +22,7 @@ import static google.registry.testing.DatabaseHelper.insertInDb;
 
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
+import google.registry.model.replay.NonReplicatedEntity;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension;
 import java.lang.reflect.Method;
@@ -168,7 +169,7 @@ class EntityCallbacksListenerTest {
   }
 
   @Entity(name = "TestEntity")
-  private static class TestEntity extends ParentEntity {
+  private static class TestEntity extends ParentEntity implements NonReplicatedEntity {
     @Id String name = "id";
     int nonTransientField = 0;
 

core/src/test/java/google/registry/persistence/transaction/JpaEntityCoverageExtension.java

@@ -49,7 +49,7 @@ public class JpaEntityCoverageExtension implements BeforeEachCallback, AfterEach
           // TransactionEntity is trivial; its persistence is tested in TransactionTest.
           "TransactionEntity");
 
-  private static final ImmutableSet<Class<?>> ALL_JPA_ENTITIES =
+  public static final ImmutableSet<Class<?>> ALL_JPA_ENTITIES =
       PersistenceXmlUtility.getManagedClasses().stream()
           .filter(e -> !IGNORE_ENTITIES.contains(e.getSimpleName()))
           .filter(e -> e.isAnnotationPresent(Entity.class))

core/src/test/java/google/registry/persistence/transaction/TransactionManagerTest.java

@@ -30,6 +30,7 @@ import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ofy.DatastoreTransactionManager;
 import google.registry.model.ofy.Ofy;
+import google.registry.model.replay.NonReplicatedEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import google.registry.testing.AppEngineExtension;
@@ -448,7 +449,7 @@ public class TransactionManagerTest {
 
   @Entity(name = "TxnMgrTestEntity")
   @javax.persistence.Entity(name = "TestEntity")
-  private static class TestEntity extends TestEntityBase {
+  private static class TestEntity extends TestEntityBase implements NonReplicatedEntity {
 
     private String data;
 

core/src/test/java/google/registry/reporting/billing/GenerateInvoicesActionTest.java

@@ -15,23 +15,26 @@
 package google.registry.reporting.billing;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.net.MediaType;
 import google.registry.beam.BeamActionTestBase;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
+import google.registry.reporting.ReportingModule;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.util.CloudTasksUtils;
 import java.io.IOException;
+import org.joda.time.Duration;
 import org.joda.time.YearMonth;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -45,6 +48,8 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
 
   private final BillingEmailUtils emailUtils = mock(BillingEmailUtils.class);
   private FakeClock clock = new FakeClock();
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
+  private CloudTasksUtils cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
   private GenerateInvoicesAction action;
 
   @TestOfyAndSql
@@ -60,6 +65,7 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
             PrimaryDatabase.DATASTORE,
             new YearMonth(2017, 10),
             emailUtils,
+            cloudTasksUtils,
             clock,
             response,
             dataflow);
@@ -68,13 +74,17 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
     assertThat(response.getStatus()).isEqualTo(SC_OK);
     assertThat(response.getPayload()).isEqualTo("Launched invoicing pipeline: jobid");
 
-    TaskMatcher matcher =
+    cloudTasksHelper.assertTasksEnqueued(
+        "beam-reporting",
         new TaskMatcher()
             .url("/_dr/task/publishInvoices")
-            .method("POST")
+            .method(HttpMethod.POST)
             .param("jobId", "jobid")
-            .param("yearMonth", "2017-10");
-    assertTasksEnqueued("beam-reporting", matcher);
+            .param("yearMonth", "2017-10")
+            .scheduleTime(
+                clock
+                    .nowUtc()
+                    .plus(Duration.standardMinutes(ReportingModule.ENQUEUE_DELAY_MINUTES))));
   }
 
   @TestOfyAndSql
@@ -90,6 +100,7 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
             PrimaryDatabase.DATASTORE,
             new YearMonth(2017, 10),
             emailUtils,
+            cloudTasksUtils,
             clock,
             response,
             dataflow);
@@ -97,7 +108,7 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
     assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
     assertThat(response.getStatus()).isEqualTo(SC_OK);
     assertThat(response.getPayload()).isEqualTo("Launched invoicing pipeline: jobid");
-    assertNoTasksEnqueued("beam-reporting");
+    cloudTasksHelper.assertNoTasksEnqueued("beam-reporting");
   }
 
   @TestOfyAndSql
@@ -114,6 +125,7 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
             PrimaryDatabase.DATASTORE,
             new YearMonth(2017, 10),
             emailUtils,
+            cloudTasksUtils,
             clock,
             response,
             dataflow);
@@ -121,6 +133,6 @@ class GenerateInvoicesActionTest extends BeamActionTestBase {
     assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
     assertThat(response.getPayload()).isEqualTo("Pipeline launch failed: Pipeline error");
     verify(emailUtils).sendAlertEmail("Pipeline Launch failed due to Pipeline error");
-    assertNoTasksEnqueued("beam-reporting");
+    cloudTasksHelper.assertNoTasksEnqueued("beam-reporting");
   }
 }

core/src/test/java/google/registry/reporting/icann/IcannReportingStagingActionTest.java

@@ -15,29 +15,31 @@
 package google.registry.reporting.icann;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import google.registry.bigquery.BigqueryJobFailureException;
 import google.registry.reporting.icann.IcannReportingModule.ReportType;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.util.EmailMessage;
 import google.registry.util.Retrier;
 import google.registry.util.SendEmailService;
 import java.util.Optional;
 import javax.mail.internet.InternetAddress;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
 import org.joda.time.YearMonth;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -51,6 +53,8 @@ class IcannReportingStagingActionTest {
   private YearMonth yearMonth = new YearMonth(2017, 6);
   private String subdir = "default/dir";
   private IcannReportingStagingAction action;
+  private FakeClock clock = new FakeClock(DateTime.parse("2021-01-02T11:00:00Z"));
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @RegisterExtension
   final AppEngineExtension appEngine =
@@ -72,6 +76,7 @@ class IcannReportingStagingActionTest {
     action.sender = new InternetAddress("sender@example.com");
     action.recipient = new InternetAddress("recipient@example.com");
     action.emailService = mock(SendEmailService.class);
+    action.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
 
     when(stager.stageReports(yearMonth, subdir, ReportType.ACTIVITY))
         .thenReturn(ImmutableList.of("a", "b"));
@@ -79,9 +84,13 @@ class IcannReportingStagingActionTest {
         .thenReturn(ImmutableList.of("c", "d"));
   }
 
-  private static void assertUploadTaskEnqueued() {
-    TaskMatcher matcher = new TaskMatcher().url("/_dr/task/icannReportingUpload").method("POST");
-    assertTasksEnqueued("retryable-cron-tasks", matcher);
+  private void assertUploadTaskEnqueued() {
+    cloudTasksHelper.assertTasksEnqueued(
+        "retryable-cron-tasks",
+        new TaskMatcher()
+            .url("/_dr/task/icannReportingUpload")
+            .method(HttpMethod.POST)
+            .scheduleTime(clock.nowUtc().plus(Duration.standardMinutes(2))));
   }
 
   @Test
@@ -157,7 +166,7 @@ class IcannReportingStagingActionTest {
                 new InternetAddress("recipient@example.com"),
                 new InternetAddress("sender@example.com")));
     // Assert no upload task enqueued
-    assertNoTasksEnqueued("retryable-cron-tasks");
+    cloudTasksHelper.assertNoTasksEnqueued("retryable-cron-tasks");
   }
 
   @Test

core/src/test/java/google/registry/reporting/spec11/GenerateSpec11ReportActionTest.java

@@ -15,20 +15,23 @@
 package google.registry.reporting.spec11;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static org.apache.http.HttpStatus.SC_OK;
 import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.net.MediaType;
 import google.registry.beam.BeamActionTestBase;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
+import google.registry.reporting.ReportingModule;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeClock;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
+import google.registry.util.CloudTasksUtils;
 import java.io.IOException;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -40,6 +43,8 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase {
       AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
 
   private final FakeClock clock = new FakeClock(DateTime.parse("2018-06-11T12:23:56Z"));
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
+  private CloudTasksUtils cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
   private GenerateSpec11ReportAction action;
 
   @Test
@@ -56,13 +61,14 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase {
             true,
             clock,
             response,
-            dataflow);
+            dataflow,
+            cloudTasksUtils);
     when(launch.execute()).thenThrow(new IOException("Dataflow failure"));
     action.run();
     assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
     assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).contains("Dataflow failure");
-    assertNoTasksEnqueued("beam-reporting");
+    cloudTasksHelper.assertNoTasksEnqueued("beam-reporting");
   }
 
   @Test
@@ -79,18 +85,24 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase {
             true,
             clock,
             response,
-            dataflow);
+            dataflow,
+            cloudTasksUtils);
     action.run();
     assertThat(response.getStatus()).isEqualTo(SC_OK);
     assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("Launched Spec11 pipeline: jobid");
-    TaskMatcher matcher =
+
+    cloudTasksHelper.assertTasksEnqueued(
+        "beam-reporting",
         new TaskMatcher()
             .url("/_dr/task/publishSpec11")
-            .method("POST")
+            .method(HttpMethod.POST)
             .param("jobId", "jobid")
-            .param("date", "2018-06-11");
-    assertTasksEnqueued("beam-reporting", matcher);
+            .param("date", "2018-06-11")
+            .scheduleTime(
+                clock
+                    .nowUtc()
+                    .plus(Duration.standardMinutes(ReportingModule.ENQUEUE_DELAY_MINUTES))));
   }
 
   @Test
@@ -107,11 +119,12 @@ class GenerateSpec11ReportActionTest extends BeamActionTestBase {
             false,
             clock,
             response,
-            dataflow);
+            dataflow,
+            cloudTasksUtils);
     action.run();
     assertThat(response.getStatus()).isEqualTo(SC_OK);
     assertThat(response.getContentType()).isEqualTo(MediaType.PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("Launched Spec11 pipeline: jobid");
-    assertNoTasksEnqueued("beam-reporting");
+    cloudTasksHelper.assertNoTasksEnqueued("beam-reporting");
   }
 }

core/src/test/java/google/registry/server/TestServer.java

@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.net.HostAndPort;
 import com.google.common.util.concurrent.SimpleTimeLimiter;
+import google.registry.ui.server.registrar.RegistrarSettingsAction;
 import google.registry.util.UrlChecker;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -96,6 +97,7 @@ public final class TestServer {
   /** Starts the HTTP server in a new thread and returns once it's online. */
   public void start() {
     try {
+      RegistrarSettingsAction.setIsInTestDriverToTrue();
       server.start();
     } catch (Exception e) {
       throwIfUnchecked(e);
@@ -128,14 +130,16 @@ public final class TestServer {
   /** Stops the HTTP server. */
   public void stop() {
     try {
-      Void unusedReturnValue = SimpleTimeLimiter.create(newCachedThreadPool())
-          .callWithTimeout(
-              () -> {
-                server.stop();
-                return null;
-              },
-              SHUTDOWN_TIMEOUT_MS,
-              TimeUnit.MILLISECONDS);
+      Void unusedReturnValue =
+          SimpleTimeLimiter.create(newCachedThreadPool())
+              .callWithTimeout(
+                  () -> {
+                    server.stop();
+                    RegistrarSettingsAction.setIsInTestDriverToFalse();
+                    return null;
+                  },
+                  SHUTDOWN_TIMEOUT_MS,
+                  TimeUnit.MILLISECONDS);
     } catch (Exception e) {
       throwIfUnchecked(e);
       throw new RuntimeException(e);

core/src/test/java/google/registry/testing/CloudTasksHelper.java

@@ -40,6 +40,7 @@ import com.google.common.net.HttpHeaders;
 import com.google.common.net.MediaType;
 import com.google.common.truth.Truth8;
 import com.google.protobuf.Timestamp;
+import com.google.protobuf.util.Timestamps;
 import google.registry.model.ImmutableObject;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.Retrier;
@@ -60,6 +61,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import javax.annotation.Nonnull;
+import org.joda.time.DateTime;
 
 /**
  * Static utility functions for testing task queues.
@@ -92,13 +94,22 @@ public class CloudTasksHelper implements Serializable {
   private static final String PROJECT_ID = "test-project";
   private static final String LOCATION_ID = "test-location";
 
-  private final Retrier retrier = new Retrier(new FakeSleeper(new FakeClock()), 1);
   private final int instanceId = nextInstanceId.getAndIncrement();
-  private final CloudTasksUtils cloudTasksUtils =
-      new CloudTasksUtils(retrier, PROJECT_ID, LOCATION_ID, new FakeCloudTasksClient());
+  private final CloudTasksUtils cloudTasksUtils;
+
+  public CloudTasksHelper(FakeClock clock) {
+    this.cloudTasksUtils =
+        new CloudTasksUtils(
+            new Retrier(new FakeSleeper(clock), 1),
+            clock,
+            PROJECT_ID,
+            LOCATION_ID,
+            new FakeCloudTasksClient());
+    testTasks.put(instanceId, Multimaps.synchronizedListMultimap(LinkedListMultimap.create()));
+  }
 
   public CloudTasksHelper() {
-    testTasks.put(instanceId, Multimaps.synchronizedListMultimap(LinkedListMultimap.create()));
+    this(new FakeClock());
   }
 
   public CloudTasksUtils getTestCloudTasksUtils() {
@@ -232,7 +243,7 @@ public class CloudTasksHelper implements Serializable {
       ImmutableMultimap.Builder<String, String> paramBuilder = new ImmutableMultimap.Builder<>();
       // Note that UriParameters.parse() does not throw an IAE on a bad query string (e.g. one
       // where parameters are not properly URL-encoded); it always does a best-effort parse.
-      if (method == HttpMethod.GET) {
+      if (method == HttpMethod.GET && uri.getQuery() != null) {
         paramBuilder.putAll(UriParameters.parse(uri.getQuery()));
       } else if (method == HttpMethod.POST && !task.getAppEngineHttpRequest().getBody().isEmpty()) {
         assertThat(
@@ -302,6 +313,10 @@ public class CloudTasksHelper implements Serializable {
       return this;
     }
 
+    public TaskMatcher scheduleTime(DateTime scheduleTime) {
+      return scheduleTime(Timestamps.fromMillis(scheduleTime.getMillis()));
+    }
+
     public TaskMatcher param(String key, String value) {
       checkNotNull(value, "Test error: A param can never have a null value, so don't assert it");
       expected.params.put(key, value);

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -472,8 +472,10 @@ public class DatabaseHelper {
 
   public static void allowRegistrarAccess(String registrarId, String tld) {
     Registrar registrar = loadRegistrar(registrarId);
-    persistResource(
-        registrar.asBuilder().setAllowedTlds(union(registrar.getAllowedTlds(), tld)).build());
+    if (!registrar.getAllowedTlds().contains(tld)) {
+      persistResource(
+          registrar.asBuilder().setAllowedTlds(union(registrar.getAllowedTlds(), tld)).build());
+    }
   }
 
   public static void disallowRegistrarAccess(String registrarId, String tld) {

core/src/test/java/google/registry/testing/ReplayExtension.java

@@ -14,18 +14,27 @@
 
 package google.registry.testing;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import static org.junit.Assert.fail;
 
 import com.google.common.base.Ascii;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
+import google.registry.model.EntityClasses;
 import google.registry.model.ImmutableObject;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.ReplayQueue;
 import google.registry.model.ofy.TransactionInfo;
@@ -33,6 +42,7 @@ import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.ReplicateToDatastoreAction;
 import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaEntityCoverageExtension;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
 import google.registry.persistence.transaction.Transaction;
 import google.registry.persistence.transaction.Transaction.Delete;
@@ -41,9 +51,15 @@ import google.registry.persistence.transaction.Transaction.Update;
 import google.registry.persistence.transaction.TransactionEntity;
 import google.registry.util.RequestStatusChecker;
 import java.io.IOException;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Optional;
 import javax.annotation.Nullable;
+import org.junit.jupiter.api.TestTemplate;
 import org.junit.jupiter.api.extension.AfterEachCallback;
 import org.junit.jupiter.api.extension.BeforeEachCallback;
 import org.junit.jupiter.api.extension.ExtensionContext;
@@ -56,28 +72,69 @@ import org.mockito.Mockito;
  * that extension are also replayed. If AppEngineExtension is not used,
  * JpaTransactionManagerExtension must be, and this extension should be ordered _after_
  * JpaTransactionManagerExtension so that writes to SQL work.
- *
- * <p>If the "compare" flag is set in the constructor, this will also compare all touched objects in
- * both databases after performing the replay.
  */
 public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static ImmutableSet<String> NON_REPLICATED_TYPES =
+      ImmutableSet.of(
+          "PremiumList",
+          "PremiumListRevision",
+          "PremiumListEntry",
+          "ReservedList",
+          "RdeRevision",
+          "ServerSecret",
+          "SignedMarkRevocationList",
+          "ClaimsListShard",
+          "TmchCrl",
+          "EppResourceIndex",
+          "ForeignKeyIndex",
+          "ForeignKeyHostIndex",
+          "ForeignKeyContactIndex",
+          "ForeignKeyDomainIndex");
+
+  // Entity classes to be ignored during the final database comparison.  Note that this is just a
+  // mash-up of Datastore and SQL classes, and used for filtering both sets.  We could split them
+  // out, but there is plenty of overlap and no name collisions so it doesn't matter very much.
+  private static ImmutableSet<String> IGNORED_ENTITIES =
+      Streams.concat(
+              ImmutableSet.of(
+                  // These entities are @Embed-ded in Datastore
+                  "DelegationSignerData",
+                  "DomainDsDataHistory",
+                  "DomainTransactionRecord",
+                  "GracePeriod",
+                  "GracePeriodHistory",
+
+                  // These entities are legitimately not comparable.
+                  "ClaimsEntry",
+                  "ClaimsList",
+                  "CommitLogBucket",
+                  "CommitLogManifest",
+                  "CommitLogMutation",
+                  "PremiumEntry",
+                  "ReservedListEntry")
+                  .stream(),
+              NON_REPLICATED_TYPES.stream())
+          .collect(toImmutableSet());
+
   FakeClock clock;
-  boolean compare;
   boolean replayed = false;
   boolean inOfyContext;
   InjectExtension injectExtension = new InjectExtension();
   @Nullable ReplicateToDatastoreAction sqlToDsReplicator;
+  List<DomainBase> expectedUpdates = new ArrayList<>();
+  boolean enableDomainTimestampChecks;
+  boolean enableDatabaseCompare = true;
 
-  private ReplayExtension(
-      FakeClock clock, boolean compare, @Nullable ReplicateToDatastoreAction sqlToDsReplicator) {
+  private ReplayExtension(FakeClock clock, @Nullable ReplicateToDatastoreAction sqlToDsReplicator) {
     this.clock = clock;
-    this.compare = compare;
     this.sqlToDsReplicator = sqlToDsReplicator;
   }
 
   public static ReplayExtension createWithCompare(FakeClock clock) {
-    return new ReplayExtension(clock, true, null);
+    return new ReplayExtension(clock, null);
   }
 
   // This allows us to disable the replay tests from an environment variable in specific
@@ -100,7 +157,6 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
     if (replayTestsEnabled()) {
       return new ReplayExtension(
           clock,
-          true,
           new ReplicateToDatastoreAction(
               clock, Mockito.mock(RequestStatusChecker.class), new FakeResponse()));
     } else {
@@ -108,8 +164,37 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
     }
   }
 
+  /**
+   * Enable checking of domain timestamps during replay.
+   *
+   * <p>This was added to facilitate testing of a very specific bug wherein create/update
+   * auto-timestamps serialized to the SQL -> DS Transaction table had different values from those
+   * actually stored in SQL.
+   *
+   * <p>In order to use this, you also need to use expectUpdateFor() to store the states of a
+   * DomainBase object at a given point in time.
+   */
+  public void enableDomainTimestampChecks() {
+    enableDomainTimestampChecks = true;
+  }
+
+  /**
+   * If we're doing domain time checks, add the current state of a domain to check against.
+   *
+   * <p>A null argument is a placeholder to deal with b/217952766. Basically it allows us to ignore
+   * one particular state in the sequence (where the timestamp is not what we expect it to be).
+   */
+  public void expectUpdateFor(@Nullable DomainBase domain) {
+    expectedUpdates.add(domain);
+  }
+
   @Override
   public void beforeEach(ExtensionContext context) {
+    Optional<Method> elem = context.getTestMethod();
+    if (elem.isPresent() && elem.get().isAnnotationPresent(NoDatabaseCompare.class)) {
+      enableDatabaseCompare = false;
+    }
+
     // Use a single bucket to expose timestamp inversion problems. This typically happens when
     // a test with this extension rolls back the fake clock in the setup method, creating inverted
     // timestamp with the canned data preloaded by AppengineExtension. The solution is to move
@@ -142,23 +227,6 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
     }
   }
 
-  private static ImmutableSet<String> NON_REPLICATED_TYPES =
-      ImmutableSet.of(
-          "PremiumList",
-          "PremiumListRevision",
-          "PremiumListEntry",
-          "ReservedList",
-          "RdeRevision",
-          "ServerSecret",
-          "SignedMarkRevocationList",
-          "ClaimsListShard",
-          "TmchCrl",
-          "EppResourceIndex",
-          "ForeignKeyIndex",
-          "ForeignKeyHostIndex",
-          "ForeignKeyContactIndex",
-          "ForeignKeyDomainIndex");
-
   public void replay() {
     if (!replayed) {
       if (inOfyContext) {
@@ -183,34 +251,32 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
     ImmutableMap<Key<?>, Object> changes = ReplayQueue.replay();
 
     // Compare JPA to OFY, if requested.
-    if (compare) {
-      for (ImmutableMap.Entry<Key<?>, Object> entry : changes.entrySet()) {
-        // Don't verify non-replicated types.
-        if (NON_REPLICATED_TYPES.contains(entry.getKey().getKind())) {
-          continue;
-        }
-
-        // Since the object may have changed in datastore by the time we're doing the replay, we
-        // have to compare the current value in SQL (which we just mutated) against the value that
-        // we originally would have persisted (that being the object in the entry).
-        VKey<?> vkey = VKey.from(entry.getKey());
-        jpaTm()
-            .transact(
-                () -> {
-                  Optional<?> jpaValue = jpaTm().loadByKeyIfPresent(vkey);
-                  if (entry.getValue().equals(TransactionInfo.Delete.SENTINEL)) {
-                    assertThat(jpaValue.isPresent()).isFalse();
-                  } else {
-                    ImmutableObject immutJpaObject = (ImmutableObject) jpaValue.get();
-                    assertAboutImmutableObjects().that(immutJpaObject).hasCorrectHashValue();
-                    assertAboutImmutableObjects()
-                        .that(immutJpaObject)
-                        .isEqualAcrossDatabases(
-                            (ImmutableObject)
-                                ((DatastoreEntity) entry.getValue()).toSqlEntity().get());
-                  }
-                });
+    for (ImmutableMap.Entry<Key<?>, Object> entry : changes.entrySet()) {
+      // Don't verify non-replicated types.
+      if (NON_REPLICATED_TYPES.contains(entry.getKey().getKind())) {
+        continue;
       }
+
+      // Since the object may have changed in datastore by the time we're doing the replay, we
+      // have to compare the current value in SQL (which we just mutated) against the value that
+      // we originally would have persisted (that being the object in the entry).
+      VKey<?> vkey = VKey.from(entry.getKey());
+      jpaTm()
+          .transact(
+              () -> {
+                Optional<?> jpaValue = jpaTm().loadByKeyIfPresent(vkey);
+                if (entry.getValue().equals(TransactionInfo.Delete.SENTINEL)) {
+                  assertThat(jpaValue.isPresent()).isFalse();
+                } else {
+                  ImmutableObject immutJpaObject = (ImmutableObject) jpaValue.get();
+                  assertAboutImmutableObjects().that(immutJpaObject).hasCorrectHashValue();
+                  assertAboutImmutableObjects()
+                      .that(immutJpaObject)
+                      .isEqualAcrossDatabases(
+                          (ImmutableObject)
+                              ((DatastoreEntity) entry.getValue()).toSqlEntity().get());
+                }
+              });
     }
   }
 
@@ -221,15 +287,18 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
 
     List<TransactionEntity> transactionBatch;
     do {
-      transactionBatch = sqlToDsReplicator.getTransactionBatch();
+      transactionBatch = sqlToDsReplicator.getTransactionBatchAtSnapshot();
       for (TransactionEntity txn : transactionBatch) {
-        sqlToDsReplicator.applyTransaction(txn);
-        if (compare) {
-          ofyTm().transact(() -> compareSqlTransaction(txn));
-        }
+        ReplicateToDatastoreAction.applyTransaction(txn);
+        ofyTm().transact(() -> compareSqlTransaction(txn));
         clock.advanceOneMilli();
       }
     } while (!transactionBatch.isEmpty());
+
+    // Now that everything has been replayed, compare the databases.
+    if (enableDatabaseCompare) {
+      compareDatabases();
+    }
   }
 
   /** Verifies that the replaying the SQL transaction created the same entities in Datastore. */
@@ -253,6 +322,21 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
         assertAboutImmutableObjects()
             .that(fromDatastore)
             .isEqualAcrossDatabases(fromTransactionEntity);
+
+        // Check DomainBase timestamps if appropriate.
+        if (enableDomainTimestampChecks && fromTransactionEntity instanceof DomainBase) {
+          DomainBase expectedDomain = expectedUpdates.remove(0);
+
+          // Just skip it if the expectedDomain is null.
+          if (expectedDomain == null) {
+            continue;
+          }
+
+          DomainBase domainEntity = (DomainBase) fromTransactionEntity;
+          assertThat(domainEntity.getCreationTime()).isEqualTo(expectedDomain.getCreationTime());
+          assertThat(domainEntity.getUpdateTimestamp())
+              .isEqualTo(expectedDomain.getUpdateTimestamp());
+        }
       } else {
         Delete delete = (Delete) mutation;
         VKey<?> key = delete.getKey();
@@ -262,4 +346,84 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
       }
     }
   }
+
+  /** Compares the final state of both databases after replay is complete. */
+  private void compareDatabases() {
+    boolean gotDiffs = false;
+
+    // Build a map containing all of the SQL entities indexed by their key.
+    HashMap<Object, Object> sqlEntities = new HashMap<>();
+    for (Class<?> cls : JpaEntityCoverageExtension.ALL_JPA_ENTITIES) {
+      if (IGNORED_ENTITIES.contains(cls.getSimpleName())) {
+        continue;
+      }
+
+      jpaTm()
+          .transact(
+              () -> jpaTm().loadAllOfStream(cls).forEach(e -> sqlEntities.put(getSqlKey(e), e)));
+    }
+
+    for (Class<? extends ImmutableObject> cls : EntityClasses.ALL_CLASSES) {
+      if (IGNORED_ENTITIES.contains(cls.getSimpleName())) {
+        continue;
+      }
+
+      for (ImmutableObject entity : auditedOfy().load().type(cls).list()) {
+        // Find the entity in SQL and verify that it's the same.
+        Key<?> ofyKey = Key.create(entity);
+        Object sqlKey = VKey.from(ofyKey).getSqlKey();
+        ImmutableObject sqlEntity = (ImmutableObject) sqlEntities.get(sqlKey);
+        Optional<SqlEntity> expectedSqlEntity = ((DatastoreEntity) entity).toSqlEntity();
+        if (expectedSqlEntity.isPresent()) {
+          // Check for null just so we get a better error message.
+          if (sqlEntity == null) {
+            logger.atSevere().log("Entity %s is in Datastore but not in SQL.", ofyKey);
+            gotDiffs = true;
+          } else {
+            try {
+              assertAboutImmutableObjects()
+                  .that((ImmutableObject) expectedSqlEntity.get())
+                  .isEqualAcrossDatabases(sqlEntity);
+            } catch (AssertionError e) {
+              // Show the message but swallow the stack trace (we'll get that from the fail() at
+              // the end of the comparison).
+              logger.atSevere().log("For entity %s: %s", ofyKey, e.getMessage());
+              gotDiffs = true;
+            }
+          }
+        } else {
+          logger.atInfo().log("Datastore entity has no sql representation for %s", ofyKey);
+        }
+        sqlEntities.remove(sqlKey);
+      }
+    }
+
+    // Report any objects in the SQL set that we didn't remove while iterating over the Datastore
+    // objects.
+    if (!sqlEntities.isEmpty()) {
+      for (Object item : sqlEntities.values()) {
+        logger.atSevere().log(
+            "Entity of %s found in SQL but not in datastore: %s", item.getClass().getName(), item);
+      }
+      gotDiffs = true;
+    }
+
+    if (gotDiffs) {
+      fail("There were differences between the final SQL and Datastore contents.");
+    }
+  }
+
+  private static Object getSqlKey(Object entity) {
+    return jpaTm()
+        .getEntityManager()
+        .getEntityManagerFactory()
+        .getPersistenceUnitUtil()
+        .getIdentifier(entity);
+  }
+
+  /** Annotation to use for test methods where we don't want to do a database comparison yet. */
+  @Target({METHOD})
+  @Retention(RUNTIME)
+  @TestTemplate
+  public @interface NoDatabaseCompare {}
 }

core/src/test/java/google/registry/tmch/NordnUploadActionTest.java

@@ -138,13 +138,26 @@ class NordnUploadActionTest {
   void test_convertTasksToCsv() {
     List<TaskHandle> tasks =
         ImmutableList.of(
-            makeTaskHandle("task1", "example", "csvLine1", "lordn-sunrise"),
             makeTaskHandle("task2", "example", "csvLine2", "lordn-sunrise"),
+            makeTaskHandle("task1", "example", "csvLine1", "lordn-sunrise"),
             makeTaskHandle("task3", "example", "ending", "lordn-sunrise"));
     assertThat(NordnUploadAction.convertTasksToCsv(tasks, clock.nowUtc(), "col1,col2"))
         .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
   }
 
+  @MockitoSettings(strictness = Strictness.LENIENT)
+  @Test
+  void test_convertTasksToCsv_dedupesDuplicates() {
+    List<TaskHandle> tasks =
+        ImmutableList.of(
+            makeTaskHandle("task2", "example", "csvLine2", "lordn-sunrise"),
+            makeTaskHandle("task1", "example", "csvLine1", "lordn-sunrise"),
+            makeTaskHandle("task3", "example", "ending", "lordn-sunrise"),
+            makeTaskHandle("task1", "example", "csvLine1", "lordn-sunrise"));
+    assertThat(NordnUploadAction.convertTasksToCsv(tasks, clock.nowUtc(), "col1,col2"))
+        .isEqualTo("1,2010-05-01T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
+  }
+
   @MockitoSettings(strictness = Strictness.LENIENT)
   @Test
   void test_convertTasksToCsv_doesntFailOnEmptyTasks() {

core/src/test/java/google/registry/tools/DomainLockUtilsTest.java

@@ -26,17 +26,16 @@ import static google.registry.testing.DatabaseHelper.persistActiveHost;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getRegistryLockByRevisionId;
 import static google.registry.testing.SqlHelper.getRegistryLockByVerificationCode;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
+import static google.registry.testing.SqlHelper.saveRegistryLock;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardHours;
-import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.batch.RelockDomainAction;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -47,12 +46,13 @@ import google.registry.model.host.HostResource;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.SqlHelper;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.testing.UserInfo;
 import google.registry.util.AppEngineServiceUtils;
@@ -62,8 +62,11 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.junit.Assert;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
 
 /** Unit tests for {@link google.registry.tools.DomainLockUtils}. */
 @DualDatabaseTest
@@ -74,6 +77,7 @@ public final class DomainLockUtilsTest {
 
   private final FakeClock clock = new FakeClock(DateTime.now(DateTimeZone.UTC));
   private DomainLockUtils domainLockUtils;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
 
   @RegisterExtension
   public final AppEngineExtension appEngineExtension =
@@ -98,8 +102,7 @@ public final class DomainLockUtilsTest {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                appEngineServiceUtils, clock, standardSeconds(90)));
+            cloudTasksHelper.getTestCloudTasksUtils());
   }
 
   @TestOfyAndSql
@@ -265,19 +268,17 @@ public final class DomainLockUtilsTest {
         domainLockUtils.saveNewRegistryUnlockRequest(
             DOMAIN_NAME, "TheRegistrar", false, Optional.of(standardHours(6)));
     domainLockUtils.verifyAndApplyUnlock(lock.getVerificationCode(), false);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
             .url(RelockDomainAction.PATH)
-            .method("POST")
-            .header("Host", "backend.hostname.fake")
+            .method(HttpMethod.POST)
+            .service("backend")
             .param(
                 RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
                 String.valueOf(lock.getRevisionId()))
             .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, "0")
-            .etaDelta(
-                standardHours(6).minus(standardSeconds(30)),
-                standardDays(6).plus(standardSeconds(30))));
+            .scheduleTime(clock.nowUtc().plus(lock.getRelockDuration().get())));
   }
 
   @TestOfyAndSql
@@ -477,6 +478,59 @@ public final class DomainLockUtilsTest {
     assertNoDomainChanges();
   }
 
+  @TestOfyAndSql
+  void testEnqueueRelock() {
+    RegistryLock lock =
+        saveRegistryLock(
+            new RegistryLock.Builder()
+                .setLockCompletionTime(clock.nowUtc())
+                .setUnlockRequestTime(clock.nowUtc())
+                .setUnlockCompletionTime(clock.nowUtc())
+                .isSuperuser(false)
+                .setDomainName("example.tld")
+                .setRepoId("repoId")
+                .setRelockDuration(standardHours(6))
+                .setRegistrarId("TheRegistrar")
+                .setRegistrarPocId("someone@example.com")
+                .setVerificationCode("hi")
+                .build());
+    domainLockUtils.enqueueDomainRelock(lock.getRelockDuration().get(), lock.getRevisionId(), 0);
+    cloudTasksHelper.assertTasksEnqueued(
+        QUEUE_ASYNC_ACTIONS,
+        new CloudTasksHelper.TaskMatcher()
+            .url(RelockDomainAction.PATH)
+            .method(HttpMethod.POST)
+            .service("backend")
+            .param(
+                RelockDomainAction.OLD_UNLOCK_REVISION_ID_PARAM,
+                String.valueOf(lock.getRevisionId()))
+            .param(RelockDomainAction.PREVIOUS_ATTEMPTS_PARAM, "0")
+            .scheduleTime(clock.nowUtc().plus(lock.getRelockDuration().get())));
+  }
+
+  @MockitoSettings(strictness = Strictness.LENIENT)
+  @TestOfyAndSql
+  void testFailure_enqueueRelock_noDuration() {
+    RegistryLock lockWithoutDuration =
+        saveRegistryLock(
+            new RegistryLock.Builder()
+                .isSuperuser(false)
+                .setDomainName("example.tld")
+                .setRepoId("repoId")
+                .setRegistrarId("TheRegistrar")
+                .setRegistrarPocId("someone@example.com")
+                .setVerificationCode("hi")
+                .build());
+    assertThat(
+            Assert.assertThrows(
+                IllegalArgumentException.class,
+                () -> domainLockUtils.enqueueDomainRelock(lockWithoutDuration)))
+        .hasMessageThat()
+        .isEqualTo(
+            String.format(
+                "Lock with ID %s not configured for relock", lockWithoutDuration.getRevisionId()));
+  }
+
   private void verifyProperlyLockedDomain(boolean isAdmin) {
     assertThat(loadByEntity(domain).getStatusValues())
         .containsAtLeastElementsIn(REGISTRY_LOCK_STATUSES);

core/src/test/java/google/registry/tools/EncryptEscrowDepositCommandTest.java

@@ -40,6 +40,8 @@ public class EncryptEscrowDepositCommandTest
     EscrowDepositEncryptor res = new EscrowDepositEncryptor();
     res.rdeReceiverKey = () -> new FakeKeyringModule().get().getRdeReceiverKey();
     res.rdeSigningKey = () -> new FakeKeyringModule().get().getRdeSigningKey();
+    res.brdaReceiverKey = () -> new FakeKeyringModule().get().getBrdaReceiverKey();
+    res.brdaSigningKey = () -> new FakeKeyringModule().get().getBrdaSigningKey();
     return res;
   }
 
@@ -61,4 +63,34 @@ public class EncryptEscrowDepositCommandTest
             "lol_2010-10-17_full_S1_R0.sig",
             "lol.pub");
   }
+
+  @Test
+  void testSuccess_brda() throws Exception {
+    Path depositFile = tmpDir.resolve("deposit.xml");
+    Files.write(depositXml.read(), depositFile.toFile());
+    runCommand(
+        "--mode=THIN", "--tld=lol", "--input=" + depositFile, "--outdir=" + tmpDir.toString());
+    assertThat(tmpDir.toFile().list())
+        .asList()
+        .containsExactly(
+            "deposit.xml",
+            "lol_2010-10-17_thin_S1_R0.ryde",
+            "lol_2010-10-17_thin_S1_R0.sig",
+            "lol.pub");
+  }
+
+  @Test
+  void testSuccess_revision() throws Exception {
+    Path depositFile = tmpDir.resolve("deposit.xml");
+    Files.write(depositXml.read(), depositFile.toFile());
+    runCommand(
+        "--revision=1", "--tld=lol", "--input=" + depositFile, "--outdir=" + tmpDir.toString());
+    assertThat(tmpDir.toFile().list())
+        .asList()
+        .containsExactly(
+            "deposit.xml",
+            "lol_2010-10-17_full_S1_R1.ryde",
+            "lol_2010-10-17_full_S1_R1.sig",
+            "lol.pub");
+  }
 }

core/src/test/java/google/registry/tools/LoadTestCommandTest.java

@@ -52,7 +52,7 @@ class LoadTestCommandTest extends CommandTestCase<LoadTestCommand> {
             .put("hostInfos", 1)
             .put("domainInfos", 1)
             .put("contactInfos", 1)
-            .put("runSeconds", 4600)
+            .put("runSeconds", 9200)
             .build();
     verify(connection)
         .sendPostRequest(

core/src/test/java/google/registry/tools/LockDomainCommandTest.java

@@ -24,19 +24,16 @@ import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.mock;
 
 import com.google.common.collect.ImmutableList;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DeterministicStringGenerator;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.StringGenerator.Alphabets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
-import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -52,8 +49,7 @@ class LockDomainCommandTest extends CommandTestCase<LockDomainCommand> {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                mock(AppEngineServiceUtils.class), fakeClock, Duration.ZERO));
+            new CloudTasksHelper(fakeClock).getTestCloudTasksUtils());
   }
 
   @Test

core/src/test/java/google/registry/tools/UnlockDomainCommandTest.java

@@ -25,21 +25,18 @@ import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.Mockito.mock;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DeterministicStringGenerator;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.StringGenerator.Alphabets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
-import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -55,8 +52,7 @@ class UnlockDomainCommandTest extends CommandTestCase<UnlockDomainCommand> {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                mock(AppEngineServiceUtils.class), fakeClock, Duration.ZERO));
+            new CloudTasksHelper(fakeClock).getTestCloudTasksUtils());
   }
 
   private DomainBase persistLockedDomain(String domainName, String registrarId) {

core/src/test/java/google/registry/ui/server/registrar/RegistrarSettingsActionTest.java

@@ -18,13 +18,12 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestDataHelper.loadFile;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 
+import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -37,9 +36,9 @@ import google.registry.model.registrar.Registrar;
 import google.registry.request.auth.AuthenticatedRegistrarAccessor;
 import google.registry.request.auth.AuthenticatedRegistrarAccessor.Role;
 import google.registry.testing.CertificateSamples;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.SystemPropertyExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.CidrAddressBlock;
 import google.registry.util.EmailMessage;
@@ -56,6 +55,7 @@ import org.mockito.ArgumentCaptor;
 @DualDatabaseTest
 class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
 
+
   @RegisterExtension
   final SystemPropertyExtension systemPropertyExtension = new SystemPropertyExtension();
 
@@ -70,10 +70,12 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
     ArgumentCaptor<EmailMessage> contentCaptor = ArgumentCaptor.forClass(EmailMessage.class);
     verify(emailService).sendEmail(contentCaptor.capture());
     assertThat(contentCaptor.getValue().body()).isEqualTo(expectedEmailBody);
-    assertTasksEnqueued("sheet", new TaskMatcher()
-        .url(SyncRegistrarsSheetAction.PATH)
-        .method("GET")
-        .header("Host", "backend.hostname"));
+    cloudTasksHelper.assertTasksEnqueued(
+        "sheet",
+        new TaskMatcher()
+            .url(SyncRegistrarsSheetAction.PATH)
+            .service("Backend")
+            .method(HttpMethod.GET));
     assertMetric(CLIENT_ID, "update", "[OWNER]", "SUCCESS");
   }
 
@@ -86,7 +88,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
         "results", ImmutableList.of(),
         "message",
         "One email address (etphonehome@example.com) cannot be used for multiple contacts");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: ContactRequirementException");
   }
 
@@ -103,7 +105,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "status", "ERROR",
             "results", ImmutableList.of(),
             "message", "TestUserId doesn't have access to registrar TheRegistrar");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "read", "[]", "ERROR: ForbiddenException");
   }
 
@@ -134,7 +136,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
         "field", "lastUpdateTime",
         "results", ImmutableList.of(),
         "message", "This field is required.");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: FormFieldException");
   }
 
@@ -153,7 +155,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
         "field", "emailAddress",
         "results", ImmutableList.of(),
         "message", "This field is required.");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: FormFieldException");
   }
 
@@ -171,7 +173,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "status", "ERROR",
             "results", ImmutableList.of(),
             "message", "TestUserId doesn't have access to registrar TheRegistrar");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "update", "[]", "ERROR: ForbiddenException");
   }
 
@@ -190,7 +192,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
         "field", "emailAddress",
         "results", ImmutableList.of(),
         "message", "Please enter a valid email address.");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: FormFieldException");
   }
 
@@ -209,7 +211,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
         "field", "lastUpdateTime",
         "results", ImmutableList.of(),
         "message", "Not a valid ISO date-time string.");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: FormFieldException");
   }
 
@@ -228,7 +230,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
         "field", "emailAddress",
         "results", ImmutableList.of(),
         "message", "Please only use ASCII-US characters.");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: FormFieldException");
   }
 
@@ -265,9 +267,16 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
     Map<String, Object> response =
         action.handleJsonRequest(
             ImmutableMap.of(
-                "op", "update",
-                "id", CLIENT_ID,
-                "args", setter.apply(registrar.asBuilder(), newValue).build().toJsonMap()));
+                "op",
+                "update",
+                "id",
+                CLIENT_ID,
+                "args",
+                setter
+                    .apply(registrar.asBuilder(), newValue)
+                    .setLastUpdateTime(registrar.getLastUpdateTime())
+                    .build()
+                    .toJsonMap()));
     Registrar updatedRegistrar = loadRegistrar(CLIENT_ID);
     persistResource(registrar);
 
@@ -318,7 +327,11 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
                 "id",
                 CLIENT_ID,
                 "args",
-                setter.apply(registrar.asBuilder(), newValue).build().toJsonMap()));
+                setter
+                    .apply(registrar.asBuilder(), newValue)
+                    .setLastUpdateTime(registrar.getLastUpdateTime())
+                    .build()
+                    .toJsonMap()));
     Registrar updatedRegistrar = loadRegistrar(CLIENT_ID);
     persistResource(registrar);
 
@@ -405,7 +418,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "Certificate validity period is too long; it must be less than or equal to 398"
                 + " days.");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: IllegalArgumentException");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql
@@ -430,7 +443,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "Certificate is expired.\nCertificate validity period is too long; it must be less"
                 + " than or equal to 398 days.");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: IllegalArgumentException");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql
@@ -473,7 +486,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
 
     assertThat(response).containsEntry("status", "SUCCESS");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "SUCCESS");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql
@@ -498,7 +511,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "Certificate validity period is too long; it must be less than or equal to 398"
                 + " days.");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: IllegalArgumentException");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql
@@ -523,7 +536,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "Certificate is expired.\nCertificate validity period is too long; it must be less"
                 + " than or equal to 398 days.");
     assertMetric(CLIENT_ID, "update", "[OWNER]", "ERROR: IllegalArgumentException");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql
@@ -555,7 +568,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "results", ImmutableList.of(),
             "message", "Cannot add allowed TLDs if there is no WHOIS abuse contact set.");
     assertMetric(CLIENT_ID, "update", "[ADMIN]", "ERROR: IllegalArgumentException");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql
@@ -577,7 +590,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "results", ImmutableList.of(),
             "message", "TLDs do not exist: invalidtld");
     assertMetric(CLIENT_ID, "update", "[ADMIN]", "ERROR: IllegalArgumentException");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql
@@ -599,7 +612,7 @@ class RegistrarSettingsActionTest extends RegistrarSettingsActionTestCase {
             "results", ImmutableList.of(),
             "message", "Can't remove allowed TLDs using the console.");
     assertMetric(CLIENT_ID, "update", "[ADMIN]", "ERROR: ForbiddenException");
-    assertNoTasksEnqueued("sheet");
+    cloudTasksHelper.assertNoTasksEnqueued("sheet");
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/ui/server/registrar/RegistrarSettingsActionTestCase.java

@@ -46,6 +46,7 @@ import google.registry.request.auth.AuthResult;
 import google.registry.request.auth.AuthenticatedRegistrarAccessor;
 import google.registry.request.auth.UserAuthInfo;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
 import google.registry.ui.server.SendEmailUtils;
@@ -97,6 +98,8 @@ public abstract class RegistrarSettingsActionTestCase {
 
   RegistrarContact techContact;
 
+  CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
+
   @BeforeEach
   public void beforeEachRegistrarSettingsActionTestCase() throws Exception {
     // Registrar "TheRegistrar" has access to TLD "currenttld" but not to "newtld".
@@ -132,6 +135,8 @@ public abstract class RegistrarSettingsActionTestCase {
             2048,
             ImmutableSet.of("secp256r1", "secp384r1"),
             clock);
+    action.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
+
     inject.setStaticField(Ofy.class, "clock", clock);
     when(req.getMethod()).thenReturn("POST");
     when(rsp.getWriter()).thenReturn(new PrintWriter(writer));

core/src/test/java/google/registry/ui/server/registrar/RegistryLockPostActionTest.java

@@ -26,7 +26,6 @@ import static google.registry.testing.SqlHelper.getRegistryLockByVerificationCod
 import static google.registry.testing.SqlHelper.saveRegistryLock;
 import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
 import static google.registry.ui.server.registrar.RegistryLockGetActionTest.userFromRegistrarContact;
-import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
@@ -35,7 +34,6 @@ import com.google.appengine.api.users.User;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.RegistryLock;
 import google.registry.request.JsonActionRunner;
@@ -47,10 +45,10 @@ import google.registry.request.auth.AuthenticatedRegistrarAccessor;
 import google.registry.request.auth.AuthenticatedRegistrarAccessor.Role;
 import google.registry.request.auth.UserAuthInfo;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.FakeClock;
 import google.registry.tools.DomainLockUtils;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import google.registry.util.StringGenerator.Alphabets;
@@ -469,8 +467,7 @@ final class RegistryLockPostActionTest {
         new DomainLockUtils(
             new DeterministicStringGenerator(Alphabets.BASE_58),
             "adminreg",
-            AsyncTaskEnqueuerTest.createForTesting(
-                mock(AppEngineServiceUtils.class), clock, Duration.ZERO));
+            new CloudTasksHelper(clock).getTestCloudTasksUtils());
     return new RegistryLockPostAction(
         mockRequest,
         jsonActionRunner,

core/src/test/java/google/registry/ui/server/registrar/RegistryLockVerifyActionTest.java

@@ -33,7 +33,6 @@ import com.google.appengine.api.users.User;
 import com.google.appengine.api.users.UserService;
 import com.google.appengine.api.users.UserServiceFactory;
 import com.google.common.collect.ImmutableMap;
-import google.registry.batch.AsyncTaskEnqueuerTest;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.DomainBase;
@@ -47,13 +46,13 @@ import google.registry.request.auth.AuthResult;
 import google.registry.request.auth.UserAuthInfo;
 import google.registry.security.XsrfTokenManager;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.CloudTasksHelper;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.UserInfo;
 import google.registry.tools.DomainLockUtils;
-import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.StringGenerator;
 import google.registry.util.StringGenerator.Alphabets;
 import javax.servlet.http.HttpServletRequest;
@@ -86,6 +85,7 @@ final class RegistryLockVerifyActionTest {
   private DomainBase domain;
   private AuthResult authResult;
   private RegistryLockVerifyAction action;
+  private CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(fakeClock);
 
   @BeforeEach
   void beforeEach() {
@@ -329,10 +329,7 @@ final class RegistryLockVerifyActionTest {
     RegistryLockVerifyAction action =
         new RegistryLockVerifyAction(
             new DomainLockUtils(
-                stringGenerator,
-                "adminreg",
-                AsyncTaskEnqueuerTest.createForTesting(
-                    mock(AppEngineServiceUtils.class), fakeClock, Duration.ZERO)),
+                stringGenerator, "adminreg", cloudTasksHelper.getTestCloudTasksUtils()),
             lockVerificationCode,
             isLock);
     authResult = AuthResult.create(AuthLevel.USER, UserAuthInfo.create(user, false));

core/src/test/resources/google/registry/module/backend/backend_routing.txt

@@ -39,6 +39,7 @@ PATH                                               CLASS
 /_dr/task/resaveAllEppResources                    ResaveAllEppResourcesAction                    GET      n  INTERNAL,API APP ADMIN
 /_dr/task/resaveEntity                             ResaveEntityAction                             POST     n  INTERNAL,API APP ADMIN
 /_dr/task/sendExpiringCertificateNotificationEmail SendExpiringCertificateNotificationEmailAction GET      n  INTERNAL,API APP ADMIN
+/_dr/task/syncDatastoreToSqlSnapshot               SyncDatastoreToSqlSnapshotAction               POST     n  INTERNAL,API APP ADMIN
 /_dr/task/syncGroupMembers                         SyncGroupMembersAction                         POST     n  INTERNAL,API APP ADMIN
 /_dr/task/syncRegistrarsSheet                      SyncRegistrarsSheetAction                      POST     n  INTERNAL,API APP ADMIN
 /_dr/task/tmchCrl                                  TmchCrlAction                                  POST     y  INTERNAL,API APP ADMIN

dependencies.gradle

@@ -89,6 +89,7 @@ ext {
       'com.google.oauth-client:google-oauth-client-jetty:1.31.4',
       'com.google.oauth-client:google-oauth-client-servlet:1.31.4',
       'com.google.protobuf:protobuf-java:3.13.0',
+      'com.google.protobuf:protobuf-java-util:3.17.3',
       'com.google.re2j:re2j:1.6',
       'com.google.template:soy:2021-02-01',
       'com.google.truth.extensions:truth-java8-extension:1.1.2',

networking/gradle/dependency-locks/compileClasspath.lockfile

@@ -30,6 +30,7 @@ com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client:1.39.2
 com.google.j2objc:j2objc-annotations:1.3
 com.google.oauth-client:google-oauth-client:1.31.4
+com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.ibm.icu:icu4j:68.2

networking/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -34,6 +34,7 @@ com.google.http-client:google-http-client-gson:1.39.2
 com.google.http-client:google-http-client:1.39.2
 com.google.j2objc:j2objc-annotations:1.3
 com.google.oauth-client:google-oauth-client:1.31.4
+com.google.protobuf:protobuf-java-util:3.17.3
 com.google.protobuf:protobuf-java:3.17.3
 com.google.re2j:re2j:1.6
 com.google.truth:truth:1.1.2