Replace schedule time calculation with createTask() variation from CloudTasksUtils (#1443)

* Filter out empty dsData objects, not just null ones

Hibernate/SQL will get mad if the digest is null or empty, and
previously we only check for null. We should filter out empty digests as
well.

config/presubmits.py

@@ -187,51 +187,6 @@ PRESUBMITS = {
         {"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
     ):
         "JavaScript files should not include console logging.",
-    # SQL injection protection rule for java source file:
-    # The sql template passed to createQuery/createNativeQuery methods must be
-    # a variable name in UPPER_CASE_UNDERSCORE format, i.e., a static final
-    # String variable. This forces the use of parameter-binding on all queries
-    # that take parameters.
-    # The rule would forbid invocation of createQuery(Criteria). However, this
-    # can be handled by adding a helper method in an exempted class to make
-    # the calls.
-    # TODO(b/179158393): enable the 'ConstantName' Java style check to ensure
-    # that non-final variables do not use the UPPER_CASE_UNDERSCORE format.
-    PresubmitCheck(
-        # Line 1: the method names we check and the opening parenthesis, which
-        #    marks the beginning of the first parameter
-        # Line 2: The first parameter is a match if is NOT any of the following:
-        #    - final variable name: \s*([A-Z_]+
-        #    - string literal: "([^"]|\\")*"
-        #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
-        # Line 3: , or the closing parenthesis, marking the end of the first
-        #    parameter
-        r'.*\.(query|createQuery|createNativeQuery)\('
-        r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
-        r'(,|\s*\))))',
-        "java",
-        # ActivityReportingQueryBuilder deals with Dremel queries
-        {"src/test", "ActivityReportingQueryBuilder.java",
-         # This class contains helper method to make queries in Beam.
-         "RegistryJpaIO.java",
-         "CreateSyntheticHistoryEntriesAction.java",
-         # TODO(b/179158393): Remove everything below, which should be done
-         # using Criteria
-         "JpaTransactionManager.java",
-         "JpaTransactionManagerImpl.java",
-         # CriteriaQueryBuilder is a false positive
-         "CriteriaQueryBuilder.java",
-         "RdapDomainSearchAction.java",
-         "RdapNameserverSearchAction.java",
-         "ReadOnlyCheckingEntityManager.java",
-         "RegistryQuery",
-         },
-    ):
-        "The first String parameter to EntityManager.create(Native)Query "
-        "methods must be one of the following:\n"
-        "  - A String literal\n"
-        "  - Concatenation of String literals only\n"
-        "  - The name of a static final String variable"
 }
 
 # Note that this regex only works for one kind of Flyway file.  If we want to

core/build.gradle

@@ -470,7 +470,7 @@ task soyToJava {
 
     outputs.each { file ->
       exec {
-        commandLine 'sed', '-i', 's/@link/LINK/g', file.getCanonicalPath()
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
       }
     }
   }
@@ -703,6 +703,12 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
+createToolTask(
+    'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
+
+createToolTask(
+    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
+
 
 createToolTask(
     'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
@@ -711,30 +717,6 @@ createToolTask(
     'createSyntheticHistoryEntries',
     'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
-project.tasks.create('initSqlPipeline', JavaExec) {
-  main = 'google.registry.beam.initsql.InitSqlPipeline'
-
-  doFirst {
-    getToolArgsList().ifPresent {
-      args it
-    }
-
-    def isDirectRunner =
-        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
-    // The dependency containing DirectRunner is intentionally excluded from the
-    // production binary, so that it won't be chosen by mistake: we definitely do
-    // not want to use it for the real jobs, yet DirectRunner is the default if
-    // the user forgets to override it.
-    // DirectRunner is required for tests and is already on testRuntimeClasspath.
-    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
-    // of defining a new configuration just for the DirectRunner dependency.
-    classpath =
-        isDirectRunner
-            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
-            : sourceSets.main.runtimeClasspath
-  }
-}
-
 // Caller must provide projectId, GCP region, runner, and the kinds to delete
 // (comma-separated kind names or '*' for all). E.g.:
 // nom_build :core:bulkDeleteDatastore --args="--project=domain-registry-crash \

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -41,11 +41,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnlyWithoutBackup().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -14,21 +14,21 @@
 
 package google.registry.backup;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -56,7 +56,8 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Inject Clock clock;
   @Inject CommitLogCheckpointStrategy strategy;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject CommitLogCheckpointAction() {}
 
   @Override
@@ -73,16 +74,20 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveWithoutBackup()
+                  .saveIgnoringReadOnlyWithoutBackup()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.
-              taskQueueUtils.enqueue(
-                  getQueue(QUEUE_NAME),
-                  withUrl(ExportCommitLogDiffAction.PATH)
-                      .param(LOWER_CHECKPOINT_TIME_PARAM, lastWrittenTime.toString())
-                      .param(
-                          UPPER_CHECKPOINT_TIME_PARAM, checkpoint.getCheckpointTime().toString()));
+              cloudTasksUtils.enqueue(
+                  QUEUE_NAME,
+                  CloudTasksUtils.createPostTask(
+                      ExportCommitLogDiffAction.PATH,
+                      Service.BACKEND.toString(),
+                      ImmutableMultimap.of(
+                          LOWER_CHECKPOINT_TIME_PARAM,
+                          lastWrittenTime.toString(),
+                          UPPER_CHECKPOINT_TIME_PARAM,
+                          checkpoint.getCheckpointTime().toString())));
             });
   }
 }

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -140,7 +140,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 
@@ -264,7 +264,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             .ifPresent(
                 sqlEntity -> {
                   sqlEntity.beforeSqlSaveOnReplay();
-                  jpaTm().putIgnoringReadOnly(sqlEntity);
+                  jpaTm().putIgnoringReadOnlyWithoutBackup(sqlEntity);
                 });
       } else {
         // this should never happen, but we shouldn't fail on it
@@ -297,7 +297,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
           && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
           && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
         ReplaySpecializer.beforeSqlDelete(entityVKey);
-        jpaTm().deleteIgnoringReadOnly(entityVKey);
+        jpaTm().deleteIgnoringReadOnlyWithoutBackup(entityVKey);
       }
     } catch (Throwable t) {
       logger.atSevere().log("Error when deleting key %s.", entityVKey);

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,10 +24,8 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
@@ -84,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -96,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -114,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -129,14 +125,13 @@ public final class AsyncTaskEnqueuer {
       String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingRegistrarId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))

core/src/main/java/google/registry/batch/BatchModule.java

@@ -79,6 +79,7 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
+  // TODO(b/207363014): figure out if this needs to be modified for vkey string replacement
   static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
     return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
   }

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -523,18 +523,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -353,8 +353,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
       VKey<HostResource> hostKey =
-          VKey.fromWebsafeKey(
-              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
           tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
               .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -76,13 +76,15 @@ public class ResaveEntityAction implements Runnable {
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
     tm().transact(
             () -> {
+              // TODO(/207363014): figure out if this should modified for vkey string replacement
               ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
               tm().put(
                       (entity instanceof EppResource)
                           ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
                           : entity);
               if (!resaveTimes.isEmpty()) {
-                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.from(resourceKey), requestedTime, resaveTimes);
               }
             });
     response.setPayload("Entity re-saved.");

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -175,7 +175,7 @@ public final class RegistryJpaIO {
      * JpaTransactionManager#setDatabaseSnapshot}.
      */
     // TODO(b/193662898): vendor-independent support for richer transaction semantics.
-    public Read<R, T> withSnapshot(String snapshotId) {
+    public Read<R, T> withSnapshot(@Nullable String snapshotId) {
       return toBuilder().snapshotId(snapshotId).build();
     }
 

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -63,7 +63,8 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
     // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
     // loads all ofy entities.
-    new AppEngineEnvironment("Beam").setEnvironmentForAllThreads();
+    new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
+        .setEnvironmentForAllThreads();
     // Set the system property so that we can call IdService.allocateId() without access to
     // datastore.
     SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");

core/src/main/java/google/registry/beam/comparedb/DatastoreSnapshots.java

@@ -0,0 +1,169 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.initsql.Transforms.createTagForKind;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.Transforms;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.joda.time.DateTime;
+
+/** Utilities for loading Datastore snapshots. */
+public final class DatastoreSnapshots {
+
+  private DatastoreSnapshots() {}
+
+  /**
+   * Datastore kinds eligible for validation. This set must be consistent with {@link
+   * SqlSnapshots#ALL_SQL_ENTITIES}.
+   */
+  @VisibleForTesting
+  static final ImmutableSet<Class<?>> ALL_DATASTORE_KINDS =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          HistoryEntry.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class);
+
+  /**
+   * Returns the Datastore snapshot right before {@code commitLogToTime} for the user specified
+   * {@code kinds}. The resulting snapshot has all changes that happened before {@code
+   * commitLogToTime}, and none at or after {@code commitLogToTime}.
+   *
+   * <p>If {@code HistoryEntry} is included in {@code kinds}, the result will contain {@code
+   * PCollections} for the child entities, {@code DomainHistory}, {@code ContactHistory}, and {@code
+   * HostHistory}.
+   */
+  static PCollectionTuple loadDatastoreSnapshotByKind(
+      Pipeline pipeline,
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<Class<?>> kinds) {
+    PCollectionTuple snapshot =
+        pipeline.apply(
+            "Load Datastore snapshot.",
+            Transforms.loadDatastoreSnapshot(
+                exportDir,
+                commitLogDir,
+                commitLogFromTime,
+                commitLogToTime,
+                kinds.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet())));
+
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<?> kind : kinds) {
+      PCollection<VersionedEntity> perKindSnapshot =
+          snapshot.get(createTagForKind(Key.getKind(kind)));
+      if (SqlEntity.class.isAssignableFrom(kind)) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag((Class<? extends SqlEntity>) kind),
+                datastoreEntityToPojo(perKindSnapshot, kind.getSimpleName()));
+        continue;
+      }
+      Verify.verify(kind == HistoryEntry.class, "Unexpected Non-SqlEntity class: %s", kind);
+      PCollectionTuple historyEntriesByType = splitHistoryEntry(perKindSnapshot);
+      for (Map.Entry<TupleTag<?>, PCollection<?>> entry :
+          historyEntriesByType.getAll().entrySet()) {
+        perTypeSnapshots = perTypeSnapshots.and(entry.getKey().getId(), entry.getValue());
+      }
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Splits a {@link PCollection} of {@link HistoryEntry HistoryEntries} into three collections of
+   * its child entities by type.
+   */
+  static PCollectionTuple splitHistoryEntry(PCollection<VersionedEntity> historyEntries) {
+    return historyEntries.apply(
+        "Split HistoryEntry by Resource Type",
+        ParDo.of(
+                new DoFn<VersionedEntity, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity historyEntry, MultiOutputReceiver out) {
+                    Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(historyEntry))
+                        .ifPresent(
+                            sqlEntity ->
+                                out.get(createSqlEntityTupleTag(sqlEntity.getClass()))
+                                    .output(sqlEntity));
+                  }
+                })
+            .withOutputTags(
+                createSqlEntityTupleTag(DomainHistory.class),
+                TupleTagList.of(createSqlEntityTupleTag(ContactHistory.class))
+                    .and(createSqlEntityTupleTag(HostHistory.class))));
+  }
+
+  /**
+   * Transforms a {@link PCollection} of {@link VersionedEntity VersionedEntities} to Ofy Java
+   * objects.
+   */
+  static PCollection<SqlEntity> datastoreEntityToPojo(
+      PCollection<VersionedEntity> entities, String desc) {
+    return entities.apply(
+        "Datastore Entity to Pojo " + desc,
+        ParDo.of(
+            new DoFn<VersionedEntity, SqlEntity>() {
+              @ProcessElement
+              public void processElement(
+                  @Element VersionedEntity entity, OutputReceiver<SqlEntity> out) {
+                Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(entity))
+                    .ifPresent(out::output);
+              }
+            }));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -0,0 +1,145 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import dagger.Component;
+import google.registry.config.CloudTasksUtilsModule;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.util.Clock;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.joda.time.DateTime;
+import org.joda.time.Instant;
+import org.joda.time.Interval;
+
+/** Finds the necessary information for loading the most recent Datastore snapshot. */
+public class LatestDatastoreSnapshotFinder {
+  private final String projectId;
+  private final GcsUtils gcsUtils;
+  private final Clock clock;
+
+  @Inject
+  LatestDatastoreSnapshotFinder(
+      @Config("projectId") String projectId, GcsUtils gcsUtils, Clock clock) {
+    this.projectId = projectId;
+    this.gcsUtils = gcsUtils;
+    this.clock = clock;
+  }
+
+  /**
+   * Finds information of the most recent Datastore snapshot, including the GCS folder of the
+   * exported data files and the start and stop times of the export. The folder of the CommitLogs is
+   * also included in the return.
+   */
+  public DatastoreSnapshotInfo getSnapshotInfo() {
+    String bucketName = RegistryConfig.getDatastoreBackupsBucket().substring("gs://".length());
+    /**
+     * Find the bucket-relative path to the overall metadata file of the last Datastore export.
+     * Since Datastore export is saved daily, we may need to look back to yesterday. If found, the
+     * return value is like
+     * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
+     */
+    Optional<String> metaFilePathOptional = findMostRecentExportMetadataFile(bucketName, 2);
+    if (!metaFilePathOptional.isPresent()) {
+      throw new NoSuchElementException("No exports found over the past 2 days.");
+    }
+    String metaFilePath = metaFilePathOptional.get();
+    String metaFileFolder = metaFilePath.substring(0, metaFilePath.indexOf('/'));
+    Instant exportStartTime = Instant.parse(metaFileFolder.replace('_', '.') + 'Z');
+    BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath));
+    Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+    return DatastoreSnapshotInfo.create(
+        String.format("gs://%s/%s", bucketName, metaFileFolder),
+        getCommitLogDir(),
+        new Interval(exportStartTime, exportEndTime));
+  }
+
+  public String getCommitLogDir() {
+    return "gs://" + projectId + "-commits";
+  }
+
+  /**
+   * Finds the bucket-relative path of the overall export metadata file, in the given bucket,
+   * searching back up to {@code lookBackDays} days, including today.
+   *
+   * <p>The overall export metadata file is the last file created during a Datastore export. All
+   * data has been exported by the creation time of this file. The name of this file, like that of
+   * all files in the same export, begins with the timestamp when the export starts.
+   *
+   * <p>An example return value: {@code
+   * 2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata}.
+   */
+  private Optional<String> findMostRecentExportMetadataFile(String bucketName, int lookBackDays) {
+    DateTime today = clock.nowUtc();
+    for (int day = 0; day < lookBackDays; day++) {
+      String dateString = today.minusDays(day).toString("yyyy-MM-dd");
+      try {
+        Optional<String> metaFilePath =
+            gcsUtils.listFolderObjects(bucketName, dateString).stream()
+                .filter(s -> s.endsWith("overall_export_metadata"))
+                .map(s -> dateString + s)
+                .sorted(Comparator.<String>naturalOrder().reversed())
+                .findFirst();
+        if (metaFilePath.isPresent()) {
+          return metaFilePath;
+        }
+      } catch (IOException ioe) {
+        throw new RuntimeException(ioe);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /** Holds information about a Datastore snapshot. */
+  @AutoValue
+  abstract static class DatastoreSnapshotInfo {
+    abstract String exportDir();
+
+    abstract String commitLogDir();
+
+    abstract Interval exportInterval();
+
+    static DatastoreSnapshotInfo create(
+        String exportDir, String commitLogDir, Interval exportOperationInterval) {
+      return new AutoValue_LatestDatastoreSnapshotFinder_DatastoreSnapshotInfo(
+          exportDir, commitLogDir, exportOperationInterval);
+    }
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
+  interface LatestDatastoreSnapshotFinderFinderComponent {
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotInfoFinder();
+  }
+}

core/src/main/java/google/registry/beam/comparedb/SqlSnapshots.java

@@ -0,0 +1,384 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.comparedb.ValidateSqlUtils.getMedianIdForHistoryTable;
+
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Streams;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.model.bulkquery.DomainBaseLite;
+import google.registry.model.bulkquery.DomainHistoryHost;
+import google.registry.model.bulkquery.DomainHistoryLite;
+import google.registry.model.bulkquery.DomainHost;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+
+/**
+ * Utilities for loading SQL snapshots.
+ *
+ * <p>For {@link DomainBase} and {@link DomainHistory}, this class assumes the presence of the
+ * {@link google.registry.persistence.PersistenceModule.JpaTransactionManagerType#BULK_QUERY
+ * bulk-query-capable JpaTransactionManager}, and takes advantage of it for higher throughput.
+ *
+ * <p>For now this class is meant for use during the database migration period only. Therefore, it
+ * contains optimizations specifically for the production database at the current size, e.g.,
+ * parallel queries for select tables.
+ */
+public final class SqlSnapshots {
+
+  private SqlSnapshots() {}
+
+  /**
+   * SQL entity types that are eligible for validation. This set must be consistent with {@link
+   * DatastoreSnapshots#ALL_DATASTORE_KINDS}.
+   */
+  static final ImmutableSet<Class<? extends SqlEntity>> ALL_SQL_ENTITIES =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class,
+          ContactHistory.class,
+          HostHistory.class,
+          DomainHistory.class);
+
+  /**
+   * Loads a SQL snapshot for the given {@code sqlEntityTypes}.
+   *
+   * <p>If {@code snapshotId} is present, all queries use the specified database snapshot,
+   * guaranteeing a consistent result.
+   */
+  public static PCollectionTuple loadCloudSqlSnapshotByType(
+      Pipeline pipeline,
+      ImmutableSet<Class<? extends SqlEntity>> sqlEntityTypes,
+      Optional<String> snapshotId) {
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<? extends SqlEntity> clazz : sqlEntityTypes) {
+      if (clazz == DomainBase.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainBase.class),
+                loadAndAssembleDomainBase(pipeline, snapshotId));
+        continue;
+      }
+      if (clazz == DomainHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainHistory.class),
+                loadAndAssembleDomainHistory(pipeline, snapshotId));
+        continue;
+      }
+      if (clazz == ContactHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(ContactHistory.class),
+                loadContactHistory(pipeline, snapshotId));
+        continue;
+      }
+      perTypeSnapshots =
+          perTypeSnapshots.and(
+              createSqlEntityTupleTag(clazz),
+              pipeline.apply(
+                  "SQL Load " + clazz.getSimpleName(),
+                  RegistryJpaIO.read(
+                          () -> CriteriaQueryBuilder.create(clazz).build(), SqlEntity.class::cast)
+                      .withSnapshot(snapshotId.orElse(null))));
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Bulk-loads parts of {@link DomainBase} and assembles them in the pipeline.
+   *
+   * @see BulkQueryEntities
+   */
+  public static PCollection<SqlEntity> loadAndAssembleDomainBase(
+      Pipeline pipeline, Optional<String> snapshotId) {
+    PCollection<KV<String, Serializable>> baseObjects =
+        readAllAndAssignKey(pipeline, DomainBaseLite.class, DomainBaseLite::getRepoId, snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(pipeline, GracePeriod.class, GracePeriod::getDomainRepoId, snapshotId);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DelegationSignerData.class,
+            DelegationSignerData::getDomainRepoId,
+            snapshotId);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(pipeline, DomainHost.class, DomainHost::getDomainRepoId, snapshotId);
+
+    return PCollectionList.of(
+            ImmutableList.of(baseObjects, gracePeriods, delegationSigners, domainHosts))
+        .apply("SQL Merge DomainBase parts", Flatten.pCollections())
+        .apply("Group by Domain Parts by RepoId", GroupByKey.create())
+        .apply(
+            "Assemble DomainBase",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainBaseLite> baseObjects =
+                        partsByType.getAllOf(DomainBaseLite.class);
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainBaseLite object per repoId: " + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainBase(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(GracePeriod.class),
+                            partsByType.getAllOf(DelegationSignerData.class),
+                            partsByType.getAllOf(DomainHost.class).stream()
+                                .map(DomainHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet())));
+                  }
+                }));
+  }
+
+  /**
+   * Loads all {@link ContactHistory} entities from the database.
+   *
+   * <p>This method uses two queries to load data in parallel. This is a performance optimization
+   * specifically for the production database.
+   */
+  static PCollection<SqlEntity> loadContactHistory(Pipeline pipeline, Optional<String> snapshotId) {
+    long medianId =
+        getMedianIdForHistoryTable("ContactHistory")
+            .orElseThrow(
+                () -> new IllegalStateException("Not a valid database: no ContactHistory."));
+    PCollection<SqlEntity> part1 =
+        pipeline.apply(
+            "SQL Load ContactHistory first half",
+            RegistryJpaIO.read(
+                    String.format("select c from ContactHistory c where id <= %s", medianId),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    PCollection<SqlEntity> part2 =
+        pipeline.apply(
+            "SQL Load ContactHistory second half",
+            RegistryJpaIO.read(
+                    String.format("select c from ContactHistory c where id > %s", medianId),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    return PCollectionList.of(part1)
+        .and(part2)
+        .apply("Combine ContactHistory parts", Flatten.pCollections());
+  }
+
+  /**
+   * Bulk-loads all parts of {@link DomainHistory} and assembles them in the pipeline.
+   *
+   * <p>This method uses two queries to load {@link DomainBaseLite} in parallel. This is a
+   * performance optimization specifically for the production database.
+   *
+   * @see BulkQueryEntities
+   */
+  static PCollection<SqlEntity> loadAndAssembleDomainHistory(
+      Pipeline pipeline, Optional<String> snapshotId) {
+    long medianId =
+        getMedianIdForHistoryTable("DomainHistory")
+            .orElseThrow(
+                () -> new IllegalStateException("Not a valid database: no DomainHistory."));
+    PCollection<KV<String, Serializable>> baseObjectsPart1 =
+        queryAndAssignKey(
+            pipeline,
+            "first half",
+            String.format("select c from DomainHistory c where id <= %s", medianId),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> baseObjectsPart2 =
+        queryAndAssignKey(
+            pipeline,
+            "second half",
+            String.format("select c from DomainHistory c where id > %s", medianId),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriodHistory.class,
+            compose(GracePeriodHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DomainDsDataHistory.class,
+            compose(DomainDsDataHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline,
+            DomainHistoryHost.class,
+            compose(DomainHistoryHost::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> transactionRecords =
+        readAllAndAssignKey(
+            pipeline,
+            DomainTransactionRecord.class,
+            compose(DomainTransactionRecord::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+
+    return PCollectionList.of(
+            ImmutableList.of(
+                baseObjectsPart1,
+                baseObjectsPart2,
+                gracePeriods,
+                delegationSigners,
+                domainHosts,
+                transactionRecords))
+        .apply("Merge DomainHistory parts", Flatten.pCollections())
+        .apply("Group by DomainHistory Parts by DomainHistoryId string", GroupByKey.create())
+        .apply(
+            "Assemble DomainHistory",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainHistoryLite> baseObjects =
+                        partsByType.getAllOf(DomainHistoryLite.class);
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainHistoryLite object per domainHistoryId: "
+                            + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainHistory(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(DomainDsDataHistory.class),
+                            partsByType.getAllOf(DomainHistoryHost.class).stream()
+                                .map(DomainHistoryHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet()),
+                            partsByType.getAllOf(GracePeriodHistory.class),
+                            partsByType.getAllOf(DomainTransactionRecord.class)));
+                  }
+                }));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> readAllAndAssignKey(
+      Pipeline pipeline,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName(),
+            RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(type).build())
+                .withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName(),
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> queryAndAssignKey(
+      Pipeline pipeline,
+      String diffrentiator,
+      String jplQuery,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName() + " " + diffrentiator,
+            RegistryJpaIO.read(jplQuery, false, type::cast).withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName() + " " + diffrentiator,
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  // TODO(b/205988530): don't use beam serializablefunction, make one that extends Java's Function.
+  private static <R, I, T> SerializableFunction<R, T> compose(
+      SerializableFunction<R, I> f1, SerializableFunction<I, T> f2) {
+    return r -> f2.apply(f1.apply(r));
+  }
+
+  /** Container that receives mixed-typed data and groups them by {@link Class}. */
+  static class TypedClassifier {
+    private final ImmutableSetMultimap<Class<?>, Object> classifiedEntities;
+
+    TypedClassifier(Iterable<Serializable> inputs) {
+      this.classifiedEntities =
+          Streams.stream(inputs)
+              .collect(ImmutableSetMultimap.toImmutableSetMultimap(Object::getClass, x -> x));
+    }
+
+    <T> ImmutableSet<T> getAllOf(Class<T> clazz) {
+      return classifiedEntities.get(clazz).stream()
+          .map(clazz::cast)
+          .collect(ImmutableSet.toImmutableSet());
+    }
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -0,0 +1,159 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+
+/**
+ * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
+ * (secondary storage).
+ */
+public class ValidateSqlPipeline {
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final int COMMIT_LOG_MARGIN_MINUTES = 10;
+
+  private final ValidateSqlPipelineOptions options;
+  private final DatastoreSnapshotInfo mostRecentExport;
+
+  public ValidateSqlPipeline(
+      ValidateSqlPipelineOptions options, DatastoreSnapshotInfo mostRecentExport) {
+    this.options = options;
+    this.mostRecentExport = mostRecentExport;
+  }
+
+  void run() {
+    run(Pipeline.create(options));
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    // TODO(weiminyu): Acquire the commit log replay lock when the lock release bug is fixed.
+    DateTime latestCommitLogTime =
+        TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+    Preconditions.checkState(
+        latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
+        "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
+    try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+      setupPipeline(pipeline, Optional.of(databaseSnapshot.getSnapshotId()), latestCommitLogTime);
+      State state = pipeline.run().waitUntilFinish();
+      if (!State.DONE.equals(state)) {
+        throw new IllegalStateException("Unexpected pipeline state: " + state);
+      }
+    }
+  }
+
+  void setupPipeline(
+      Pipeline pipeline, Optional<String> sqlSnapshotId, DateTime latestCommitLogTime) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minusMinutes(COMMIT_LOG_MARGIN_MINUTES),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      PCollectionList.of(datastoreSnapshot.get(tag))
+          .and(cloudSqlSnapshot.get(tag))
+          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
+          .apply(
+              "Assign primary key to merged " + clazz.getSimpleName(),
+              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
+          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  public static void main(String[] args) {
+    ValidateSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    DatastoreSnapshotInfo mostRecentExport =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder()
+            .getSnapshotInfo();
+
+    new ValidateSqlPipeline(options, mostRecentExport).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -0,0 +1,20 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+
+/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
+public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -0,0 +1,270 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.contact.ContactBase;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.eppcommon.AuthInfo;
+import google.registry.model.host.HostHistory;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import java.lang.reflect.Field;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Helpers for use by {@link ValidateSqlPipeline}. */
+final class ValidateSqlUtils {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ValidateSqlUtils() {}
+
+  /**
+   * Query template for finding the median value of the {@code history_revision_id} column in one of
+   * the History tables.
+   *
+   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * history tables. Although the {@code repo_id} column is the leading column in the primary keys
+   * of these tables, in practice and with production data, division by {@code history_revision_id}
+   * works slightly faster for unknown reasons.
+   */
+  private static final String MEDIAN_ID_QUERY_TEMPLATE =
+      "SELECT history_revision_id FROM (                                                        "
+          + "  SELECT"
+          + "    ROW_NUMBER() OVER (ORDER BY history_revision_id ASC) AS rownumber,"
+          + "    history_revision_id"
+          + "  FROM \"%TABLE%\""
+          + ") AS foo\n"
+          + "WHERE rownumber in (select count(*) / 2 + 1 from \"%TABLE%\")";
+
+  static Optional<Long> getMedianIdForHistoryTable(String tableName) {
+    Preconditions.checkArgument(
+        tableName.endsWith("History"), "Table must be one of the History tables.");
+    String sqlText = MEDIAN_ID_QUERY_TEMPLATE.replace("%TABLE%", tableName);
+    List results =
+        jpaTm()
+            .transact(() -> jpaTm().getEntityManager().createNativeQuery(sqlText).getResultList());
+    verify(results.size() < 2, "MidPoint query should have at most one result.");
+    if (results.isEmpty()) {
+      return Optional.empty();
+    }
+    return Optional.of(((BigInteger) results.get(0)).longValue());
+  }
+
+  static TupleTag<SqlEntity> createSqlEntityTupleTag(Class<? extends SqlEntity> actualType) {
+    return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
+  }
+
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+    private final HashMap<String, Counter> totalCounters = new HashMap<>();
+    private final HashMap<String, Counter> missingCounters = new HashMap<>();
+    private final HashMap<String, Counter> unequalCounters = new HashMap<>();
+    private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
+
+    private volatile boolean logPrinted = false;
+
+    private String getCounterKey(Class<?> clazz) {
+      return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
+    }
+
+    private synchronized void ensureCounterExists(String counterKey) {
+      if (totalCounters.containsKey(counterKey)) {
+        return;
+      }
+      totalCounters.put(counterKey, Metrics.counter("CompareDB", "Total Compared: " + counterKey));
+      missingCounters.put(
+          counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
+      unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
+      badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
+    }
+
+    /**
+     * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
+     * This will be removed when we start exporting such entities to GCS.
+     */
+    void logDiff(String key, Object entry0, Object entry1) {
+      if (logPrinted) {
+        return;
+      }
+      logPrinted = true;
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      StringBuilder sb = new StringBuilder();
+      fields0.forEach(
+          (field, value) -> {
+            if (fields1.containsKey(field)) {
+              if (!Objects.equals(value, fields1.get(field))) {
+                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
+              }
+            } else {
+              sb.append(field + "Not found in entity 2\n");
+            }
+          });
+      fields1.forEach(
+          (field, value) -> {
+            if (!fields0.containsKey(field)) {
+              sb.append(field + "Not found in entity 1\n");
+            }
+          });
+      logger.atWarning().log(key + "  " + sb.toString());
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+      ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
+
+      verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
+      verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
+
+      String counterKey = getCounterKey(entities.get(0).getClass());
+      ensureCounterExists(counterKey);
+      totalCounters.get(counterKey).inc();
+
+      if (entities.size() == 1) {
+        missingCounters.get(counterKey).inc();
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
+        }
+        return;
+      }
+      SqlEntity entity0;
+      SqlEntity entity1;
+
+      try {
+        entity0 = normalizeEntity(entities.get(0));
+        entity1 = normalizeEntity(entities.get(1));
+      } catch (Exception e) {
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          badEntityCounters.get(counterKey).inc();
+        }
+        return;
+      }
+
+      if (!Objects.equals(entity0, entity1)) {
+        unequalCounters.get(counterKey).inc();
+        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+      }
+    }
+  }
+
+  static SqlEntity normalizeEntity(SqlEntity sqlEntity) {
+    if (sqlEntity instanceof EppResource) {
+      return normalizeEppResource(sqlEntity);
+    }
+    if (sqlEntity instanceof HistoryEntry) {
+      return (SqlEntity) normalizeHistoryEntry((HistoryEntry) sqlEntity);
+    }
+    return sqlEntity;
+  }
+
+  /**
+   * Normalizes an {@link EppResource} instance for comparison.
+   *
+   * <p>This method may modify the input object using reflection instead of making a copy with
+   * {@code eppResource.asBuilder().build()}, because when {@code eppResource} is a {@link
+   * google.registry.model.domain.DomainBase}, the {@code build} method accesses the Database, which
+   * we want to avoid.
+   */
+  static SqlEntity normalizeEppResource(SqlEntity eppResource) {
+    try {
+      Field authField =
+          eppResource instanceof DomainContent
+              ? DomainContent.class.getDeclaredField("authInfo")
+              : eppResource instanceof ContactBase
+                  ? ContactBase.class.getDeclaredField("authInfo")
+                  : null;
+      if (authField != null) {
+        authField.setAccessible(true);
+        AuthInfo authInfo = (AuthInfo) authField.get(eppResource);
+        // When AuthInfo is missing, the authInfo field is null if the object is loaded from
+        // Datastore, or a PasswordAuth with null properties if loaded from SQL. In the second case
+        // we set the authInfo field to null.
+        if (authInfo != null
+            && authInfo.getPw() != null
+            && authInfo.getPw().getRepoId() == null
+            && authInfo.getPw().getValue() == null) {
+          authField.set(eppResource, null);
+        }
+      }
+
+      Field field = EppResource.class.getDeclaredField("revisions");
+      field.setAccessible(true);
+      field.set(eppResource, null);
+      return eppResource;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Normalizes a {@link HistoryEntry} for comparison.
+   *
+   * <p>This method modifies the input using reflection because relevant builder methods performs
+   * unwanted checks and changes.
+   */
+  static HistoryEntry normalizeHistoryEntry(HistoryEntry historyEntry) {
+    // History objects from Datastore do not have details of their EppResource objects
+    // (domainContent, contactBase, hostBase).
+    try {
+      if (historyEntry instanceof DomainHistory) {
+        Field domainContent = DomainHistory.class.getDeclaredField("domainContent");
+        domainContent.setAccessible(true);
+        domainContent.set(historyEntry, null);
+        Field domainTransactionRecords =
+            HistoryEntry.class.getDeclaredField("domainTransactionRecords");
+        domainTransactionRecords.setAccessible(true);
+        Set<?> domainTransactionRecordsValue = (Set<?>) domainTransactionRecords.get(historyEntry);
+        if (domainTransactionRecordsValue != null && domainTransactionRecordsValue.isEmpty()) {
+          domainTransactionRecords.set(historyEntry, null);
+        }
+      } else if (historyEntry instanceof ContactHistory) {
+        Field contactBase = ContactHistory.class.getDeclaredField("contactBase");
+        contactBase.setAccessible(true);
+        contactBase.set(historyEntry, null);
+      } else if (historyEntry instanceof HostHistory) {
+        Field hostBase = HostHistory.class.getDeclaredField("hostBase");
+        hostBase.setAccessible(true);
+        hostBase.set(historyEntry, null);
+      }
+      return historyEntry;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -101,8 +101,9 @@ public final class Transforms {
   }
 
   /**
-   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
-   * commitLogToTime} for caller specified {@code kinds}.
+   * Composite {@link PTransform transform} that loads the Datastore snapshot right before {@code
+   * commitLogToTime} for caller specified {@code kinds}. The resulting snapshot has all changes
+   * that happened before {@code commitLogToTime}, and none at or after {@code commitLogToTime}.
    *
    * <p>Caller must provide the location of a Datastore export that started AFTER {@code
    * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
@@ -363,7 +364,7 @@ public final class Transforms {
    *     to make Optional work with BEAM)
    */
   @Nullable
-  public static Object convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
+  public static SqlEntity convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
     return dsEntity
         .getEntity()
         .filter(Transforms::isMigratable)

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -57,7 +57,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=invoicing}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -75,6 +75,8 @@ public class RdeIO {
   abstract static class Write
       extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
 
+    private static final long serialVersionUID = 3334807737227087760L;
+
     abstract GcsUtils gcsUtils();
 
     abstract CloudTasksUtils cloudTasksUtils();
@@ -113,8 +115,9 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()))
-          .apply("Enqueue upload action", ParDo.of(new UploadEnqueuer(cloudTasksUtils())));
+          .apply(
+              "Update cursor and enqueue next action",
+              ParDo.of(new CursorUpdater(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -123,6 +126,7 @@ public class RdeIO {
       extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5496375923068400382L;
 
     private final GcsUtils gcsUtils;
     private final String rdeBucket;
@@ -169,7 +173,7 @@ public class RdeIO {
         checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
         prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
       } else {
-        prefix = prefix + "/" + basename;
+        prefix = prefix + '/' + basename;
       }
       BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
       // This file will contain the byte length (ASCII) of the raw unencrypted XML.
@@ -250,12 +254,20 @@ public class RdeIO {
     }
   }
 
-  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, PendingDeposit> {
+  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5822176227753327224L;
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private CursorUpdater(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
 
     @ProcessElement
     public void processElement(
-        @Element KV<PendingDeposit, Integer> input, OutputReceiver<PendingDeposit> outputReceiver) {
+        @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -282,46 +294,32 @@ public class RdeIO {
                 logger.atInfo().log(
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                if (key.mode() == RdeMode.FULL) {
+                  cloudTasksUtils.enqueue(
+                      RDE_UPLOAD_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          RdeUploadAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                } else {
+                  cloudTasksUtils.enqueue(
+                      BRDA_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          BrdaCopyAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_WATERMARK,
+                              key.watermark().toString(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                }
               });
-      outputReceiver.output(input.getKey());
-    }
-  }
-
-  private static class UploadEnqueuer extends DoFn<PendingDeposit, Void> {
-
-    private final CloudTasksUtils cloudTasksUtils;
-
-    private UploadEnqueuer(CloudTasksUtils cloudTasksUtils) {
-      this.cloudTasksUtils = cloudTasksUtils;
-    }
-
-    @ProcessElement
-    public void processElement(@Element PendingDeposit input, PipelineOptions options) {
-      if (input.mode() == RdeMode.FULL) {
-        cloudTasksUtils.enqueue(
-            RDE_UPLOAD_QUEUE,
-            CloudTasksUtils.createPostTask(
-                RdeUploadAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      } else {
-        cloudTasksUtils.enqueue(
-            BRDA_QUEUE,
-            CloudTasksUtils.createPostTask(
-                BrdaCopyAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_WATERMARK,
-                    input.watermark().toString(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      }
     }
   }
 }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -14,36 +14,50 @@
 
 package google.registry.beam.rde;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.beam.rde.RdePipeline.TupleTags.DOMAIN_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.EXTERNAL_HOST_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.HOST_TO_PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_CONTACTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
+import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
+import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
 import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
-import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.UtilsModule;
@@ -54,74 +68,158 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.persistence.Entity;
+import javax.persistence.IdClass;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.DateTime;
 
 /**
  * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=rde}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
+ * <p>This pipeline only works for pending deposits with the same watermark, the {@link
+ * google.registry.rde.RdeStagingAction} will batch such pending deposits together and launch
+ * multiple pipelines if multiple watermarks exist.
+ *
+ * <p>The pipeline is broadly divided into two parts -- creating the {@link DepositFragment}s, and
+ * processing them.
+ *
+ * <h1>Creating {@link DepositFragment}</h1>
+ *
+ * <h2>{@link Registrar}</h2>
+ *
+ * Non-test registrar entities are loaded from Cloud SQL and marshalled into deposit fragments. They
+ * are <b>NOT</b> rewound to the watermark.
+ *
+ * <h2>{@link EppResource}</h2>
+ *
+ * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
+ * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
+ * revision ID) from the SQL query.
+ *
+ * <h3>{@link DomainBase}</h3>
+ *
+ * After the most recent (live) domain resources are loaded from the corresponding history objects,
+ * we marshall them to deposit fragments and emit the (pending deposit: deposit fragment) pairs for
+ * further processing. We also find all the contacts and hosts referenced by a given domain and emit
+ * pairs of (contact/host repo ID: pending deposit) for all RDE pending deposits for further
+ * processing.
+ *
+ * <h3>{@link ContactResource}</h3>
+ *
+ * We first join most recent contact histories, represented by (contact repo ID: contact history
+ * revision ID) pairs, with referenced contacts, represented by (contact repo ID: pending deposit)
+ * pairs, on the contact repo ID, to remove unreferenced contact histories. Contact resources are
+ * then loaded from the remaining referenced contact histories, and marshalled into (pending
+ * deposit: deposit fragment) pairs.
+ *
+ * <h3>{@link HostResource}</h3>
+ *
+ * Similar to {@link ContactResource}, we join the most recent host history with referenced hosts to
+ * find most recent referenced hosts. For external hosts we do the same treatment as we did on
+ * contacts and obtain the (pending deposit: deposit fragment) pairs. For subordinate hosts, we need
+ * to find the superordinate domain in order to properly handle pending transfer in the deposit as
+ * well. So we first find the superordinate domain repo ID from the host and join the (superordinate
+ * domain repo ID: (subordinate host repo ID: (pending deposit: revision ID))) pair with the (domain
+ * repo ID: revision ID) pair obtained from the domain history query in order to map the host at
+ * watermark to the domain at watermark. We then proceed to create the (pending deposit: deposit
+ * fragment) pair for subordinate hosts using the added domain information.
+ *
+ * <h1>Processing {@link DepositFragment}</h1>
+ *
+ * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
+ * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
+ * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
+ * pending deposit is then rolled forward, and the next action is enqueued. The latter two
+ * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
+ *
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
 @Singleton
 public class RdePipeline implements Serializable {
 
+  private static final long serialVersionUID = -4866795928854754666L;
   private final transient RdePipelineOptions options;
   private final ValidationMode mode;
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final ImmutableSet<PendingDeposit> pendingDeposits;
+  private final DateTime watermark;
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
   private final CloudTasksUtils cloudTasksUtils;
+  private final RdeMarshaller marshaller;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
   private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
       Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
 
-  private static final String EPP_RESOURCE_QUERY =
-      "SELECT id FROM %entity% "
-          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
-
-  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
-    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
-        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
-  }
+  // The field name of the EPP resource embedded in its corresponding history entry.
+  private static final ImmutableMap<Class<? extends HistoryEntry>, String> EPP_RESOURCE_FIELD_NAME =
+      ImmutableMap.of(
+          DomainHistory.class,
+          "domainContent",
+          ContactHistory.class,
+          "contactBase",
+          HostHistory.class,
+          "hostBase");
 
   @Inject
   RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
-    this.pendings = decodePendings(options.getPendings());
+    this.pendingDeposits = decodePendingDeposits(options.getPendings());
+    ImmutableSet<DateTime> potentialWatermarks =
+        pendingDeposits.stream()
+            .map(PendingDeposit::watermark)
+            .distinct()
+            .collect(toImmutableSet());
+    checkArgument(
+        potentialWatermarks.size() == 1,
+        String.format(
+            "RDE pipeline should only work on pending deposits "
+                + "with the same watermark, but %d were given: %s",
+            potentialWatermarks.size(), potentialWatermarks));
+    this.watermark = potentialWatermarks.asList().get(0);
     this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
     this.cloudTasksUtils = cloudTasksUtils;
+    this.marshaller = new RdeMarshaller(mode);
   }
 
   PipelineResult run() {
@@ -133,13 +231,46 @@ public class RdePipeline implements Serializable {
   }
 
   PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
-    return PCollectionList.of(processRegistrars(pipeline))
-        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
-        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
-        .and(processNonRegistrarEntities(pipeline, HostResource.class))
-        .apply(Flatten.pCollections())
+    PCollection<KV<PendingDeposit, DepositFragment>> registrarFragments =
+        processRegistrars(pipeline);
+
+    PCollection<KV<String, Long>> domainHistories =
+        getMostRecentHistoryEntries(pipeline, DomainHistory.class);
+
+    PCollection<KV<String, Long>> contactHistories =
+        getMostRecentHistoryEntries(pipeline, ContactHistory.class);
+
+    PCollection<KV<String, Long>> hostHistories =
+        getMostRecentHistoryEntries(pipeline, HostHistory.class);
+
+    PCollectionTuple processedDomainHistories = processDomainHistories(domainHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> domainFragments =
+        processedDomainHistories.get(DOMAIN_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> contactFragments =
+        processContactHistories(
+            processedDomainHistories.get(REFERENCED_CONTACTS), contactHistories);
+
+    PCollectionTuple processedHosts =
+        processHostHistories(processedDomainHistories.get(REFERENCED_HOSTS), hostHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> externalHostFragments =
+        processedHosts.get(EXTERNAL_HOST_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> subordinateHostFragments =
+        processSubordinateHosts(processedHosts.get(SUPERORDINATE_DOMAINS), domainHistories);
+
+    return PCollectionList.of(registrarFragments)
+        .and(domainFragments)
+        .and(contactFragments)
+        .and(externalHostFragments)
+        .and(subordinateHostFragments)
+        .apply(
+            "Combine PendingDeposit:DepositFragment pairs from all entities",
+            Flatten.pCollections())
         .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply("Group by PendingDeposit", GroupByKey.create());
+        .apply("Group DepositFragment by PendingDeposit", GroupByKey.create());
   }
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
@@ -154,127 +285,398 @@ public class RdePipeline implements Serializable {
             .build());
   }
 
-  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+  private PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    // Note that the namespace in the metric is not being used by Stackdriver, it just has to be
+    // non-empty.
+    // See:
+    // https://stackoverflow.com/questions/48530496/google-dataflow-custom-metrics-not-showing-on-stackdriver
+    Counter includedRegistrarCounter = Metrics.counter("RDE", "IncludedRegistrar");
+    Counter registrarFragmentCounter = Metrics.counter("RDE", "RegistrarFragment");
     return pipeline
         .apply(
-            "Read all production Registrar entities",
+            "Read all production Registrars",
             RegistryJpaIO.read(
                 "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
                 ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
                 String.class,
-                // TODO: consider adding coders for entities and pass them directly instead of using
-                // VKeys.
                 id -> VKey.createSql(Registrar.class, id)))
         .apply(
-            "Marshal Registrar into DepositFragment",
+            "Marshall Registrar into DepositFragment",
             FlatMapElements.into(
-                    TypeDescriptors.kvs(
+                    kvs(
                         TypeDescriptor.of(PendingDeposit.class),
                         TypeDescriptor.of(DepositFragment.class)))
                 .via(
                     (VKey<Registrar> key) -> {
+                      includedRegistrarCounter.inc();
                       Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
-                      DepositFragment fragment =
-                          new RdeMarshaller(mode).marshalRegistrar(registrar);
-                      return pendings.values().stream()
-                          .map(pending -> KV.of(pending, fragment))
-                          .collect(toImmutableSet());
+                      DepositFragment fragment = marshaller.marshalRegistrar(registrar);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          pendingDeposits.stream()
+                              .map(pending -> KV.of(pending, fragment))
+                              .collect(toImmutableSet());
+                      registrarFragmentCounter.inc(fragments.size());
+                      return fragments;
                     }));
   }
 
-  <T extends EppResource>
-      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
-          Pipeline pipeline, Class<T> clazz) {
-    return createInputs(pipeline, clazz)
-        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(
-            KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)));
+  /**
+   * Load the most recent history entry before the watermark for a given history entry type.
+   *
+   * <p>Note that deleted and non-production resources are not included.
+   *
+   * @return A KV pair of (repoId, revisionId), used to reconstruct the composite key for the
+   *     history entry.
+   */
+  private <T extends HistoryEntry> PCollection<KV<String, Long>> getMostRecentHistoryEntries(
+      Pipeline pipeline, Class<T> historyClass) {
+    String repoIdFieldName = HistoryEntryDao.REPO_ID_FIELD_NAMES.get(historyClass);
+    String resourceFieldName = EPP_RESOURCE_FIELD_NAME.get(historyClass);
+    return pipeline
+        .apply(
+            String.format("Load most recent %s", historyClass.getSimpleName()),
+            RegistryJpaIO.read(
+                ("SELECT %repoIdField%, id FROM %entity% WHERE (%repoIdField%, modificationTime)"
+                     + " IN (SELECT %repoIdField%, MAX(modificationTime) FROM %entity% WHERE"
+                     + " modificationTime <= :watermark GROUP BY %repoIdField%) AND"
+                     + " %resourceField%.deletionTime > :watermark AND"
+                     + " COALESCE(%resourceField%.creationClientId, '') NOT LIKE 'prober-%' AND"
+                     + " COALESCE(%resourceField%.currentSponsorClientId, '') NOT LIKE 'prober-%'"
+                     + " AND COALESCE(%resourceField%.lastEppUpdateClientId, '') NOT LIKE"
+                     + " 'prober-%' "
+                        + (historyClass == DomainHistory.class
+                            ? "AND %resourceField%.tld IN "
+                                + "(SELECT id FROM Tld WHERE tldType = 'REAL')"
+                            : ""))
+                    .replace("%entity%", historyClass.getSimpleName())
+                    .replace("%repoIdField%", repoIdFieldName)
+                    .replace("%resourceField%", resourceFieldName),
+                ImmutableMap.of("watermark", watermark),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1])))
+        .setCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));
   }
 
-  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
-    return pipeline.apply(
-        "Read all production " + clazz.getSimpleName() + " entities",
-        RegistryJpaIO.read(
-            createEppResourceQuery(clazz),
-            clazz.equals(DomainBase.class)
-                ? ImmutableMap.of("tlds", pendings.keySet())
-                : ImmutableMap.of(),
-            String.class,
-            // TODO: consider adding coders for entities and pass them directly instead of using
-            // VKeys.
-            x -> VKey.createSql(clazz, x)));
+  private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
+      Class<T> historyEntryClazz, String repoId, long revisionId) {
+    try {
+      Class<?> idClazz = historyEntryClazz.getAnnotation(IdClass.class).value();
+      Serializable idObject =
+          (Serializable)
+              idClazz.getConstructor(String.class, long.class).newInstance(repoId, revisionId);
+      return jpaTm()
+          .transact(() -> jpaTm().loadByKey(VKey.createSql(historyEntryClazz, idObject)))
+          .getResourceAtPointInTime()
+          .map(resource -> resource.cloneProjectedAtTime(watermark))
+          .get();
+    } catch (NoSuchMethodException
+        | InvocationTargetException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot load resource from %s with repoId %s and revisionId %s",
+              historyEntryClazz.getSimpleName(), repoId, revisionId),
+          e);
+    }
   }
 
-  <T extends EppResource>
-      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
-    return FlatMapElements.into(
-            TypeDescriptors.kvs(
-                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
-        .via(
-            (VKey<T> key) -> {
-              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
-              // The set of all TLDs to which this resource should be emitted.
-              ImmutableSet<String> tlds =
-                  clazz.equals(DomainBase.class)
-                      ? ImmutableSet.of(((DomainBase) resource).getTld())
-                      : pendings.keySet();
-              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
-              ImmutableSet<DateTime> dates =
-                  tlds.stream()
-                      .map(pendings::get)
-                      .flatMap(ImmutableSet::stream)
-                      .map(PendingDeposit::watermark)
-                      .collect(toImmutableSet());
-              // Launch asynchronous fetches of point-in-time representations of resource.
-              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
-                  ImmutableMap.copyOf(
-                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
-              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
-              // the result.
-              RdeFragmenter fragmenter =
-                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
-              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
-              for (String tld : tlds) {
-                for (PendingDeposit pending : pendings.get(tld)) {
-                  // Hosts and contacts don't get included in BRDA deposits.
-                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
-                    continue;
+  /**
+   * Remove unreferenced resources by joining the (repoId, pendingDeposit) pair with the (repoId,
+   * revisionId) on the repoId.
+   *
+   * <p>The (repoId, pendingDeposit) pairs denote resources (contact, host) that are referenced from
+   * a domain, that are to be included in the corresponding pending deposit.
+   *
+   * <p>The (repoId, revisionId) paris come from the most recent history entry query, which can be
+   * used to load the embedded resources themselves.
+   *
+   * @return a pair of (repoId, ([pendingDeposit], [revisionId])) where neither the pendingDeposit
+   *     nor the revisionId list is empty.
+   */
+  private static PCollection<KV<String, CoGbkResult>> removeUnreferencedResource(
+      PCollection<KV<String, PendingDeposit>> referencedResources,
+      PCollection<KV<String, Long>> historyEntries,
+      Class<? extends EppResource> resourceClazz) {
+    String resourceName = resourceClazz.getSimpleName();
+    Class<? extends HistoryEntry> historyEntryClazz =
+        RESOURCE_TYPES_TO_HISTORY_TYPES.get(resourceClazz);
+    String historyEntryName = historyEntryClazz.getSimpleName();
+    Counter referencedResourceCounter = Metrics.counter("RDE", "Referenced" + resourceName);
+    return KeyedPCollectionTuple.of(PENDING_DEPOSIT, referencedResources)
+        .and(REVISION_ID, historyEntries)
+        .apply(
+            String.format(
+                "Join PendingDeposit with %s revision ID on %s", historyEntryName, resourceName),
+            CoGroupByKey.create())
+        .apply(
+            String.format("Remove unreferenced %s", resourceName),
+            Filter.by(
+                (KV<String, CoGbkResult> kv) -> {
+                  boolean toInclude =
+                      // If a resource does not have corresponding pending deposit, it is not
+                      // referenced and should not be included.
+                      kv.getValue().getAll(PENDING_DEPOSIT).iterator().hasNext()
+                          // If a resource does not have revision id (this should not happen, as
+                          // every referenced resource must be valid at watermark time, therefore
+                          // be embedded in a history entry valid at watermark time, otherwise
+                          // the domain cannot reference it), there is no way for us to find the
+                          // history entry and load the embedded resource. So we ignore the resource
+                          // to keep the downstream process simple.
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedResourceCounter.inc();
                   }
-                  Optional<DepositFragment> fragment =
-                      fragmenter.marshal(pending.watermark(), pending.mode());
-                  fragment.ifPresent(
-                      depositFragment -> results.add(KV.of(pending, depositFragment)));
-                }
-              }
-              return results;
-            });
+                  return toInclude;
+                }));
+  }
+
+  private PCollectionTuple processDomainHistories(PCollection<KV<String, Long>> domainHistories) {
+    Counter activeDomainCounter = Metrics.counter("RDE", "ActiveDomainBase");
+    Counter domainFragmentCounter = Metrics.counter("RDE", "DomainFragment");
+    Counter referencedContactCounter = Metrics.counter("RDE", "ReferencedContactResource");
+    Counter referencedHostCounter = Metrics.counter("RDE", "ReferencedHostResource");
+    return domainHistories.apply(
+        "Map DomainHistory to DepositFragment "
+            + "and emit referenced ContactResource and HostResource",
+        ParDo.of(
+                new DoFn<KV<String, Long>, KV<PendingDeposit, DepositFragment>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Long> kv, MultiOutputReceiver receiver) {
+                    activeDomainCounter.inc();
+                    DomainBase domain =
+                        (DomainBase)
+                            loadResourceByHistoryEntryId(
+                                DomainHistory.class, kv.getKey(), kv.getValue());
+                    pendingDeposits.stream()
+                        .filter(pendingDeposit -> pendingDeposit.tld().equals(domain.getTld()))
+                        .forEach(
+                            pendingDeposit -> {
+                              // Domains are always deposited in both modes.
+                              domainFragmentCounter.inc();
+                              receiver
+                                  .get(DOMAIN_FRAGMENTS)
+                                  .output(
+                                      KV.of(
+                                          pendingDeposit,
+                                          marshaller.marshalDomain(domain, pendingDeposit.mode())));
+                              // Contacts and hosts are only deposited in RDE, not BRDA.
+                              if (pendingDeposit.mode() == RdeMode.FULL) {
+                                HashSet<Serializable> contacts = new HashSet<>();
+                                contacts.add(domain.getAdminContact().getSqlKey());
+                                contacts.add(domain.getTechContact().getSqlKey());
+                                contacts.add(domain.getRegistrant().getSqlKey());
+                                // Billing contact is not mandatory.
+                                if (domain.getBillingContact() != null) {
+                                  contacts.add(domain.getBillingContact().getSqlKey());
+                                }
+                                referencedContactCounter.inc(contacts.size());
+                                contacts.forEach(
+                                    contactRepoId ->
+                                        receiver
+                                            .get(REFERENCED_CONTACTS)
+                                            .output(KV.of((String) contactRepoId, pendingDeposit)));
+                                if (domain.getNsHosts() != null) {
+                                  referencedHostCounter.inc(domain.getNsHosts().size());
+                                  domain
+                                      .getNsHosts()
+                                      .forEach(
+                                          hostKey ->
+                                              receiver
+                                                  .get(REFERENCED_HOSTS)
+                                                  .output(
+                                                      KV.of(
+                                                          (String) hostKey.getSqlKey(),
+                                                          pendingDeposit)));
+                                }
+                              }
+                            });
+                  }
+                })
+            .withOutputTags(
+                DOMAIN_FRAGMENTS, TupleTagList.of(REFERENCED_CONTACTS).and(REFERENCED_HOSTS)));
+  }
+
+  private PCollection<KV<PendingDeposit, DepositFragment>> processContactHistories(
+      PCollection<KV<String, PendingDeposit>> referencedContacts,
+      PCollection<KV<String, Long>> contactHistories) {
+    Counter contactFragmentCounter = Metrics.counter("RDE", "ContactFragment");
+    return removeUnreferencedResource(referencedContacts, contactHistories, ContactResource.class)
+        .apply(
+            "Map ContactResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      ContactResource contact =
+                          (ContactResource)
+                              loadResourceByHistoryEntryId(
+                                  ContactHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      DepositFragment fragment = marshaller.marshalContact(contact);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same contact could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .map(pendingDeposit -> KV.of(pendingDeposit, fragment))
+                              .collect(toImmutableSet());
+                      contactFragmentCounter.inc(fragments.size());
+                      return fragments;
+                    }));
+  }
+
+  private PCollectionTuple processHostHistories(
+      PCollection<KV<String, PendingDeposit>> referencedHosts,
+      PCollection<KV<String, Long>> hostHistories) {
+    Counter subordinateHostCounter = Metrics.counter("RDE", "SubordinateHostResource");
+    Counter externalHostCounter = Metrics.counter("RDE", "ExternalHostResource");
+    Counter externalHostFragmentCounter = Metrics.counter("RDE", "ExternalHostFragment");
+    return removeUnreferencedResource(referencedHosts, hostHistories, HostResource.class)
+        .apply(
+            "Map external DomainResource to DepositFragment and process subordinate domains",
+            ParDo.of(
+                    new DoFn<KV<String, CoGbkResult>, KV<PendingDeposit, DepositFragment>>() {
+                      @ProcessElement
+                      public void processElement(
+                          @Element KV<String, CoGbkResult> kv, MultiOutputReceiver receiver) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    kv.getKey(),
+                                    kv.getValue().getOnly(REVISION_ID));
+                        // When a host is subordinate, we need to find it's superordinate domain and
+                        // include it in the deposit as well.
+                        if (host.isSubordinate()) {
+                          subordinateHostCounter.inc();
+                          receiver
+                              .get(SUPERORDINATE_DOMAINS)
+                              .output(
+                                  // The output are pairs of
+                                  // (superordinateDomainRepoId,
+                                  //   (subordinateHostRepoId, (pendingDeposit, revisionId))).
+                                  KV.of((String) host.getSuperordinateDomain().getSqlKey(), kv));
+                        } else {
+                          externalHostCounter.inc();
+                          DepositFragment fragment = marshaller.marshalExternalHost(host);
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same host could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .forEach(
+                                  pendingDeposit -> {
+                                    externalHostFragmentCounter.inc();
+                                    receiver
+                                        .get(EXTERNAL_HOST_FRAGMENTS)
+                                        .output(KV.of(pendingDeposit, fragment));
+                                  });
+                        }
+                      }
+                    })
+                .withOutputTags(EXTERNAL_HOST_FRAGMENTS, TupleTagList.of(SUPERORDINATE_DOMAINS)));
+  }
+
+  /**
+   * Process subordinate hosts by making a deposit fragment with pending transfer information
+   * obtained from its superordinate domain.
+   *
+   * @param superordinateDomains Pairs of (superordinateDomainRepoId, (subordinateHostRepoId,
+   *     (pendingDeposit, revisionId))). This collection maps the subordinate host and the pending
+   *     deposit to include it to its superordinate domain.
+   * @param domainHistories Pairs of (domainRepoId, revisionId). This collection helps us find the
+   *     historical superordinate domain from its history entry and is obtained from calling {@link
+   *     #getMostRecentHistoryEntries} for domains.
+   */
+  private PCollection<KV<PendingDeposit, DepositFragment>> processSubordinateHosts(
+      PCollection<KV<String, KV<String, CoGbkResult>>> superordinateDomains,
+      PCollection<KV<String, Long>> domainHistories) {
+    Counter subordinateHostFragmentCounter = Metrics.counter("RDE", "SubordinateHostFragment");
+    Counter referencedSubordinateHostCounter = Metrics.counter("RDE", "ReferencedSubordinateHost");
+    return KeyedPCollectionTuple.of(HOST_TO_PENDING_DEPOSIT, superordinateDomains)
+        .and(REVISION_ID, domainHistories)
+        .apply(
+            "Join HostResource:PendingDeposits with DomainHistory on DomainResource",
+            CoGroupByKey.create())
+        .apply(
+            " Remove unreferenced DomainResource",
+            Filter.by(
+                kv -> {
+                  boolean toInclude =
+                      kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT).iterator().hasNext()
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedSubordinateHostCounter.inc();
+                  }
+                  return toInclude;
+                }))
+        .apply(
+            "Map subordinate HostResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      DomainBase superordinateDomain =
+                          (DomainBase)
+                              loadResourceByHistoryEntryId(
+                                  DomainHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      ImmutableSet.Builder<KV<PendingDeposit, DepositFragment>> results =
+                          new ImmutableSet.Builder<>();
+                      for (KV<String, CoGbkResult> hostToPendingDeposits :
+                          kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT)) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    hostToPendingDeposits.getKey(),
+                                    hostToPendingDeposits.getValue().getOnly(REVISION_ID));
+                        DepositFragment fragment =
+                            marshaller.marshalSubordinateHost(host, superordinateDomain);
+                        Streams.stream(hostToPendingDeposits.getValue().getAll(PENDING_DEPOSIT))
+                            .distinct()
+                            .forEach(
+                                pendingDeposit -> {
+                                  subordinateHostFragmentCounter.inc();
+                                  results.add(KV.of(pendingDeposit, fragment));
+                                });
+                      }
+                      return results.build();
+                    }));
   }
 
   /**
    * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
-   * the original TLD to pending deposit map.
+   * the original pending deposit set.
    */
   @SuppressWarnings("unchecked")
-  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+  static ImmutableSet<PendingDeposit> decodePendingDeposits(String encodedPendingDeposits) {
     try (ObjectInputStream ois =
         new ObjectInputStream(
             new ByteArrayInputStream(
-                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
-      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+                BaseEncoding.base64Url().omitPadding().decode(encodedPendingDeposits)))) {
+      return (ImmutableSet<PendingDeposit>) ois.readObject();
     } catch (IOException | ClassNotFoundException e) {
       throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
     }
   }
 
   /**
-   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
-   * worker by the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
+   * the pipeline launcher as a pipeline option.
    */
-  public static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
-      oos.writeObject(pendings);
+      oos.writeObject(pendingDeposits);
       oos.flush();
       return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
     }
@@ -284,14 +686,40 @@ public class RdePipeline implements Serializable {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
     RdePipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
-    // RegistryPipelineWorkerInitializer only initializes before pipeline executions, after the
-    // main() function constructed the graph. We need the registry environment set up so that we
-    // can create a CloudTasksUtils which uses the environment-dependent config file.
-    options.getRegistryEnvironment().setup();
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
+  /**
+   * A utility class that contains {@link TupleTag}s when {@link PCollectionTuple}s and {@link
+   * CoGbkResult}s are used.
+   */
+  protected abstract static class TupleTags {
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> DOMAIN_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_CONTACTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_HOSTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, KV<String, CoGbkResult>>> SUPERORDINATE_DOMAINS =
+        new TupleTag<KV<String, KV<String, CoGbkResult>>>() {};
+
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> EXTERNAL_HOST_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<PendingDeposit> PENDING_DEPOSIT =
+        new TupleTag<PendingDeposit>() {};
+
+    protected static final TupleTag<KV<String, CoGbkResult>> HOST_TO_PENDING_DEPOSIT =
+        new TupleTag<KV<String, CoGbkResult>>() {};
+
+    protected static final TupleTag<Long> REVISION_ID = new TupleTag<Long>() {};
+  }
+
   @Singleton
   @Component(
       modules = {

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -61,7 +61,8 @@ import org.json.JSONObject;
 /**
  * Definition of a Dataflow Flex template, which generates a given month's spec11 report.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=spec11}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -557,11 +557,23 @@ public final class RegistryConfig {
       return config.registryPolicy.requireSslCertificates;
     }
 
+    /**
+     * Returns the GCE machine type that a CPU-demanding pipeline should use.
+     *
+     * @see google.registry.beam.rde.RdePipeline
+     */
+    @Provides
+    @Config("highPerformanceMachineType")
+    public static String provideHighPerformanceMachineType(RegistryConfigSettings config) {
+      return config.beam.highPerformanceMachineType;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
      * @see google.registry.beam.invoicing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
+     * @see google.registry.beam.invoicing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -133,6 +133,7 @@ public class RegistryConfigSettings {
   /** Configuration for Apache Beam (Cloud Dataflow). */
   public static class Beam {
     public String defaultJobRegion;
+    public String highPerformanceMachineType;
     public String stagingBucketUrl;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -419,6 +419,14 @@ misc:
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
   defaultJobRegion: us-east1
+  # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
+  # to check the VM CPU quota for the job region. In a massively parallel
+  # pipeline this quota can be easily reached and needs to be raised, otherwise
+  # the job will run very slowly. Also note that there is a separate quota for
+  # external IPv4 address in a region, which means that machine type with higher
+  # core count per machine may be preferable in order to preserve IP addresses.
+  # See: https://cloud.google.com/compute/quotas#cpu_quota
+  highPerformanceMachineType: n2-standard-4
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -14,18 +14,15 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
-import java.time.Duration;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import javax.inject.Inject;
 
 /** Action for fanning out cron tasks for each commit log bucket. */
@@ -38,25 +35,27 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  private static final Random random = new Random();
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
-  @Inject TaskQueueUtils taskQueueUtils;
   @Inject @Parameter("endpoint") String endpoint;
   @Inject @Parameter("queue") String queue;
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
+
+
   @Override
   public void run() {
-    Queue taskQueue = getQueue(queue);
     for (int bucketId : CommitLogBucket.getBucketIds()) {
-      long delay =
-          jitterSeconds.map(i -> random.nextInt((int) Duration.ofSeconds(i).toMillis())).orElse(0);
-      TaskOptions taskOptions =
-          TaskOptions.Builder.withUrl(endpoint)
-              .param(BUCKET_PARAM, Integer.toString(bucketId))
-              .countdownMillis(delay);
-      taskQueueUtils.enqueue(taskQueue, taskOptions);
+      cloudTasksUtils.enqueue(
+          queue,
+          CloudTasksUtils.createPostTask(
+              endpoint,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
+              clock,
+              jitterSeconds));
     }
   }
 }

core/src/main/java/google/registry/cron/TldFanoutAction.java

@@ -30,7 +30,6 @@ import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
 import static google.registry.model.tld.Registries.getTldsOfType;
 import static google.registry.model.tld.Registry.TldType.REAL;
 import static google.registry.model.tld.Registry.TldType.TEST;
-import static java.util.concurrent.TimeUnit.SECONDS;
 
 import com.google.cloud.tasks.v2.Task;
 import com.google.common.collect.ArrayListMultimap;
@@ -39,7 +38,6 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
-import com.google.protobuf.Timestamp;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
@@ -49,7 +47,6 @@ import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
-import java.time.Instant;
 import java.util.Optional;
 import java.util.Random;
 import java.util.stream.Stream;
@@ -165,22 +162,7 @@ public final class TldFanoutAction implements Runnable {
       params = ArrayListMultimap.create(params);
       params.put(RequestParameters.PARAM_TLD, tld);
     }
-    Instant scheduleTime =
-        Instant.ofEpochMilli(
-            clock
-                .nowUtc()
-                .plusMillis(
-                    jitterSeconds
-                        .map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
-                        .orElse(0))
-                .getMillis());
-    return Task.newBuilder(
-            CloudTasksUtils.createPostTask(endpoint, Service.BACKEND.toString(), params))
-        .setScheduleTime(
-            Timestamp.newBuilder()
-                .setSeconds(scheduleTime.getEpochSecond())
-                .setNanos(scheduleTime.getNano())
-                .build())
-        .build();
+    return CloudTasksUtils.createPostTask(
+        endpoint, Service.BACKEND.toString(), params, clock, jitterSeconds);
   }
 }

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -157,6 +157,12 @@
     <url-pattern>/_dr/cron/readDnsQueue</url-pattern>
   </servlet-mapping>
 
+  <!-- Replicates SQL transactions to Datastore during the Registry 3.0 migration. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/cron/replicateToDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Publishes DNS updates. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/flows/EppController.java

@@ -143,9 +143,6 @@ public final class EppController {
   /** Creates a response indicating an EPP failure. */
   @VisibleForTesting
   static EppOutput getErrorResponse(Result result, Trid trid) {
-    return EppOutput.create(new EppResponse.Builder()
-        .setResult(result)
-        .setTrid(trid)
-        .build());
+    return EppOutput.create(new EppResponse.Builder().setResult(result).setTrid(trid).build());
   }
 }

core/src/main/java/google/registry/flows/EppException.java

@@ -25,10 +25,12 @@ import google.registry.model.eppinput.EppInput.InnerCommand;
 import google.registry.model.eppinput.EppInput.ResourceCommandWrapper;
 import google.registry.model.eppoutput.Result;
 import google.registry.model.eppoutput.Result.Code;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Inherited;
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
+import javax.annotation.Nullable;
 
 /** Exception used to propagate all failures containing one or more EPP responses. */
 public abstract class EppException extends Exception {
@@ -37,7 +39,12 @@ public abstract class EppException extends Exception {
 
   /** Create an EppException with a custom message. */
   private EppException(String message) {
-    super(message);
+    this(message, null);
+  }
+
+  /** Create an EppException with a custom message and cause. */
+  private EppException(String message, @Nullable Throwable cause) {
+    super(message, cause);
     Code code = getClass().getAnnotation(EppResultCode.class).value();
     Preconditions.checkState(!code.isSuccess());
     this.result = Result.create(code, message);
@@ -255,4 +262,12 @@ public abstract class EppException extends Exception {
       super("Specified protocol version is not implemented");
     }
   }
+
+  /** Registry is currently undergoing maintenance and is in read-only mode. */
+  @EppResultCode(Code.COMMAND_FAILED)
+  public static class ReadOnlyModeEppException extends EppException {
+    ReadOnlyModeEppException(ReadOnlyModeException cause) {
+      super("Registry is currently undergoing maintenance and is in read-only mode", cause);
+    }
+  }
 }

core/src/main/java/google/registry/flows/FlowRunner.java

@@ -19,6 +19,7 @@ import static google.registry.xml.XmlTransformer.prettyPrint;
 
 import com.google.common.base.Strings;
 import com.google.common.flogger.FluentLogger;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.FlowModule.DryRun;
 import google.registry.flows.FlowModule.InputXml;
 import google.registry.flows.FlowModule.RegistrarId;
@@ -28,6 +29,7 @@ import google.registry.flows.session.LoginFlow;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppOutput;
 import google.registry.monitoring.whitebox.EppMetric;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import javax.inject.Inject;
 import javax.inject.Provider;
 
@@ -97,6 +99,8 @@ public class FlowRunner {
       return e.output;
     } catch (EppRuntimeException e) {
       throw e.getCause();
+    } catch (ReadOnlyModeException e) {
+      throw new ReadOnlyModeEppException(e);
     }
   }
 

core/src/main/java/google/registry/flows/contact/ContactCheckFlow.java

@@ -42,6 +42,7 @@ import javax.inject.Inject;
  * <p>This flows can check the existence of multiple contacts simultaneously.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  */
 @ReportingSpec(ActivityReportField.CONTACT_CHECK)
 public final class ContactCheckFlow implements Flow {

core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java

@@ -50,6 +50,8 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that creates a new contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link ContactFlowUtils.BadInternationalizedPostalInfoException}

core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java

@@ -60,6 +60,8 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/contact/ContactInfoFlow.java

@@ -46,6 +46,7 @@ import org.joda.time.DateTime;
  * ever been transferred. Any registrar can see any contact's information, but the authInfo is only
  * visible to the registrar that owns the contact or to a registrar that already supplied it.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  */

core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java

@@ -54,6 +54,8 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * explicitly approve the transfer request, which then becomes effective immediately.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}

core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java

@@ -54,6 +54,8 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the gaining client to
  * withdraw the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactTransferQueryFlow.java

@@ -40,11 +40,12 @@ import javax.inject.Inject;
  *
  * <p>The "gaining" registrar requests a transfer from the "losing" (aka current) registrar. The
  * losing registrar has a "transfer" time period to respond (by default five days) after which the
- * transfer is automatically approved. This flow can be used by the gaining or losing registrars
- * (or anyone with the correct authId) to see the status of a transfer, which may still be pending
- * or may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
+ * transfer is automatically approved. This flow can be used by the gaining or losing registrars (or
+ * anyone with the correct authId) to see the status of a transfer, which may still be pending or
+ * may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
  * period expiring.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NoTransferHistoryToQueryException}

core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java

@@ -53,6 +53,8 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * reject the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java

@@ -63,6 +63,8 @@ import org.joda.time.Duration;
  * by the losing registrar or rejected, and the gaining registrar can also cancel the transfer
  * request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.AlreadyPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java

@@ -55,6 +55,8 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that updates a contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainCheckFlow.java

@@ -85,6 +85,7 @@ import org.joda.time.DateTime;
  * <p>This flow also supports the EPP fee extension and can return pricing information.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link DomainFlowUtils.BadDomainNameCharacterException}
  * @error {@link DomainFlowUtils.BadDomainNamePartsCountException}

core/src/main/java/google/registry/flows/domain/DomainClaimsCheckFlow.java

@@ -57,13 +57,14 @@ import org.joda.time.DateTime;
  * An EPP flow that checks whether domain labels are trademarked.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link DomainFlowUtils.BadCommandForRegistryPhaseException}
  * @error {@link DomainFlowUtils.ClaimsPeriodEndedException}
  * @error {@link DomainFlowUtils.NotAuthorizedForTldException}
  * @error {@link DomainFlowUtils.TldDoesNotExistException}
  * @error {@link DomainClaimsCheckNotAllowedWithAllocationTokens}
  */
-@ReportingSpec(ActivityReportField.DOMAIN_CHECK)  // Claims check is a special domain check.
+@ReportingSpec(ActivityReportField.DOMAIN_CHECK) // Claims check is a special domain check.
 public final class DomainClaimsCheckFlow implements Flow {
 
   @Inject ExtensionManager extensionManager;

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -133,11 +133,13 @@ import org.joda.time.Duration;
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException}
  * @error {@link
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException}
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.ExtensionManager.UndeclaredServiceExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link DomainCreateFlow.AnchorTenantCreatePeriodException}
  * @error {@link DomainCreateFlow.MustHaveSignedMarksInCurrentPhaseException}

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -103,7 +103,9 @@ import org.joda.time.Duration;
 /**
  * An EPP flow that deletes a domain.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException}
@@ -187,7 +189,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
     } else {
       DateTime redemptionTime = now.plus(redemptionGracePeriodLength);
       asyncTaskEnqueuer.enqueueAsyncResave(
-          existingDomain, now, ImmutableSortedSet.of(redemptionTime, deletionTime));
+          existingDomain.createVKey(), now, ImmutableSortedSet.of(redemptionTime, deletionTime));
       builder
           .setDeletionTime(deletionTime)
           .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE))

core/src/main/java/google/registry/flows/domain/DomainInfoFlow.java

@@ -63,6 +63,7 @@ import org.joda.time.DateTime;
  * domain, will get a rich result with all of the domain's fields. All other requests will be
  * answered with a minimal result containing only basic information about the domain.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -94,6 +94,8 @@ import org.joda.time.Duration;
  * longer than 10 years unless it comes in at the exact millisecond that the domain would have
  * expired.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -93,7 +93,9 @@ import org.joda.time.DateTime;
  * restored to a single year expiration starting at the restore time, regardless of what the
  * original expiration time was.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
@@ -110,7 +112,7 @@ import org.joda.time.DateTime;
  * @error {@link DomainRestoreRequestFlow.RestoreCommandIncludesChangesException}
  */
 @ReportingSpec(ActivityReportField.DOMAIN_RGP_RESTORE_REQUEST)
-public final class DomainRestoreRequestFlow implements TransactionalFlow  {
+public final class DomainRestoreRequestFlow implements TransactionalFlow {
 
   @Inject ResourceCommand resourceCommand;
   @Inject ExtensionManager extensionManager;

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -78,6 +78,8 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted and replaced with new ones with the correct approval time.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -65,6 +65,8 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/domain/DomainTransferQueryFlow.java

@@ -44,11 +44,12 @@ import org.joda.time.DateTime;
  *
  * <p>The "gaining" registrar requests a transfer from the "losing" (aka current) registrar. The
  * losing registrar has a "transfer" time period to respond (by default five days) after which the
- * transfer is automatically approved. This flow can be used by the gaining or losing registrars
- * (or anyone with the correct authId) to see the status of a transfer, which may still be pending
- * or may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
+ * transfer is automatically approved. This flow can be used by the gaining or losing registrars (or
+ * anyone with the correct authId) to see the status of a transfer, which may still be pending or
+ * may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
  * period expiring.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NoTransferHistoryToQueryException}

core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java

@@ -67,6 +67,8 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -93,6 +93,8 @@ import org.joda.time.DateTime;
  * rejection or cancellation of the request, they will be deleted (and in the approval case,
  * replaced with new ones with the correct approval time).
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -102,7 +104,8 @@ import org.joda.time.DateTime;
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}
  * @error {@link google.registry.flows.exceptions.TransferPeriodMustBeOneYearException}
  * @error {@link InvalidTransferPeriodValueException}
- * @error {@link google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
+ * @error {@link
+ *     google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
  * @error {@link DomainFlowUtils.BadPeriodUnitException}
  * @error {@link DomainFlowUtils.CurrencyUnitMismatchException}
  * @error {@link DomainFlowUtils.CurrencyValueScaleException}
@@ -235,7 +238,7 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
             .build();
     DomainHistory domainHistory = buildDomainHistory(newDomain, registry, now, period);
 
-    asyncTaskEnqueuer.enqueueAsyncResave(newDomain, now, automaticTransferTime);
+    asyncTaskEnqueuer.enqueueAsyncResave(newDomain.createVKey(), now, automaticTransferTime);
     tm().putAll(
             new ImmutableSet.Builder<>()
                 .add(newDomain, domainHistory, requestPollMessage)

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -16,6 +16,7 @@ package google.registry.flows.domain;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
 import static com.google.common.collect.Sets.symmetricDifference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.flows.FlowUtils.persistEntityChanges;
@@ -43,6 +44,9 @@ import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_UPDATE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
 import com.google.common.net.InternetDomainName;
 import google.registry.dns.DnsQueue;
 import google.registry.flows.EppException;
@@ -76,6 +80,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
+import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.model.tld.Registry;
 import java.util.Optional;
@@ -92,7 +97,9 @@ import org.joda.time.DateTime;
  * superuser. As such, adding or removing these statuses incurs a billing event. There will be only
  * one charge per update, even if several such statuses are updated at once.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
@@ -175,6 +182,9 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     Optional<BillingEvent.OneTime> statusUpdateBillingEvent =
         createBillingEventForStatusUpdates(existingDomain, newDomain, domainHistory, now);
     statusUpdateBillingEvent.ifPresent(entitiesToSave::add);
+    Optional<PollMessage.OneTime> serverStatusUpdatePollMessage =
+        createPollMessageForServerStatusUpdates(existingDomain, newDomain, domainHistory, now);
+    serverStatusUpdatePollMessage.ifPresent(entitiesToSave::add);
     EntityChanges entityChanges =
         flowCustomLogic.beforeSave(
             BeforeSaveParameters.newBuilder()
@@ -306,4 +316,50 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     }
     return Optional.empty();
   }
+
+  /** Enqueues a poll message iff a superuser is adding/removing server statuses. */
+  private Optional<PollMessage.OneTime> createPollMessageForServerStatusUpdates(
+      DomainBase existingDomain, DomainBase newDomain, DomainHistory historyEntry, DateTime now) {
+    if (registrarId.equals(existingDomain.getPersistedCurrentSponsorRegistrarId())) {
+      // Don't send a poll message when a superuser registrar is updating its own domain.
+      return Optional.empty();
+    }
+    ImmutableSortedSet<String> addedServerStatuses =
+        Sets.difference(newDomain.getStatusValues(), existingDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+    ImmutableSortedSet<String> removedServerStatuses =
+        Sets.difference(existingDomain.getStatusValues(), newDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+
+    String msg = "";
+    if (addedServerStatuses.size() > 0 && removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s and removed the status(es)"
+                  + " %s.",
+              addedServerStatuses, removedServerStatuses);
+    } else if (addedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s.", addedServerStatuses);
+    } else if (removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has removed the status(es) %s.", removedServerStatuses);
+    } else {
+      return Optional.empty();
+    }
+
+    return Optional.ofNullable(
+        new PollMessage.OneTime.Builder()
+            .setParent(historyEntry)
+            .setEventTime(now)
+            .setRegistrarId(existingDomain.getCurrentSponsorRegistrarId())
+            .setMsg(msg)
+            .build());
+  }
 }

core/src/main/java/google/registry/flows/host/HostCheckFlow.java

@@ -42,6 +42,7 @@ import javax.inject.Inject;
  * <p>This flows can check the existence of multiple hosts simultaneously.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  */
 @ReportingSpec(ActivityReportField.HOST_CHECK)
 public final class HostCheckFlow implements Flow {

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -65,7 +65,9 @@ import org.joda.time.DateTime;
  * hosts cannot have any. This flow allows creating a host name, and if necessary enqueues tasks to
  * update DNS.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.IpAddressVersionMismatchException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link HostFlowUtils.HostNameTooLongException}

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -58,6 +58,8 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/host/HostInfoFlow.java

@@ -44,6 +44,7 @@ import org.joda.time.DateTime;
  * <p>The returned information included IP addresses, if any, and details of the host's most recent
  * transfer if it has ever been transferred. Any registrar can see the information for any host.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link HostFlowUtils.HostNameNotLowerCaseException}
  * @error {@link HostFlowUtils.HostNameNotNormalizedException}

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -73,11 +73,13 @@ import org.joda.time.DateTime;
  * hosts. Internal hosts must have at least one IP address associated with them, whereas external
  * hosts cannot have any.
  *
- * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When
- * a host is renamed from internal to external all IP addresses must be simultaneously removed, and
+ * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When a
+ * host is renamed from internal to external all IP addresses must be simultaneously removed, and
  * when it is renamed from external to internal at least one must be added. If the host is renamed
  * or IP addresses are added, tasks are enqueued to update DNS accordingly.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/model/CreateAutoTimestamp.java

@@ -14,8 +14,20 @@
 
 package google.registry.model;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.OnLoad;
 import google.registry.model.translators.CreateAutoTimestampTranslatorFactory;
+import google.registry.util.DateTimeUtils;
+import java.time.ZonedDateTime;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.PostLoad;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -23,9 +35,37 @@ import org.joda.time.DateTime;
  *
  * @see CreateAutoTimestampTranslatorFactory
  */
+@Embeddable
 public class CreateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
-  DateTime timestamp;
+  @Transient DateTime timestamp;
+
+  @Column(nullable = false)
+  @Ignore
+  ZonedDateTime creationTime;
+
+  @PrePersist
+  @PreUpdate
+  void setTimestamp() {
+    if (creationTime == null) {
+      timestamp = jpaTm().getTransactionTime();
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @OnLoad
+  void onLoad() {
+    if (timestamp != null) {
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (creationTime != null) {
+      timestamp = DateTimeUtils.toJodaDateTime(creationTime);
+    }
+  }
 
   /** Returns the timestamp. */
   @Nullable
@@ -36,6 +76,7 @@ public class CreateAutoTimestamp extends ImmutableObject implements UnsafeSerial
   public static CreateAutoTimestamp create(@Nullable DateTime timestamp) {
     CreateAutoTimestamp instance = new CreateAutoTimestamp();
     instance.timestamp = timestamp;
+    instance.creationTime = (timestamp == null) ? null : DateTimeUtils.toZonedDateTime(timestamp);
     return instance;
   }
 }

core/src/main/java/google/registry/model/EntityClasses.java

@@ -43,7 +43,6 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.server.Lock;
 import google.registry.model.server.ServerSecret;
 import google.registry.model.tld.Registry;
-import google.registry.model.tmch.TmchCrl;
 
 /** Sets of classes of the Objectify-registered entities in use throughout the model. */
 public final class EntityClasses {
@@ -85,8 +84,7 @@ public final class EntityClasses {
           Registrar.class,
           RegistrarContact.class,
           Registry.class,
-          ServerSecret.class,
-          TmchCrl.class);
+          ServerSecret.class);
 
   private EntityClasses() {}
 }

core/src/main/java/google/registry/model/EppResource.java

@@ -50,6 +50,8 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -112,7 +114,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
    * resource fields.
    */
-  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  @AttributeOverrides({@AttributeOverride(name = "creationTime", column = @Column())})
+  @Index
+  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -224,10 +228,8 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /** Used when replaying from SQL to DS to populate the Datastore indexes. */
   protected void saveIndexesToDatastore() {
-    ofyTm()
-        .putAll(
-            ForeignKeyIndex.create(this, getDeletionTime()),
-            EppResourceIndex.create(Key.create(this)));
+    ofyTm().putIgnoringReadOnlyWithBackup(ForeignKeyIndex.create(this, getDeletionTime()));
+    ofyTm().putIgnoringReadOnlyWithBackup(EppResourceIndex.create(Key.create(this)));
   }
 
   /** EppResources that are loaded via foreign keys should implement this marker interface. */

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -156,6 +156,10 @@ public abstract class ImmutableObject implements Cloneable {
    *   }
    * }
    * </pre>
+   *
+   * <p>This method makes use of {@link #toStringHelper}, which embeds {@link
+   * System#identityHashCode} in the output string. Subclasses that require deterministic string
+   * representations <em>across</em> JVM instances should override this method.
    */
   @Override
   public String toString() {

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -27,7 +27,6 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -54,6 +53,7 @@ import google.registry.persistence.BillingVKey.BillingEventVKey;
 import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
+import google.registry.persistence.converter.JodaMoneyType;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -67,6 +67,8 @@ import javax.persistence.Enumerated;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.PostLoad;
 import javax.persistence.Transient;
+import org.hibernate.annotations.Columns;
+import org.hibernate.annotations.Type;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
@@ -77,14 +79,29 @@ public abstract class BillingEvent extends ImmutableObject
 
   /** The reason for the bill, which maps 1:1 to skus in go/registry-billing-skus. */
   public enum Reason {
-    CREATE,
+    CREATE(true),
     @Deprecated // TODO(b/31676071): remove this legacy value once old data is cleaned up.
-    ERROR,
-    FEE_EARLY_ACCESS,
-    RENEW,
-    RESTORE,
-    SERVER_STATUS,
-    TRANSFER
+    ERROR(false),
+    FEE_EARLY_ACCESS(true),
+    RENEW(true),
+    RESTORE(true),
+    SERVER_STATUS(false),
+    TRANSFER(true);
+
+    private final boolean requiresPeriod;
+
+    Reason(boolean requiresPeriod) {
+      this.requiresPeriod = requiresPeriod;
+    }
+
+    /**
+     * Returns whether billing events with this reason have a period years associated with them.
+     *
+     * <p>Note that this is an "if an only if" condition.
+     */
+    public boolean hasPeriodYears() {
+      return requiresPeriod;
+    }
   }
 
   /** Set of flags that can be applied to billing events. */
@@ -268,7 +285,7 @@ public abstract class BillingEvent extends ImmutableObject
     public T build() {
       T instance = getInstance();
       checkNotNull(instance.reason, "Reason must be set");
-      checkNotNull(instance.clientId, "Client ID must be set");
+      checkNotNull(instance.clientId, "Registrar ID must be set");
       checkNotNull(instance.eventTime, "Event time must be set");
       checkNotNull(instance.targetId, "Target ID must be set");
       checkNotNull(instance.parent, "Parent must be set");
@@ -298,10 +315,8 @@ public abstract class BillingEvent extends ImmutableObject
   public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
-    @AttributeOverrides({
-      @AttributeOverride(name = "money.amount", column = @Column(name = "cost_amount")),
-      @AttributeOverride(name = "money.currency", column = @Column(name = "cost_currency"))
-    })
+    @Type(type = JodaMoneyType.TYPE_NAME)
+    @Columns(columns = {@Column(name = "cost_amount"), @Column(name = "cost_currency")})
     Money cost;
 
     /** When the cost should be billed. */
@@ -462,17 +477,14 @@ public abstract class BillingEvent extends ImmutableObject
         checkNotNull(instance.billingTime);
         checkNotNull(instance.cost);
         checkState(!instance.cost.isNegative(), "Costs should be non-negative.");
-        ImmutableSet<Reason> reasonsWithPeriods =
-            Sets.immutableEnumSet(
-                Reason.CREATE,
-                Reason.FEE_EARLY_ACCESS,
-                Reason.RENEW,
-                Reason.RESTORE,
-                Reason.TRANSFER);
-        checkState(
-            reasonsWithPeriods.contains(instance.reason) == (instance.periodYears != null),
-            "Period years must be set if and only if reason is "
-                + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        // TODO(mcilwain): Enforce this check on all billing events (not just more recent ones)
+        //                 post-migration after we add the missing period years values in SQL.
+        if (instance.eventTime.isAfter(DateTime.parse("2019-01-01T00:00:00Z"))) {
+          checkState(
+              instance.reason.hasPeriodYears() == (instance.periodYears != null),
+              "Period years must be set if and only if reason is "
+                  + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        }
         checkState(
             instance.getFlags().contains(Flag.SYNTHETIC)
                 == (instance.syntheticCreationTime != null),

core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java

@@ -107,6 +107,10 @@ public class DomainHistoryLite extends HistoryEntry implements SqlOnlyEntity {
     return VKey.create(DomainBase.class, getDomainRepoId());
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), getId());
+  }
+
   @PostLoad
   void postLoad() {
     if (domainContent == null) {

core/src/main/java/google/registry/model/common/Cursor.java

@@ -19,7 +19,9 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
+import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -294,5 +296,22 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity, Un
       this.type = type;
       this.scope = scope;
     }
+
+    /**
+     * A deterministic string representation of a {@link CursorId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of("scope", scope, "type", type)
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 }

core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java

@@ -204,7 +204,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
             MigrationState.DATASTORE_ONLY,
             "migrationTransitionMap must start with DATASTORE_ONLY");
     validateTransitionAtCurrentTime(transitions);
-    jpaTm().putIgnoringReadOnly(new DatabaseMigrationStateSchedule(transitions));
+    jpaTm().putIgnoringReadOnlyWithoutBackup(new DatabaseMigrationStateSchedule(transitions));
     CACHE.invalidateAll();
   }
 

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -16,6 +16,8 @@ package google.registry.model.contact;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
@@ -200,6 +202,24 @@ public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSer
     private void setId(long id) {
       this.id = id;
     }
+
+    /**
+     * A deterministic string representation of a {@link ContactHistoryId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "contactRepoId", getContactRepoId(), "id", getId())
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   @Override

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.domain;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+
 import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
@@ -168,6 +170,10 @@ public class DomainBase extends DomainContent
   @Override
   public void beforeSqlSaveOnReplay() {
     fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+    dsData =
+        dsData.stream()
+            .filter(datum -> datum.getDigest() != null && datum.getDigest().length > 0)
+            .collect(toImmutableSet());
   }
 
   @Override

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -18,7 +18,9 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
@@ -194,7 +196,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   @Access(AccessType.PROPERTY)
   @OneToMany(
       cascade = {CascadeType.ALL},
-      fetch = FetchType.EAGER)
+      fetch = FetchType.EAGER,
+      orphanRemoval = true)
   @JoinColumn(name = "historyRevisionId", referencedColumnName = "historyRevisionId")
   @JoinColumn(name = "domainRepoId", referencedColumnName = "domainRepoId")
   @SuppressWarnings("unused")
@@ -315,6 +318,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       domainHistory.nsHosts = nullToEmptyImmutableCopy(domainHistory.domainContent.nsHosts);
       domainHistory.dsDataHistories =
           nullToEmptyImmutableCopy(domainHistory.domainContent.getDsData()).stream()
+              .filter(dsData -> dsData.getDigest() != null && dsData.getDigest().length > 0)
               .map(dsData -> DomainDsDataHistory.createFrom(domainHistory.id, dsData))
               .collect(toImmutableSet());
       domainHistory.gracePeriodHistories =
@@ -382,6 +386,24 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     private void setId(long id) {
       this.id = id;
     }
+
+    /**
+     * A deterministic string representation of a {@link DomainHistoryId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "domainRepoId", getDomainRepoId(), "id", getId())
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   @Override

core/src/main/java/google/registry/model/domain/RegistryLock.java

@@ -28,6 +28,8 @@ import google.registry.util.DateTimeUtils;
 import java.time.ZonedDateTime;
 import java.util.Optional;
 import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
@@ -100,7 +102,11 @@ public final class RegistryLock extends ImmutableObject implements Buildable, Sq
   private String registrarPocId;
 
   /** When the lock is first requested. */
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "lockRequestTime", nullable = false))
+  })
   private CreateAutoTimestamp lockRequestTime = CreateAutoTimestamp.create(null);
 
   /** When the unlock is first requested. */

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -122,7 +122,6 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @Nullable @Index String domainName;
 
   /** When this token was created. */
-  @Column(nullable = false)
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /** Allowed registrar client IDs for this token, or null if all registrars are allowed. */

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -112,7 +112,6 @@ public enum StatusValue implements EppEnum {
    */
   PENDING_UPDATE(AllowedOn.NONE),
 
-
   /** A non-client-settable status that prevents deletes of EPP resources. */
   SERVER_DELETE_PROHIBITED(AllowedOn.ALL),
 
@@ -162,6 +161,10 @@ public enum StatusValue implements EppEnum {
     return xmlName.startsWith("client");
   }
 
+  public boolean isServerSettable() {
+    return xmlName.startsWith("server");
+  }
+
   public boolean isChargedStatus() {
     return xmlName.startsWith("server") && xmlName.endsWith("Prohibited");
   }

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -16,6 +16,8 @@ package google.registry.model.host;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
@@ -203,6 +205,24 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial
     private void setId(long id) {
       this.id = id;
     }
+
+    /**
+     * A deterministic string representation of a {@link HostHistoryId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "hostRepoId", getHostRepoId(), "id", getId())
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   @Override

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -64,6 +64,7 @@ public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEn
     EppResourceIndex instance = instantiate(EppResourceIndex.class);
     instance.reference = resourceKey;
     instance.kind = resourceKey.getKind();
+    // TODO(b/207368050): figure out if this value has ever been used other than test cases
     instance.id = resourceKey.getString(); // creates a web-safe key string
     instance.bucket = bucket;
     return instance;

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -82,6 +82,7 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
       com.google.appengine.api.datastore.Entity rawEntity) {
     CommitLogMutation instance = new CommitLogMutation();
     instance.parent = checkNotNull(parent);
+    // TODO(b/207516684): figure out if this should be converted to a vkey string via stringify()
     // Creates a web-safe key string.
     instance.entityKey = KeyFactory.keyToString(rawEntity.getKey());
     instance.entityProtoBytes = convertToPb(rawEntity).toByteArray();
@@ -91,6 +92,8 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
   /** Returns the key of a mutation based on the {@code entityKey} of the entity it stores. */
   public static
       Key<CommitLogMutation> createKey(Key<CommitLogManifest> parent, Key<?> entityKey) {
+    // TODO(b/207516684): figure out if the return type needs to be VKey and
+    //  if the string used to create a key should remain the same
     return Key.create(parent, CommitLogMutation.class, entityKey.getString());
   }
 }

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -356,13 +356,25 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public void putIgnoringReadOnly(Object entity) {
-    syncIfTransactionless(getOfy().saveIgnoringReadOnly().entities(toDatastoreEntity(entity)));
+  public void putIgnoringReadOnlyWithoutBackup(Object entity) {
+    syncIfTransactionless(
+        getOfy().saveIgnoringReadOnlyWithoutBackup().entities(toDatastoreEntity(entity)));
   }
 
   @Override
-  public void deleteIgnoringReadOnly(VKey<?> key) {
-    syncIfTransactionless(getOfy().deleteIgnoringReadOnly().key(key.getOfyKey()));
+  public void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key) {
+    syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithoutBackup().key(key.getOfyKey()));
+  }
+
+  /** Performs the write ignoring read-only restrictions and also writes commit logs. */
+  public void putIgnoringReadOnlyWithBackup(Object entity) {
+    syncIfTransactionless(
+        getOfy().saveIgnoringReadOnlyWithBackup().entities(toDatastoreEntity(entity)));
+  }
+
+  /** Performs the delete ignoring read-only restrictions and also writes commit logs. */
+  public void deleteIgnoringReadOnlyWithBackup(VKey<?> key) {
+    syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithBackup().key(key.getOfyKey()));
   }
 
   /**

core/src/main/java/google/registry/model/ofy/Ofy.java

@@ -133,15 +133,7 @@ public class Ofy {
    */
   public Deleter delete() {
     assertNotReadOnlyMode();
-    return new AugmentedDeleter() {
-      @Override
-      protected void handleDeletion(Iterable<Key<?>> keys) {
-        assertInTransaction();
-        checkState(Streams.stream(keys).allMatch(Objects::nonNull), "Can't delete a null key.");
-        checkProhibitedAnnotations(keys, NotBackedUp.class, VirtualEntity.class);
-        TRANSACTION_INFO.get().putDeletes(keys);
-      }
-    };
+    return deleteIgnoringReadOnlyWithBackup();
   }
 
   /**
@@ -151,7 +143,7 @@ public class Ofy {
    */
   public Deleter deleteWithoutBackup() {
     assertNotReadOnlyMode();
-    return deleteIgnoringReadOnly();
+    return deleteIgnoringReadOnlyWithoutBackup();
   }
 
   /**
@@ -162,6 +154,41 @@ public class Ofy {
    */
   public Saver save() {
     assertNotReadOnlyMode();
+    return saveIgnoringReadOnlyWithBackup();
+  }
+
+  /**
+   * Save, without any augmentations except to check that we're not saving any virtual entities.
+   *
+   * <p>No backups get written.
+   */
+  public Saver saveWithoutBackup() {
+    assertNotReadOnlyMode();
+    return saveIgnoringReadOnlyWithoutBackup();
+  }
+
+  /** Save, ignoring any backups or any read-only settings. */
+  public Saver saveIgnoringReadOnlyWithoutBackup() {
+    return new AugmentedSaver() {
+      @Override
+      protected void handleSave(Iterable<?> entities) {
+        checkProhibitedAnnotations(entities, VirtualEntity.class);
+      }
+    };
+  }
+
+  /** Delete, ignoring any backups or any read-only settings. */
+  public Deleter deleteIgnoringReadOnlyWithoutBackup() {
+    return new AugmentedDeleter() {
+      @Override
+      protected void handleDeletion(Iterable<Key<?>> keys) {
+        checkProhibitedAnnotations(keys, VirtualEntity.class);
+      }
+    };
+  }
+
+  /** Save, ignoring any read-only settings (but still write commit logs). */
+  public Saver saveIgnoringReadOnlyWithBackup() {
     return new AugmentedSaver() {
       @Override
       protected void handleSave(Iterable<?> entities) {
@@ -175,32 +202,15 @@ public class Ofy {
     };
   }
 
-  /**
-   * Save, without any augmentations except to check that we're not saving any virtual entities.
-   *
-   * <p>No backups get written.
-   */
-  public Saver saveWithoutBackup() {
-    assertNotReadOnlyMode();
-    return saveIgnoringReadOnly();
-  }
-
-  /** Save, ignoring any backups or any read-only settings. */
-  public Saver saveIgnoringReadOnly() {
-    return new AugmentedSaver() {
-      @Override
-      protected void handleSave(Iterable<?> entities) {
-        checkProhibitedAnnotations(entities, VirtualEntity.class);
-      }
-    };
-  }
-
-  /** Delete, ignoring any backups or any read-only settings. */
-  public Deleter deleteIgnoringReadOnly() {
+  /** Delete, ignoring any read-only settings (but still write commit logs). */
+  public Deleter deleteIgnoringReadOnlyWithBackup() {
     return new AugmentedDeleter() {
       @Override
       protected void handleDeletion(Iterable<Key<?>> keys) {
-        checkProhibitedAnnotations(keys, VirtualEntity.class);
+        assertInTransaction();
+        checkState(Streams.stream(keys).allMatch(Objects::nonNull), "Can't delete a null key.");
+        checkProhibitedAnnotations(keys, NotBackedUp.class, VirtualEntity.class);
+        TRANSACTION_INFO.get().putDeletes(keys);
       }
     };
   }

core/src/main/java/google/registry/model/ofy/ReplayQueue.java

@@ -95,22 +95,21 @@ public class ReplayQueue {
       // Sort the changes into an order that will work for insertion into the database.
       jpaTm()
           .transact(
-              () -> {
-                changes.entrySet().stream()
-                    .sorted(ReplayQueue::compareByPriority)
-                    .forEach(
-                        entry -> {
-                          if (entry.getValue().equals(TransactionInfo.Delete.SENTINEL)) {
-                            VKey<?> vkey = VKey.from(entry.getKey());
-                            ReplaySpecializer.beforeSqlDelete(vkey);
-                            jpaTm().delete(vkey);
-                          } else {
-                            ((DatastoreEntity) entry.getValue())
-                                .toSqlEntity()
-                                .ifPresent(jpaTm()::put);
-                          }
-                        });
-              });
+              () ->
+                  changes.entrySet().stream()
+                      .sorted(ReplayQueue::compareByPriority)
+                      .forEach(
+                          entry -> {
+                            if (entry.getValue().equals(TransactionInfo.Delete.SENTINEL)) {
+                              VKey<?> vkey = VKey.from(entry.getKey());
+                              ReplaySpecializer.beforeSqlDelete(vkey);
+                              jpaTm().delete(vkey);
+                            } else {
+                              ((DatastoreEntity) entry.getValue())
+                                  .toSqlEntity()
+                                  .ifPresent(jpaTm()::put);
+                            }
+                          }));
     }
   }
 }

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -354,6 +354,10 @@ public abstract class PollMessage extends ImmutableObject
     @Column(name = "transfer_response_contact_id")
     String contactId;
 
+    @Ignore
+    @Column(name = "transfer_response_host_id")
+    String hostId;
+
     @Override
     public VKey<OneTime> createVKey() {
       return VKey.create(OneTime.class, getId(), Key.create(this));
@@ -393,6 +397,9 @@ public abstract class PollMessage extends ImmutableObject
       if (!isNullOrEmpty(contactPendingActionNotificationResponses)) {
         pendingActionNotificationResponse = contactPendingActionNotificationResponses.get(0);
       }
+      if (!isNullOrEmpty(hostPendingActionNotificationResponses)) {
+        pendingActionNotificationResponse = hostPendingActionNotificationResponses.get(0);
+      }
       if (!isNullOrEmpty(contactTransferResponses)) {
         contactId = contactTransferResponses.get(0).getContactId();
         transferResponse = contactTransferResponses.get(0);
@@ -433,6 +440,16 @@ public abstract class PollMessage extends ImmutableObject
                   pendingActionNotificationResponse.processedDate);
           pendingActionNotificationResponse = domainPendingResponse;
           domainPendingActionNotificationResponses = ImmutableList.of(domainPendingResponse);
+        } else if (hostId != null) {
+          HostPendingActionNotificationResponse hostPendingActionNotificationResponse =
+              HostPendingActionNotificationResponse.create(
+                  pendingActionNotificationResponse.nameOrId.value,
+                  pendingActionNotificationResponse.getActionResult(),
+                  pendingActionNotificationResponse.getTrid(),
+                  pendingActionNotificationResponse.processedDate);
+          pendingActionNotificationResponse = hostPendingActionNotificationResponse;
+          hostPendingActionNotificationResponses =
+              ImmutableList.of(hostPendingActionNotificationResponse);
         }
       }
       if (transferResponse != null) {
@@ -527,6 +544,7 @@ public abstract class PollMessage extends ImmutableObject
         } else if (instance.hostPendingActionNotificationResponses != null) {
           instance.pendingActionNotificationResponse =
               instance.hostPendingActionNotificationResponses.get(0);
+          instance.hostId = instance.hostPendingActionNotificationResponses.get(0).nameOrId.value;
         }
         // Set the generic transfer response field as appropriate
         if (instance.contactTransferResponses != null) {

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -32,7 +32,9 @@ import static java.util.stream.Collectors.joining;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Enums;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
@@ -413,6 +415,24 @@ public class RegistrarContact extends ImmutableObject
       this.emailAddress = emailAddress;
       this.registrarId = registrarId;
     }
+
+    /**
+     * A deterministic string representation of a {@link RegistrarPocId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "emailAddress", emailAddress, "registrarId", registrarId)
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   /** A builder for constructing a {@link RegistrarContact}, since it is immutable. */

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -148,7 +148,9 @@ public class ReplicateToDatastoreAction implements Runnable {
 
                 // Write the updated last transaction id to Datastore as part of this Datastore
                 // transaction.
-                auditedOfy().save().entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
+                auditedOfy()
+                    .saveIgnoringReadOnlyWithoutBackup()
+                    .entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
                 logger.atInfo().log(
                     "Finished applying single transaction Cloud SQL -> Cloud Datastore.");
               });
@@ -194,7 +196,7 @@ public class ReplicateToDatastoreAction implements Runnable {
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 

core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java

@@ -41,6 +41,6 @@ public class SqlReplayCheckpoint extends CrossTldSingleton implements SqlOnlyEnt
     SqlReplayCheckpoint checkpoint = new SqlReplayCheckpoint();
     checkpoint.lastReplayTime = lastReplayTime;
     // this will overwrite the existing object due to the constant revisionId
-    jpaTm().putIgnoringReadOnly(checkpoint);
+    jpaTm().putIgnoringReadOnlyWithoutBackup(checkpoint);
   }
 }

core/src/main/java/google/registry/model/server/Lock.java

@@ -293,7 +293,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
               create(resourceName, scope, requestStatusChecker.getLogId(), now, leaseLength);
           // Locks are not parented under an EntityGroupRoot (so as to avoid write
           // contention) and don't need to be backed up.
-          transactionManager.putIgnoringReadOnly(newLock);
+          transactionManager.putIgnoringReadOnlyWithoutBackup(newLock);
 
           return AcquireResult.create(now, lock, newLock, lockState);
         };
@@ -325,7 +325,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
             // Use deleteIgnoringReadOnly() so that we don't create a commit log entry for deleting
             // the lock.
             logger.atInfo().log("Deleting lock: %s", lockId);
-            transactionManager.deleteIgnoringReadOnly(key);
+            transactionManager.deleteIgnoringReadOnlyWithoutBackup(key);
 
             lockMetrics.recordRelease(
                 resourceName,

core/src/main/java/google/registry/model/tld/Registry.java

@@ -64,6 +64,7 @@ import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.ReservedList;
 import google.registry.persistence.VKey;
+import google.registry.persistence.converter.JodaMoneyType;
 import google.registry.util.Idn;
 import java.util.Map;
 import java.util.Optional;
@@ -72,13 +73,13 @@ import java.util.concurrent.ExecutionException;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
-import javax.persistence.AttributeOverride;
-import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.persistence.PostLoad;
 import javax.persistence.Transient;
+import org.hibernate.annotations.Columns;
+import org.hibernate.annotations.Type;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -466,47 +467,39 @@ public class Registry extends ImmutableObject
   CurrencyUnit currency = DEFAULT_CURRENCY;
 
   /** The per-year billing cost for registering a new domain name. */
-  @AttributeOverrides({
-    @AttributeOverride(
-        name = "money.amount",
-        column = @Column(name = "create_billing_cost_amount")),
-    @AttributeOverride(
-        name = "money.currency",
-        column = @Column(name = "create_billing_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "create_billing_cost_amount"),
+        @Column(name = "create_billing_cost_currency")
+      })
   Money createBillingCost = DEFAULT_CREATE_BILLING_COST;
 
   /** The one-time billing cost for restoring a domain name from the redemption grace period. */
-  @AttributeOverrides({
-    @AttributeOverride(
-        name = "money.amount",
-        column = @Column(name = "restore_billing_cost_amount")),
-    @AttributeOverride(
-        name = "money.currency",
-        column = @Column(name = "restore_billing_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "restore_billing_cost_amount"),
+        @Column(name = "restore_billing_cost_currency")
+      })
   Money restoreBillingCost = DEFAULT_RESTORE_BILLING_COST;
 
   /** The one-time billing cost for changing the server status (i.e. lock). */
-  @AttributeOverrides({
-    @AttributeOverride(
-        name = "money.amount",
-        column = @Column(name = "server_status_change_billing_cost_amount")),
-    @AttributeOverride(
-        name = "money.currency",
-        column = @Column(name = "server_status_change_billing_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "server_status_change_billing_cost_amount"),
+        @Column(name = "server_status_change_billing_cost_currency")
+      })
   Money serverStatusChangeBillingCost = DEFAULT_SERVER_STATUS_CHANGE_BILLING_COST;
 
   /** The one-time billing cost for a registry lock/unlock action initiated by a registrar. */
-  @AttributeOverrides({
-      @AttributeOverride(
-          name = "money.amount",
-          column = @Column(name = "registry_lock_or_unlock_cost_amount")),
-      @AttributeOverride(
-          name = "money.currency",
-          column = @Column(name = "registry_lock_or_unlock_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "registry_lock_or_unlock_cost_amount"),
+        @Column(name = "registry_lock_or_unlock_cost_currency")
+      })
   Money registryLockOrUnlockBillingCost = DEFAULT_REGISTRY_LOCK_OR_UNLOCK_BILLING_COST;
 
   /**

core/src/main/java/google/registry/model/tmch/ClaimsList.java

@@ -27,6 +27,8 @@ import google.registry.model.replay.SqlOnlyEntity;
 import google.registry.model.tld.label.ReservedList.ReservedListEntry;
 import java.util.Map;
 import java.util.Optional;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
@@ -56,7 +58,11 @@ public class ClaimsList extends ImmutableObject implements SqlOnlyEntity {
   @GeneratedValue(strategy = GenerationType.IDENTITY)
   Long revisionId;
 
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "creationTimestamp", nullable = false))
+  })
   CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
 
   /**

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -16,25 +16,18 @@ package google.registry.model.tmch;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
-import com.googlecode.objectify.annotation.Entity;
-import google.registry.model.annotations.NotBackedUp;
-import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
-import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
 import javax.persistence.Column;
 import org.joda.time.DateTime;
 
-/** Datastore singleton for ICANN's TMCH CA certificate revocation list (CRL). */
-@Entity
+/** Singleton for ICANN's TMCH CA certificate revocation list (CRL). */
 @javax.persistence.Entity
 @Immutable
-@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEntity {
+public final class TmchCrl extends CrossTldSingleton implements SqlOnlyEntity {
 
   @Column(name = "certificateRevocations", nullable = false)
   String crl;
@@ -47,25 +40,23 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
 
   /** Returns the singleton instance of this entity, without memoization. */
   public static Optional<TmchCrl> get() {
-    return tm().transact(() -> tm().loadSingleton(TmchCrl.class));
+    return jpaTm().transact(() -> jpaTm().loadSingleton(TmchCrl.class));
   }
 
   /**
-   * Change the Datastore singleton to a new ASCII-armored X.509 CRL.
+   * Change the singleton to a new ASCII-armored X.509 CRL.
    *
    * <p>Please do not call this function unless your CRL is properly formatted, signed by the root,
    * and actually newer than the one currently in Datastore.
-   *
-   * <p>During the dual-write period, we write to both Datastore and SQL
    */
   public static void set(final String crl, final String url) {
-    tm().transact(
+    jpaTm()
+        .transact(
             () -> {
               TmchCrl tmchCrl = new TmchCrl();
-              tmchCrl.updated = tm().getTransactionTime();
+              tmchCrl.updated = jpaTm().getTransactionTime();
               tmchCrl.crl = checkNotNull(crl, "crl");
               tmchCrl.url = checkNotNull(url, "url");
-              ofyTm().transactNew(() -> ofyTm().putWithoutBackup(tmchCrl));
               jpaTm().transactNew(() -> jpaTm().putWithoutBackup(tmchCrl));
             });
   }
@@ -80,7 +71,7 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
     return crl;
   }
 
-  /** Time we last updated the Datastore with a newer ICANN CRL. */
+  /** Time we last updated the Database with a newer ICANN CRL. */
   public final DateTime getUpdated() {
     return updated;
   }

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -60,6 +60,7 @@ import google.registry.export.sheet.SheetModule;
 import google.registry.export.sheet.SyncRegistrarsSheetAction;
 import google.registry.flows.FlowComponent;
 import google.registry.mapreduce.MapreduceModule;
+import google.registry.model.replay.ReplicateToDatastoreAction;
 import google.registry.monitoring.whitebox.WhiteboxModule;
 import google.registry.rdap.UpdateRegistrarRdapBaseUrlsAction;
 import google.registry.rde.BrdaCopyAction;
@@ -190,6 +191,8 @@ interface BackendRequestComponent {
 
   ReplayCommitLogsToSqlAction replayCommitLogsToSqlAction();
 
+  ReplicateToDatastoreAction replicateToDatastoreAction();
+
   ResaveAllEppResourcesAction resaveAllEppResourcesAction();
 
   ResaveEntityAction resaveEntityAction();

core/src/main/java/google/registry/persistence/NomulusPostgreSQLDialect.java

@@ -14,6 +14,7 @@
 package google.registry.persistence;
 
 import google.registry.persistence.converter.IntervalDescriptor;
+import google.registry.persistence.converter.JodaMoneyType;
 import google.registry.persistence.converter.StringCollectionDescriptor;
 import google.registry.persistence.converter.StringMapDescriptor;
 import java.sql.Types;
@@ -34,6 +35,7 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     registerColumnType(IntervalDescriptor.COLUMN_TYPE, IntervalDescriptor.COLUMN_NAME);
   }
 
+  @SuppressWarnings("deprecation") // See comments below on JodaMoneyType.
   @Override
   public void contributeTypes(
       TypeContributions typeContributions, ServiceRegistry serviceRegistry) {
@@ -44,5 +46,8 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     typeContributions.contributeSqlTypeDescriptor(StringMapDescriptor.getInstance());
     typeContributions.contributeJavaTypeDescriptor(IntervalDescriptor.getInstance());
     typeContributions.contributeSqlTypeDescriptor(IntervalDescriptor.getInstance());
+    // Below method (contributing CompositeUserType) is deprecated. Please see javadoc of
+    // JodaMoneyType for reasons.
+    typeContributions.contributeType(JodaMoneyType.INSTANCE, JodaMoneyType.TYPE_NAME);
   }
 }

core/src/main/java/google/registry/persistence/VKey.java

@@ -311,4 +311,29 @@ public class VKey<T> extends ImmutableObject implements Serializable {
     }
     return key;
   }
+
+  /**
+   * Constructs the readable string representation of a {@link VKey}.
+   *
+   * <p>This readable string representation of a vkey contains its type and its sql key or ofy key,
+   * or both.
+   */
+  @Override
+  public String toString() {
+    if (maybeGetOfyKey().isPresent() && maybeGetSqlKey().isPresent()) {
+      return String.format(
+          "VKey<%s>(%s:%s,%s:%s)",
+          getKind().getSimpleName(), SQL_LOOKUP_KEY, sqlKey, OFY_LOOKUP_KEY, ofyKeyToString());
+    } else if (maybeGetSqlKey().isPresent()) {
+      return String.format("VKey<%s>(%s:%s)", getKind().getSimpleName(), SQL_LOOKUP_KEY, sqlKey);
+    } else if (maybeGetOfyKey().isPresent()) {
+      return String.format("VKey<%s>(%s:%s)", ofyKey.getKind(), OFY_LOOKUP_KEY, ofyKeyToString());
+    } else {
+      throw new IllegalStateException("VKey should contain at least one form of key");
+    }
+  }
+
+  private String ofyKeyToString() {
+    return ofyKey.getName() == null ? String.valueOf(ofyKey.getId()) : ofyKey.getName();
+  }
 }

core/src/main/java/google/registry/persistence/converter/CreateAutoTimestampConverter.java

@@ -1,53 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package google.registry.persistence.converter;
-
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-
-import google.registry.model.CreateAutoTimestamp;
-import google.registry.util.DateTimeUtils;
-import java.sql.Timestamp;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-import javax.annotation.Nullable;
-import javax.persistence.AttributeConverter;
-import javax.persistence.Converter;
-import org.joda.time.DateTime;
-
-/** JPA converter to for storing/retrieving CreateAutoTimestamp objects. */
-@Converter(autoApply = true)
-public class CreateAutoTimestampConverter
-    implements AttributeConverter<CreateAutoTimestamp, Timestamp> {
-
-  @Override
-  @Nullable
-  public Timestamp convertToDatabaseColumn(@Nullable CreateAutoTimestamp entity) {
-    if (entity == null) {
-      return null;
-    }
-    DateTime dateTime = firstNonNull(entity.getTimestamp(), jpaTm().getTransactionTime());
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(dateTime).toInstant());
-  }
-
-  @Override
-  @Nullable
-  public CreateAutoTimestamp convertToEntityAttribute(@Nullable Timestamp columnValue) {
-    if (columnValue == null) {
-      return null;
-    }
-    ZonedDateTime zdt = ZonedDateTime.ofInstant(columnValue.toInstant(), ZoneOffset.UTC);
-    return CreateAutoTimestamp.create(DateTimeUtils.toJodaDateTime(zdt));
-  }
-}

core/src/main/java/google/registry/persistence/converter/JodaMoneyType.java

@@ -0,0 +1,178 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import com.google.common.collect.ImmutableList;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Objects;
+import org.hibernate.HibernateException;
+import org.hibernate.engine.spi.SharedSessionContractImplementor;
+import org.hibernate.type.StandardBasicTypes;
+import org.hibernate.type.Type;
+import org.hibernate.usertype.CompositeUserType;
+import org.joda.money.CurrencyUnit;
+import org.joda.money.Money;
+
+/**
+ * Defines JPA mapping for {@link Money Joda Money type}.
+ *
+ * <p>{@code Money} is mapped to two table columns, a text {@code currency} column that stores the
+ * currency code, and a numeric {@code amount} column that stores the amount.
+ *
+ * <p>The main purpose of this class is to normalize the amount loaded from the database. To support
+ * all currency types, the scale of the numeric column is set to 2. As a result, the {@link
+ * BigDecimal} instances obtained from query ResultSets all have their scale at 2. However, some
+ * currency types, e.g., JPY requires that the scale be zero. This class strips trailing zeros from
+ * each loaded BigDecimal, then calls the appropriate factory method for Money, which will adjust
+ * the scale appropriately.
+ *
+ * <p>Although {@link CompositeUserType} is likely to suffer breaking change in Hibernate 6, it is
+ * the only option. The suggested alternatives such as Hibernate component or Java Embeddable do not
+ * work in this case. Hibernate component (our previous solution that is replaced by this class)
+ * does not allow manipulation of the loaded amount objects. Java Embeddable is not applicable since
+ * we do not own the Joda money classes.
+ *
+ * <p>Usage:
+ *
+ * <pre>{@code
+ * '@'Type(type = JodaMoneyType.TYPE_NAME)
+ * '@'Columns(
+ *   columns = {
+ *     '@'Column(name = "cost_amount"),
+ *     '@'Column(name = "cost_currency")
+ *   }
+ * )
+ * Money cost;
+ * }</pre>
+ */
+public class JodaMoneyType implements CompositeUserType {
+
+  public static final JodaMoneyType INSTANCE = new JodaMoneyType();
+
+  /** The name of this type registered with JPA. See the example in class doc. */
+  public static final String TYPE_NAME = "JodaMoney";
+
+  // JPA property names that can be used in JPQL queries.
+  private static final ImmutableList<String> JPA_PROPERTY_NAMES =
+      ImmutableList.of("amount", "currency");
+  private static final ImmutableList<Type> PROPERTY_TYPES =
+      ImmutableList.of(StandardBasicTypes.BIG_DECIMAL, StandardBasicTypes.STRING);
+  private static final int AMOUNT_ID = JPA_PROPERTY_NAMES.indexOf("amount");
+  private static final int CURRENCY_ID = JPA_PROPERTY_NAMES.indexOf("currency");
+
+  @Override
+  public String[] getPropertyNames() {
+    return JPA_PROPERTY_NAMES.toArray(new String[0]);
+  }
+
+  @Override
+  public Type[] getPropertyTypes() {
+    return PROPERTY_TYPES.toArray(new Type[0]);
+  }
+
+  @Override
+  public Object getPropertyValue(Object component, int property) throws HibernateException {
+    if (property >= JPA_PROPERTY_NAMES.size()) {
+      throw new HibernateException("Property index too large: " + property);
+    }
+    Money money = (Money) component;
+    return property == AMOUNT_ID ? money.getAmount() : money.getCurrencyUnit().getCode();
+  }
+
+  @Override
+  public void setPropertyValue(Object component, int property, Object value)
+      throws HibernateException {
+    throw new HibernateException("Money is immutable");
+  }
+
+  @Override
+  public Class returnedClass() {
+    return Money.class;
+  }
+
+  @Override
+  public boolean equals(Object x, Object y) throws HibernateException {
+    return Objects.equals(x, y);
+  }
+
+  @Override
+  public int hashCode(Object x) throws HibernateException {
+    return Objects.hashCode(x);
+  }
+
+  @Override
+  public Object nullSafeGet(
+      ResultSet rs, String[] names, SharedSessionContractImplementor session, Object owner)
+      throws HibernateException, SQLException {
+    BigDecimal amount = StandardBasicTypes.BIG_DECIMAL.nullSafeGet(rs, names[AMOUNT_ID], session);
+    CurrencyUnit currencyUnit =
+        CurrencyUnit.of(StandardBasicTypes.STRING.nullSafeGet(rs, names[CURRENCY_ID], session));
+    if (amount != null && currencyUnit != null) {
+      return Money.of(currencyUnit, amount.stripTrailingZeros());
+    }
+    if (amount == null && currencyUnit == null) {
+      return null;
+    }
+    throw new HibernateException("Mismatching null state between currency and amount.");
+  }
+
+  @Override
+  public void nullSafeSet(
+      PreparedStatement st, Object value, int index, SharedSessionContractImplementor session)
+      throws HibernateException, SQLException {
+    BigDecimal amount = value == null ? null : ((Money) value).getAmount();
+    String currencyUnit = value == null ? null : ((Money) value).getCurrencyUnit().getCode();
+
+    if ((amount == null && currencyUnit != null) || (amount != null && currencyUnit == null)) {
+      throw new HibernateException("Mismatching null state between currency and amount.");
+    }
+    StandardBasicTypes.BIG_DECIMAL.nullSafeSet(st, amount, index, session);
+    StandardBasicTypes.STRING.nullSafeSet(st, currencyUnit, index + 1, session);
+  }
+
+  @Override
+  public Object deepCopy(Object value) throws HibernateException {
+    return value;
+  }
+
+  @Override
+  public boolean isMutable() {
+    return false;
+  }
+
+  @Override
+  public Serializable disassemble(Object value, SharedSessionContractImplementor session)
+      throws HibernateException {
+    return ((Money) value);
+  }
+
+  @Override
+  public Object assemble(
+      Serializable cached, SharedSessionContractImplementor session, Object owner)
+      throws HibernateException {
+    return cached;
+  }
+
+  @Override
+  public Object replace(
+      Object original, Object target, SharedSessionContractImplementor session, Object owner)
+      throws HibernateException {
+    return original;
+  }
+}

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -569,7 +569,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     }
     EntityType<?> entityType = getEntityType(key.getKind());
     ImmutableSet<EntityId> entityIds = getEntityIdsFromSqlKey(entityType, key.getSqlKey());
-    // TODO(b/179158393): use Criteria for query to leave not doubt about sql injection risk.
     String sql =
         String.format("DELETE FROM %s WHERE %s", entityType.getName(), getAndClause(entityIds));
     Query query = query(sql);
@@ -638,7 +637,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void putIgnoringReadOnly(Object entity) {
+  public void putIgnoringReadOnlyWithoutBackup(Object entity) {
     checkArgumentNotNull(entity);
     if (isEntityOfIgnoredClass(entity)) {
       return;
@@ -653,7 +652,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void deleteIgnoringReadOnly(VKey<?> key) {
+  public void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key) {
     checkArgumentNotNull(key, "key must be specified");
     assertInTransaction();
     if (IGNORED_ENTITY_CLASSES.contains(key.getKind())) {

core/src/main/java/google/registry/persistence/transaction/Transaction.java

@@ -242,7 +242,7 @@ public class Transaction extends ImmutableObject implements Buildable {
       if (entity instanceof DatastoreEntity) {
         ((DatastoreEntity) entity).beforeDatastoreSaveOnReplay();
       }
-      ofyTm().putIgnoringReadOnly(entity);
+      ofyTm().putIgnoringReadOnlyWithBackup(entity);
     }
 
     @Override
@@ -280,7 +280,7 @@ public class Transaction extends ImmutableObject implements Buildable {
 
     @Override
     public void writeToDatastore() {
-      ofyTm().deleteIgnoringReadOnly(key);
+      ofyTm().deleteIgnoringReadOnlyWithBackup(key);
     }
 
     @Override

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -312,9 +312,9 @@ public interface TransactionManager {
   /** Returns true if the transaction manager is DatastoreTransactionManager, false otherwise. */
   boolean isOfy();
 
-  /** Performs the given write ignoring any read-only restrictions, for use only in replay. */
-  void putIgnoringReadOnly(Object entity);
+  /** Performs the write ignoring any read-only restrictions or backup, for use only in replay. */
+  void putIgnoringReadOnlyWithoutBackup(Object entity);
 
-  /** Performs the given delete ignoring any read-only restrictions, for use only in replay. */
-  void deleteIgnoringReadOnly(VKey<?> key);
+  /** Performs the delete ignoring any read-only restrictions or backup, for use only in replay. */
+  void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key);
 }

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -157,7 +157,7 @@ public class TransactionManagerFactory {
 
   /** Registry is currently undergoing maintenance and is in read-only mode. */
   public static class ReadOnlyModeException extends IllegalStateException {
-    ReadOnlyModeException() {
+    public ReadOnlyModeException() {
       super("Registry is currently undergoing maintenance and is in read-only mode");
     }
   }

core/src/main/java/google/registry/privileges/secretmanager/SqlUser.java

@@ -54,7 +54,10 @@ public abstract class SqlUser {
      * Credential for RegistryTool. This is temporary, and will be removed when tool users are
      * assigned their personal credentials.
      */
-    TOOL
+    TOOL,
+
+    /** The Postgres admin account. */
+    POSTGRES
   }
 
   /** Information of a RobotUser for privilege management purposes. */

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -14,12 +14,14 @@
 
 package google.registry.rde;
 
+import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
 import static google.registry.beam.BeamUtils.createJobName;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.xml.ValidationMode.LENIENT;
 import static google.registry.xml.ValidationMode.STRICT;
+import static java.util.function.Function.identity;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -29,6 +31,7 @@ import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
 import com.google.common.base.Ascii;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -48,6 +51,7 @@ import google.registry.model.EppResource;
 import google.registry.model.common.Cursor;
 import google.registry.model.common.Cursor.CursorType;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.rde.RdeMode;
@@ -67,14 +71,17 @@ import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /**
- * MapReduce that idempotently stages escrow deposit XML files on GCS for RDE/BRDA for all TLDs.
+ * Action that kicks off either a MapReduce (for Datastore) or Dataflow (for Cloud SQL) job to stage
+ * escrow deposit XML files on GCS for RDE/BRDA for all TLDs.
  *
- * <h3>MapReduce Operation</h3>
+ * <h3>Pending Deposits</h3>
  *
  * <p>This task starts by asking {@link PendingDepositChecker} which deposits need to be generated.
  * If there's nothing to deposit, we return 204 No Content; otherwise, we fire off a MapReduce job
  * and redirect to its status GUI. The task can also be run in manual operation, as described below.
  *
+ * <h3>MapReduce</h3>
+ *
  * <p>The mapreduce job scans every {@link EppResource} in Datastore. It maps a point-in-time
  * representation of each entity to the escrow XML files in which it should appear.
  *
@@ -88,11 +95,26 @@ import org.joda.time.Duration;
  * <p>{@link Registrar} entities, both active and inactive, are included in all deposits. They are
  * not rewinded point-in-time.
  *
+ * <h3>Dataflow</h3>
+ *
+ * The Dataflow job finds the most recent history entry on or before watermark for each resource
+ * type and loads the embedded resource from it, which is then projected to watermark time to
+ * account for things like pending transfer.
+ *
+ * <p>Only {@link ContactResource}s and {@link HostResource}s that are referenced by an included
+ * {@link DomainBase} will be included in the corresponding pending deposit.
+ *
+ * <p>{@link Registrar} entities, both active and inactive, are included in all deposits. They are
+ * not rewinded point-in-time.
+ *
+ * <h3>Afterward</h3>
+ *
  * <p>The XML deposit files generated by this job are humongous. A tiny XML report file is generated
  * for each deposit, telling us how much of what it contains.
  *
- * <p>Once a deposit is successfully generated, an {@link RdeUploadAction} is enqueued which will
- * upload it via SFTP to the third-party escrow provider.
+ * <p>Once a deposit is successfully generated, For RDE an {@link RdeUploadAction} is enqueued which
+ * will upload it via SFTP to the third-party escrow provider; for BRDA an {@link BrdaCopyAction} is
+ * enqueued which will copy it to a GCS bucket and be rsynced to a third-party escrow provider.
  *
  * <p>To generate escrow deposits manually and locally, use the {@code nomulus} tool command {@code
  * GenerateEscrowDepositCommand}.
@@ -106,7 +128,7 @@ import org.joda.time.Duration;
  *
  * <p>Valid model objects might not be valid to the RDE XML schema. A single invalid object will
  * cause the whole deposit to fail. You need to check the logs, find out which entities are broken,
- * and perform Datastore surgery.
+ * and perform database surgery.
  *
  * <p>If a deposit fails, an error is emitted to the logs for each broken entity. It tells you the
  * key and shows you its representation in lenient XML.
@@ -137,33 +159,40 @@ import org.joda.time.Duration;
  * <p>The deposit and report are encrypted using {@link Ghostryde}. Administrators can use the
  * {@code GhostrydeCommand} command in the {@code nomulus} tool to view them.
  *
- * <p>Unencrypted XML fragments are stored temporarily between the map and reduce steps. The
- * ghostryde encryption on the full archived deposits makes life a little more difficult for an
- * attacker. But security ultimately depends on the bucket.
+ * <p>Unencrypted XML fragments are stored temporarily between the map and reduce steps and between
+ * Dataflow transforms. The ghostryde encryption on the full archived deposits makes life a little
+ * more difficult for an attacker. But security ultimately depends on the bucket.
  *
  * <h3>Idempotency</h3>
  *
- * <p>We lock the reduce tasks. This is necessary because: a) App Engine tasks might get double
- * executed; and b) Cloud Storage file handles get committed on close <i>even if our code throws an
- * exception.</i>
+ * <p>We lock the reduce tasks for the MapReduce job. This is necessary because: a) App Engine tasks
+ * might get double executed; and b) Cloud Storage file handles get committed on close <i>even if
+ * our code throws an exception.</i>
+ *
+ * <p>For the Dataflow job we do not employ a lock because it is difficult to span a lock across
+ * three subsequent transforms (save to GCS, roll forward cursor, enqueue next action). Instead, we
+ * get around the issue by saving the deposit to a unique folder named after the job name so there
+ * is no possibility of overwriting.
  *
  * <p>Deposits are generated serially for a given (watermark, mode) pair. A deposit is never started
  * beyond the cursor. Once a deposit is completed, its cursor is rolled forward transactionally.
  * Duplicate jobs may exist {@code <=cursor}. So a transaction will not bother changing the cursor
  * if it's already been rolled forward.
  *
- * <p>Enqueuing {@code RdeUploadAction} is also part of the cursor transaction. This is necessary
- * because the first thing the upload task does is check the staging cursor to verify it's been
- * completed, so we can't enqueue before we roll. We also can't enqueue after the roll, because then
- * if enqueuing fails, the upload might never be enqueued.
+ * <p>Enqueuing {@code RdeUploadAction} or {@code BrdaCopyAction} is also part of the cursor
+ * transaction. This is necessary because the first thing the upload task does is check the staging
+ * cursor to verify it's been completed, so we can't enqueue before we roll. We also can't enqueue
+ * after the roll, because then if enqueuing fails, the upload might never be enqueued.
  *
  * <h3>Determinism</h3>
  *
  * <p>The filename of an escrow deposit is determistic for a given (TLD, watermark, {@linkplain
  * RdeMode mode}) triplet. Its generated contents is deterministic in all the ways that we care
- * about. Its view of the database is strongly consistent.
+ * about. Its view of the database is strongly consistent in Cloud SQL automatically by nature of
+ * the initial query for the history entry running at {@code READ_COMMITTED} transaction isolation
+ * level.
  *
- * <p>This is because:
+ * <p>This is also true in Datastore because:
  *
  * <ol>
  *   <li>{@code EppResource} queries are strongly consistent thanks to {@link EppResourceIndex}
@@ -226,6 +255,11 @@ public final class RdeStagingAction implements Runnable {
   @Inject MapreduceRunner mrRunner;
   @Inject @Config("projectId") String projectId;
   @Inject @Config("defaultJobRegion") String jobRegion;
+
+  @Inject
+  @Config("highPerformanceMachineType")
+  String machineType;
+
   @Inject @Config("transactionCooldown") Duration transactionCooldown;
   @Inject @Config("beamStagingBucketUrl") String stagingBucketUrl;
   @Inject @Config("rdeBucket") String rdeBucket;
@@ -270,43 +304,65 @@ public final class RdeStagingAction implements Runnable {
                   new NullInput<>(), EppResourceInputs.createEntityInput(EppResource.class)))
           .sendLinkToMapreduceConsole(response);
     } else {
-      try {
-        LaunchFlexTemplateParameter parameter =
-            new LaunchFlexTemplateParameter()
-                .setJobName(createJobName("rde", clock))
-                .setContainerSpecGcsPath(
-                    String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-                .setParameters(
-                    ImmutableMap.of(
-                        "pendings",
-                        RdePipeline.encodePendings(pendings),
-                        "validationMode",
-                        validationMode.name(),
-                        "rdeStagingBucket",
-                        rdeBucket,
-                        "stagingKey",
-                        BaseEncoding.base64Url().omitPadding().encode(stagingKeyBytes),
-                        "registryEnvironment",
-                        RegistryEnvironment.get().name()));
-        LaunchFlexTemplateResponse launchResponse =
-            dataflow
-                .projects()
-                .locations()
-                .flexTemplates()
-                .launch(
-                    projectId,
-                    jobRegion,
-                    new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
-                .execute();
-        logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
-        response.setStatus(SC_OK);
-        response.setPayload(
-            String.format("Launched RDE pipeline: %s", launchResponse.getJob().getId()));
-      } catch (IOException e) {
-        logger.atWarning().withCause(e).log("Pipeline Launch failed");
-        response.setStatus(SC_INTERNAL_SERVER_ERROR);
-        response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
-      }
+      ImmutableList.Builder<String> jobNameBuilder = new ImmutableList.Builder<>();
+      pendings.values().stream()
+          .collect(toImmutableSetMultimap(PendingDeposit::watermark, identity()))
+          .asMap()
+          .forEach(
+              (watermark, pendingDeposits) -> {
+                try {
+                  LaunchFlexTemplateParameter parameter =
+                      new LaunchFlexTemplateParameter()
+                          .setJobName(
+                              createJobName(
+                                  String.format(
+                                      "rde-%s", watermark.toString("yyyy-MM-dd't'HH-mm-ss'z'")),
+                                  clock))
+                          .setContainerSpecGcsPath(
+                              String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+                          .setParameters(
+                              new ImmutableMap.Builder<String, String>()
+                                  .put(
+                                      "pendings",
+                                      RdePipeline.encodePendingDeposits(
+                                          ImmutableSet.copyOf(pendingDeposits)))
+                                  .put("validationMode", validationMode.name())
+                                  .put("rdeStagingBucket", rdeBucket)
+                                  .put(
+                                      "stagingKey",
+                                      BaseEncoding.base64Url()
+                                          .omitPadding()
+                                          .encode(stagingKeyBytes))
+                                  .put("registryEnvironment", RegistryEnvironment.get().name())
+                                  .put("workerMachineType", machineType)
+                                  // TODO (jianglai): Investigate turning off public IPs (for which
+                                  // there is a quota) in order to increase the total number of
+                                  // workers allowed (also under quota).
+                                  // See:
+                                  // https://cloud.google.com/dataflow/docs/guides/routes-firewall
+                                  .put("usePublicIps", "true")
+                                  .build());
+                  LaunchFlexTemplateResponse launchResponse =
+                      dataflow
+                          .projects()
+                          .locations()
+                          .flexTemplates()
+                          .launch(
+                              projectId,
+                              jobRegion,
+                              new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
+                          .execute();
+                  logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
+                  jobNameBuilder.add(launchResponse.getJob().getId());
+                } catch (IOException e) {
+                  logger.atWarning().withCause(e).log("Pipeline Launch failed");
+                  response.setStatus(SC_INTERNAL_SERVER_ERROR);
+                  response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
+                }
+              });
+      response.setStatus(SC_OK);
+      response.setPayload(
+          String.format("Launched RDE pipeline: %s", Joiner.on(", ").join(jobNameBuilder.build())));
     }
   }
 
@@ -349,7 +405,7 @@ public final class RdeStagingAction implements Runnable {
       throw new BadRequestException("Directory must not start with a slash");
     }
     String directoryWithTrailingSlash =
-        directory.get().endsWith("/") ? directory.get() : (directory.get() + '/');
+        directory.get().endsWith("/") ? directory.get() : directory.get() + '/';
 
     if (modeStrings.isEmpty()) {
       throw new BadRequestException("Mode parameter required in manual operation");
@@ -381,7 +437,7 @@ public final class RdeStagingAction implements Runnable {
       }
     }
 
-    if (revision.isPresent() && (revision.get() < 0)) {
+    if (revision.isPresent() && revision.get() < 0) {
       throw new BadRequestException("Revision must be greater than or equal to zero");
     }
 
@@ -394,11 +450,7 @@ public final class RdeStagingAction implements Runnable {
           pendingsBuilder.put(
               tld,
               PendingDeposit.createInManualOperation(
-                  tld,
-                  watermark,
-                  mode,
-                  directoryWithTrailingSlash,
-                  revision.orElse(null)));
+                  tld, watermark, mode, directoryWithTrailingSlash, revision.orElse(null)));
         }
       }
     }

core/src/main/java/google/registry/reporting/billing/PublishInvoicesAction.java

@@ -23,16 +23,17 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 
 import com.google.api.services.dataflow.Dataflow;
 import com.google.api.services.dataflow.model.Job;
-import com.google.appengine.api.taskqueue.QueueFactory;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.reporting.ReportingModule;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
+import google.registry.util.CloudTasksUtils;
 import java.io.IOException;
 import javax.inject.Inject;
 import org.joda.time.YearMonth;
@@ -65,6 +66,7 @@ public class PublishInvoicesAction implements Runnable {
   private final Dataflow dataflow;
   private final Response response;
   private final YearMonth yearMonth;
+  private final CloudTasksUtils cloudTasksUtils;
 
   @Inject
   PublishInvoicesAction(
@@ -74,7 +76,8 @@ public class PublishInvoicesAction implements Runnable {
       BillingEmailUtils emailUtils,
       Dataflow dataflow,
       Response response,
-      YearMonth yearMonth) {
+      YearMonth yearMonth,
+      CloudTasksUtils cloudTasksUtils) {
     this.projectId = projectId;
     this.jobRegion = jobRegion;
     this.jobId = jobId;
@@ -82,6 +85,7 @@ public class PublishInvoicesAction implements Runnable {
     this.dataflow = dataflow;
     this.response = response;
     this.yearMonth = yearMonth;
+    this.cloudTasksUtils = cloudTasksUtils;
   }
 
   static final String PATH = "/_dr/task/publishInvoices";
@@ -119,10 +123,11 @@ public class PublishInvoicesAction implements Runnable {
   }
 
   private void enqueueCopyDetailReportsTask() {
-    TaskOptions copyDetailTask =
-        TaskOptions.Builder.withUrl(CopyDetailReportsAction.PATH)
-            .method(TaskOptions.Method.POST)
-            .param(PARAM_YEAR_MONTH, yearMonth.toString());
-    QueueFactory.getQueue(BillingModule.CRON_QUEUE).add(copyDetailTask);
+    cloudTasksUtils.enqueue(
+        BillingModule.CRON_QUEUE,
+        CloudTasksUtils.createPostTask(
+            CopyDetailReportsAction.PATH,
+            Service.BACKEND.toString(),
+            ImmutableMultimap.of(PARAM_YEAR_MONTH, yearMonth.toString())));
   }
 }