Remove converter for CreateAutoTimestamp (#1429)

We can handle it the same way that we handle UpdateAutoTimestamp, where
we simply populate it in SQL if it doesn't exist. This has the following
benefits:

1. The converter is unnecessary code
2. We get non-null column definitions for free (overridden in
EppResource to allow null creation times so that legacy *History objects
can contain null in that field
3. More importantly, this allows us for proper SQL->DS replay. If the
field is filled out using a converter (as before this PR) then the field
is only actually filled out on transaction commit (rather than when the
write occurs within the transaction). This means that when we serialize
the Transaction object during the transaction (the data that gets
replayed to Datastore), we are crucially missing the creation time.

If the creation time is written on commit, we have to start a new
transaction to write the Transaction object, and it's an absolute
necessity that the record of the transaction be included in the
transaction itself so as to avoid situations where the transaction
succeeds but the record fails.

If the field is filled out in a @PrePersist method, crucially that
occurs on the object write itself (before transaction commit).

build.gradle

@@ -200,11 +200,28 @@ rootProject.ext {
   pyver = { exe ->
     try {
       ext.execInBash(
-          exe + " -c 'import sys; print(sys.hexversion)'", "/") as Integer
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
     } catch (org.gradle.process.internal.ExecException e) {
       return -1;
     }
   }
+
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
+    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
+    // Debian so it seems like that should be available anywhere).
+    def MIN_PY_VER = 0x3070300
+    if (pyver('python') >= MIN_PY_VER) {
+      return 'python'
+    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
+      return '/usr/bin/python3'
+    } else {
+      throw new GradleException("No usable Python version found (build " +
+                                "requires at least python 3.7.3)");
+    }
+  }
 }
 
 task runPresubmits(type: Exec) {
@@ -212,18 +229,7 @@ task runPresubmits(type: Exec) {
   args('config/presubmits.py')
 
   doFirst {
-    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
-    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
-    // Debian so it seems like that should be available anywhere).
-    def MIN_PY_VER = 0x3070300
-    if (pyver('python') >= MIN_PY_VER) {
-      executable 'python'
-    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
-      executable '/usr/bin/python3'
-    } else {
-      throw new GradleException("No usable Python version found (build " +
-                                "requires at least python 3.7.3)");
-    }
+    executable getPythonExecutable()
   }
 }
 
@@ -438,9 +444,10 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
@@ -448,18 +455,23 @@ rootProject.ext {
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
-    def checkResult = invokeJavaDiffFormatScript("check")
-    if (checkResult == 'true') {
-      throw new IllegalStateException(
-          "Some Java files need to be reformatted. You may use the "
-              + "'javaIncrementalFormatDryRun' task to review\n "
-              + "the changes, or the 'javaIncrementalFormatApply' task "
-              + "to reformat.")
-    } else if (checkResult != 'false') {
-      throw new RuntimeException(
-          "Failed to invoke format check script:\n" + checkResult)
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
+      def checkResult = invokeJavaDiffFormatScript("check")
+      if (checkResult == 'true') {
+        throw new IllegalStateException(
+            "Some Java files need to be reformatted. You may use the "
+                + "'javaIncrementalFormatDryRun' task to review\n "
+                + "the changes, or the 'javaIncrementalFormatApply' task "
+                + "to reformat.")
+      } else if (checkResult != 'false') {
+        throw new RuntimeException(
+            "Failed to invoke format check script:\n" + checkResult)
+      }
+      println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
     }
-    println("Incremental Java format check ok.")
   }
 }
 
@@ -489,7 +501,8 @@ task javadoc(type: Javadoc) {
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
   options.tags = ["type:a:Generic Type",
-                  "error:a:Expected Error"]
+                  "error:a:Expected Error",
+                  "invariant:a:Guaranteed Property"]
 }
 
 tasks.build.dependsOn(tasks.javadoc)

config/presubmits.py

@@ -119,9 +119,10 @@ PRESUBMITS = {
       "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
+    # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
-        "java", {}):
+        "java", {"DatabaseSnapshotTest.java"}):
       "PostgreSQLContainer instantiation must specify docker tag.",
 
     # Various Soy linting checks
@@ -186,51 +187,6 @@ PRESUBMITS = {
         {"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
     ):
         "JavaScript files should not include console logging.",
-    # SQL injection protection rule for java source file:
-    # The sql template passed to createQuery/createNativeQuery methods must be
-    # a variable name in UPPER_CASE_UNDERSCORE format, i.e., a static final
-    # String variable. This forces the use of parameter-binding on all queries
-    # that take parameters.
-    # The rule would forbid invocation of createQuery(Criteria). However, this
-    # can be handled by adding a helper method in an exempted class to make
-    # the calls.
-    # TODO(b/179158393): enable the 'ConstantName' Java style check to ensure
-    # that non-final variables do not use the UPPER_CASE_UNDERSCORE format.
-    PresubmitCheck(
-        # Line 1: the method names we check and the opening parenthesis, which
-        #    marks the beginning of the first parameter
-        # Line 2: The first parameter is a match if is NOT any of the following:
-        #    - final variable name: \s*([A-Z_]+
-        #    - string literal: "([^"]|\\")*"
-        #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
-        # Line 3: , or the closing parenthesis, marking the end of the first
-        #    parameter
-        r'.*\.(query|createQuery|createNativeQuery)\('
-        r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
-        r'(,|\s*\))))',
-        "java",
-        # ActivityReportingQueryBuilder deals with Dremel queries
-        {"src/test", "ActivityReportingQueryBuilder.java",
-         # This class contains helper method to make queries in Beam.
-         "RegistryJpaIO.java",
-         "CreateSyntheticHistoryEntriesAction.java",
-         # TODO(b/179158393): Remove everything below, which should be done
-         # using Criteria
-         "JpaTransactionManager.java",
-         "JpaTransactionManagerImpl.java",
-         # CriteriaQueryBuilder is a false positive
-         "CriteriaQueryBuilder.java",
-         "RdapDomainSearchAction.java",
-         "RdapNameserverSearchAction.java",
-         "ReadOnlyCheckingEntityManager.java",
-         "RegistryQuery",
-         },
-    ):
-        "The first String parameter to EntityManager.create(Native)Query "
-        "methods must be one of the following:\n"
-        "  - A String literal\n"
-        "  - Concatenation of String literals only\n"
-        "  - The name of a static final String variable"
 }
 
 # Note that this regex only works for one kind of Flyway file.  If we want to

core/build.gradle

@@ -470,7 +470,7 @@ task soyToJava {
 
     outputs.each { file ->
       exec {
-        commandLine 'sed', '-i', 's/@link/LINK/g', file.getCanonicalPath()
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
       }
     }
   }

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -41,11 +41,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnly().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -14,21 +14,21 @@
 
 package google.registry.backup;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -56,7 +56,8 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Inject Clock clock;
   @Inject CommitLogCheckpointStrategy strategy;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject CommitLogCheckpointAction() {}
 
   @Override
@@ -73,16 +74,20 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveWithoutBackup()
+                  .saveIgnoringReadOnly()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.
-              taskQueueUtils.enqueue(
-                  getQueue(QUEUE_NAME),
-                  withUrl(ExportCommitLogDiffAction.PATH)
-                      .param(LOWER_CHECKPOINT_TIME_PARAM, lastWrittenTime.toString())
-                      .param(
-                          UPPER_CHECKPOINT_TIME_PARAM, checkpoint.getCheckpointTime().toString()));
+              cloudTasksUtils.enqueue(
+                  QUEUE_NAME,
+                  CloudTasksUtils.createPostTask(
+                      ExportCommitLogDiffAction.PATH,
+                      Service.BACKEND.toString(),
+                      ImmutableMultimap.of(
+                          LOWER_CHECKPOINT_TIME_PARAM,
+                          lastWrittenTime.toString(),
+                          UPPER_CHECKPOINT_TIME_PARAM,
+                          checkpoint.getCheckpointTime().toString())));
             });
   }
 }

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -112,7 +112,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
@@ -140,7 +140,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -113,9 +113,11 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   }
 
   /**
-   * Returns a list of registrars that should receive expiring notification emails. There are two
-   * certificates that should be considered (the main certificate and failOver certificate). The
-   * registrars should receive notifications if one of the certificate checks returns true.
+   * Returns a list of registrars that should receive expiring notification emails.
+   *
+   * <p>There are two certificates that should be considered (the main certificate and failOver
+   * certificate). The registrars should receive notifications if one of the certificate checks
+   * returns true.
    */
   @VisibleForTesting
   ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
@@ -157,15 +159,17 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
     }
     try {
       ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      ImmutableSet<InternetAddress> ccs = getEmailAddresses(registrar, Type.ADMIN);
       Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
       logger.atInfo().log(
-          "Registrar %s should receive an email that its %s SSL certificate will expire on %s.",
-          registrar.getRegistrarName(),
+          " %s SSL certificate of registrar '%s' will expire on %s.",
           certificateType.getDisplayName(),
+          registrar.getRegistrarName(),
           expirationDate.toString());
-      if (recipients.isEmpty()) {
+      if (recipients.isEmpty() && ccs.isEmpty()) {
         logger.atWarning().log(
-            "Registrar %s contains no email addresses to receive notification email.",
+            "Registrar %s contains no TECH nor ADMIN email addresses to receive notification"
+                + " email.",
             registrar.getRegistrarName());
         return false;
       }
@@ -180,7 +184,7 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
                       expirationDate,
                       registrar.getRegistrarId()))
               .setRecipients(recipients)
-              .setCcs(getEmailAddresses(registrar, Type.ADMIN))
+              .setCcs(ccs)
               .build());
       /*
        * A duration time offset is used here to ensure that date comparison between two
@@ -249,30 +253,32 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   /** Sends notification emails to registrars with expiring certificates. */
   @VisibleForTesting
   int sendNotificationEmails() {
-    int emailsSent = 0;
+    int numEmailsSent = 0;
     for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
       Registrar registrar = registrarInfo.registrar();
-      if (registrarInfo.isCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringCertNotificationSentDate(),
-            CertificateType.PRIMARY,
-            registrar.getClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringCertNotificationSentDate(),
+              CertificateType.PRIMARY,
+              registrar.getClientCertificate())) {
+        numEmailsSent++;
       }
-      if (registrarInfo.isFailOverCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringFailoverCertNotificationSentDate(),
-            CertificateType.FAILOVER,
-            registrar.getFailoverClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isFailOverCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringFailoverCertNotificationSentDate(),
+              CertificateType.FAILOVER,
+              registrar.getFailoverClientCertificate())) {
+        numEmailsSent++;
       }
     }
-    return emailsSent;
+    return numEmailsSent;
   }
 
-  /** Returns a list of email addresses of the registrar that should receive a notification email */
+  /**
+   * Returns a list of email addresses of the registrar that should receive a notification email.
+   */
   @VisibleForTesting
   ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
     ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -0,0 +1,88 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import java.util.List;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+
+/**
+ * A database snapshot shareable by concurrent queries from multiple database clients. A snapshot is
+ * uniquely identified by its {@link #getSnapshotId snapshotId}, and must stay open until all
+ * concurrent queries to this snapshot have attached to it by calling {@link
+ * google.registry.persistence.transaction.JpaTransactionManager#setDatabaseSnapshot}. However, it
+ * can be closed before those queries complete.
+ *
+ * <p>This feature is <em>Postgresql-only</em>.
+ *
+ * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to avoid
+ * exhausting predicate locks at the SERIALIZABLE level.
+ */
+// TODO(b/193662898): vendor-independent support for richer transaction semantics.
+public class DatabaseSnapshot implements AutoCloseable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private String snapshotId;
+  private EntityManager entityManager;
+  private EntityTransaction transaction;
+
+  private DatabaseSnapshot() {}
+
+  public String getSnapshotId() {
+    checkState(entityManager != null, "Snapshot not opened yet.");
+    checkState(entityManager.isOpen(), "Snapshot already closed.");
+    return snapshotId;
+  }
+
+  private DatabaseSnapshot open() {
+    entityManager = jpaTm().getStandaloneEntityManager();
+    transaction = entityManager.getTransaction();
+    transaction.setRollbackOnly();
+    transaction.begin();
+
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+
+    List<?> snapshotIds =
+        entityManager.createNativeQuery("SELECT pg_export_snapshot();").getResultList();
+    checkState(snapshotIds.size() == 1, "Unexpected number of snapshots: %s", snapshotIds.size());
+    snapshotId = (String) snapshotIds.get(0);
+    return this;
+  }
+
+  @Override
+  public void close() {
+    if (transaction != null && transaction.isActive()) {
+      try {
+        transaction.rollback();
+      } catch (Exception e) {
+        logger.atWarning().withCause(e).log("Failed to close a Database Snapshot");
+      }
+    }
+    if (entityManager != null && entityManager.isOpen()) {
+      entityManager.close();
+    }
+  }
+
+  public static DatabaseSnapshot createSnapshot() {
+    return new DatabaseSnapshot().open();
+  }
+}

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -138,6 +138,9 @@ public final class RegistryJpaIO {
 
     abstract Coder<T> coder();
 
+    @Nullable
+    abstract String snapshotId();
+
     abstract Builder<R, T> toBuilder();
 
     @Override
@@ -145,7 +148,9 @@ public final class RegistryJpaIO {
     public PCollection<T> expand(PBegin input) {
       return input
           .apply("Starting " + name(), Create.of((Void) null))
-          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
+          .apply(
+              "Run query for " + name(),
+              ParDo.of(new QueryRunner<>(query(), resultMapper(), snapshotId())))
           .setCoder(coder())
           .apply("Reshuffle", Reshuffle.viaRandomKey());
     }
@@ -162,6 +167,18 @@ public final class RegistryJpaIO {
       return toBuilder().coder(coder).build();
     }
 
+    /**
+     * Specifies the database snapshot to use for this query.
+     *
+     * <p>This feature is <em>Postgresql-only</em>. User is responsible for keeping the snapshot
+     * available until all JVM workers have started using it by calling {@link
+     * JpaTransactionManager#setDatabaseSnapshot}.
+     */
+    // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+    public Read<R, T> withSnapshot(String snapshotId) {
+      return toBuilder().snapshotId(snapshotId).build();
+    }
+
     static <R, T> Builder<R, T> builder() {
       return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
           .name(DEFAULT_NAME)
@@ -179,6 +196,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<R, T> coder(Coder coder);
 
+      abstract Builder<R, T> snapshotId(@Nullable String sharedSnapshotId);
+
       abstract Read<R, T> build();
 
       Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
@@ -201,17 +220,28 @@ public final class RegistryJpaIO {
     static class QueryRunner<R, T> extends DoFn<Void, T> {
       private final RegistryQuery<R> query;
       private final SerializableFunction<R, T> resultMapper;
+      // java.util.Optional is not serializable. Use of Guava Optional is discouraged.
+      @Nullable private final String snapshotId;
 
-      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+      QueryRunner(
+          RegistryQuery<R> query,
+          SerializableFunction<R, T> resultMapper,
+          @Nullable String snapshotId) {
         this.query = query;
         this.resultMapper = resultMapper;
+        this.snapshotId = snapshotId;
       }
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
         jpaTm()
             .transactNoRetry(
-                () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
+                () -> {
+                  if (snapshotId != null) {
+                    jpaTm().setDatabaseSnapshot(snapshotId);
+                  }
+                  query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
+                });
       }
     }
   }

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -21,6 +21,7 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -45,9 +46,19 @@ public interface RegistryPipelineComponent {
   @Config("projectId")
   String getProjectId();
 
+  /** Returns the regular {@link JpaTransactionManager} for general use. */
   @BeamJpaTm
   Lazy<JpaTransactionManager> getJpaTransactionManager();
 
+  /**
+   * Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
+   * ({@link google.registry.model.domain.DomainBase} and {@link
+   * google.registry.model.domain.DomainHistory}). Please refer to {@link
+   * google.registry.model.bulkquery.BulkQueryEntities} for more information.
+   */
+  @BeamBulkQueryJpaTm
+  Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -44,6 +45,12 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setIsolationOverride(TransactionIsolationLevel isolationOverride);
 
+  @Description("The JPA Transaction Manager to use.")
+  @Default.Enum(value = "REGULAR")
+  JpaTransactionManagerType getJpaTransactionManagerType();
+
+  void setJpaTransactionManagerType(JpaTransactionManagerType jpaTransactionManagerType);
+
   @Description("The number of entities to write to the SQL database in one operation.")
   @Default.Integer(20)
   int getSqlWriteBatchSize();

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -49,10 +49,19 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
     }
     logger.atInfo().log("Setting up RegistryEnvironment %s.", environment);
     environment.setup();
-    Lazy<JpaTransactionManager> transactionManagerLazy =
-        toRegistryPipelineComponent(registryOptions).getJpaTransactionManager();
+    RegistryPipelineComponent registryPipelineComponent =
+        toRegistryPipelineComponent(registryOptions);
+    Lazy<JpaTransactionManager> transactionManagerLazy;
+    switch (registryOptions.getJpaTransactionManagerType()) {
+      case BULK_QUERY:
+        transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
+        break;
+      case REGULAR:
+      default:
+        transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
+    }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
-    // Masquarade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
     // loads all ofy entities.
     new AppEngineEnvironment("Beam").setEnvironmentForAllThreads();
     // Set the system property so that we can call IdService.allocateId() without access to

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -262,8 +262,8 @@ public final class Transforms {
 
   // Production data repair configs go below. See b/185954992.
 
-  // Prober domains in bad state, without associated contacts, hosts, billings, and history.
-  // They can be safely ignored.
+  // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized
+  // history. They can be safely ignored.
   private static final ImmutableSet<String> IGNORED_DOMAINS =
       ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT");
 
@@ -299,7 +299,7 @@ public final class Transforms {
       return !IGNORED_HOSTS.contains(roid);
     }
     if (entity.getKind().equals("HistoryEntry")) {
-      // Remove production bad data: History of the contacts to be ignored:
+      // Remove production bad data: Histories of ignored EPP resources:
       com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent();
       if (parentKey.getKind().equals("ContactResource")) {
         String contactRoid = parentKey.getName();
@@ -309,6 +309,10 @@ public final class Transforms {
         String hostRoid = parentKey.getName();
         return !IGNORED_HOSTS.contains(hostRoid);
       }
+      if (parentKey.getKind().equals("DomainBase")) {
+        String domainRoid = parentKey.getName();
+        return !IGNORED_DOMAINS.contains(domainRoid);
+      }
     }
     // End of production-specific checks.
 

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -57,7 +57,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=invoicing}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -75,6 +75,8 @@ public class RdeIO {
   abstract static class Write
       extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
 
+    private static final long serialVersionUID = 3334807737227087760L;
+
     abstract GcsUtils gcsUtils();
 
     abstract CloudTasksUtils cloudTasksUtils();
@@ -113,8 +115,9 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()))
-          .apply("Enqueue upload action", ParDo.of(new UploadEnqueuer(cloudTasksUtils())));
+          .apply(
+              "Update cursor and enqueue next action",
+              ParDo.of(new CursorUpdater(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -123,6 +126,7 @@ public class RdeIO {
       extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5496375923068400382L;
 
     private final GcsUtils gcsUtils;
     private final String rdeBucket;
@@ -169,7 +173,7 @@ public class RdeIO {
         checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
         prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
       } else {
-        prefix = prefix + "/" + basename;
+        prefix = prefix + '/' + basename;
       }
       BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
       // This file will contain the byte length (ASCII) of the raw unencrypted XML.
@@ -250,12 +254,20 @@ public class RdeIO {
     }
   }
 
-  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, PendingDeposit> {
+  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5822176227753327224L;
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private CursorUpdater(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
 
     @ProcessElement
     public void processElement(
-        @Element KV<PendingDeposit, Integer> input, OutputReceiver<PendingDeposit> outputReceiver) {
+        @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -282,46 +294,32 @@ public class RdeIO {
                 logger.atInfo().log(
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                if (key.mode() == RdeMode.FULL) {
+                  cloudTasksUtils.enqueue(
+                      RDE_UPLOAD_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          RdeUploadAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                } else {
+                  cloudTasksUtils.enqueue(
+                      BRDA_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          BrdaCopyAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_WATERMARK,
+                              key.watermark().toString(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                }
               });
-      outputReceiver.output(input.getKey());
-    }
-  }
-
-  private static class UploadEnqueuer extends DoFn<PendingDeposit, Void> {
-
-    private final CloudTasksUtils cloudTasksUtils;
-
-    private UploadEnqueuer(CloudTasksUtils cloudTasksUtils) {
-      this.cloudTasksUtils = cloudTasksUtils;
-    }
-
-    @ProcessElement
-    public void processElement(@Element PendingDeposit input, PipelineOptions options) {
-      if (input.mode() == RdeMode.FULL) {
-        cloudTasksUtils.enqueue(
-            RDE_UPLOAD_QUEUE,
-            CloudTasksUtils.createPostTask(
-                RdeUploadAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      } else {
-        cloudTasksUtils.enqueue(
-            BRDA_QUEUE,
-            CloudTasksUtils.createPostTask(
-                BrdaCopyAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_WATERMARK,
-                    input.watermark().toString(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      }
     }
   }
 }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -14,36 +14,50 @@
 
 package google.registry.beam.rde;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.beam.rde.RdePipeline.TupleTags.DOMAIN_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.EXTERNAL_HOST_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.HOST_TO_PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_CONTACTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
+import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
+import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
 import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
-import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.UtilsModule;
@@ -54,74 +68,158 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.persistence.Entity;
+import javax.persistence.IdClass;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.DateTime;
 
 /**
  * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=rde}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
+ * <p>This pipeline only works for pending deposits with the same watermark, the {@link
+ * google.registry.rde.RdeStagingAction} will batch such pending deposits together and launch
+ * multiple pipelines if multiple watermarks exist.
+ *
+ * <p>The pipeline is broadly divided into two parts -- creating the {@link DepositFragment}s, and
+ * processing them.
+ *
+ * <h1>Creating {@link DepositFragment}</h1>
+ *
+ * <h2>{@link Registrar}</h2>
+ *
+ * Non-test registrar entities are loaded from Cloud SQL and marshalled into deposit fragments. They
+ * are <b>NOT</b> rewound to the watermark.
+ *
+ * <h2>{@link EppResource}</h2>
+ *
+ * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
+ * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
+ * revision ID) from the SQL query.
+ *
+ * <h3>{@link DomainBase}</h3>
+ *
+ * After the most recent (live) domain resources are loaded from the corresponding history objects,
+ * we marshall them to deposit fragments and emit the (pending deposit: deposit fragment) pairs for
+ * further processing. We also find all the contacts and hosts referenced by a given domain and emit
+ * pairs of (contact/host repo ID: pending deposit) for all RDE pending deposits for further
+ * processing.
+ *
+ * <h3>{@link ContactResource}</h3>
+ *
+ * We first join most recent contact histories, represented by (contact repo ID: contact history
+ * revision ID) pairs, with referenced contacts, represented by (contact repo ID: pending deposit)
+ * pairs, on the contact repo ID, to remove unreferenced contact histories. Contact resources are
+ * then loaded from the remaining referenced contact histories, and marshalled into (pending
+ * deposit: deposit fragment) pairs.
+ *
+ * <h3>{@link HostResource}</h3>
+ *
+ * Similar to {@link ContactResource}, we join the most recent host history with referenced hosts to
+ * find most recent referenced hosts. For external hosts we do the same treatment as we did on
+ * contacts and obtain the (pending deposit: deposit fragment) pairs. For subordinate hosts, we need
+ * to find the superordinate domain in order to properly handle pending transfer in the deposit as
+ * well. So we first find the superordinate domain repo ID from the host and join the (superordinate
+ * domain repo ID: (subordinate host repo ID: (pending deposit: revision ID))) pair with the (domain
+ * repo ID: revision ID) pair obtained from the domain history query in order to map the host at
+ * watermark to the domain at watermark. We then proceed to create the (pending deposit: deposit
+ * fragment) pair for subordinate hosts using the added domain information.
+ *
+ * <h1>Processing {@link DepositFragment}</h1>
+ *
+ * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
+ * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
+ * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
+ * pending deposit is then rolled forward, and the next action is enqueued. The latter two
+ * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
+ *
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
 @Singleton
 public class RdePipeline implements Serializable {
 
+  private static final long serialVersionUID = -4866795928854754666L;
   private final transient RdePipelineOptions options;
   private final ValidationMode mode;
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final ImmutableSet<PendingDeposit> pendingDeposits;
+  private final DateTime watermark;
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
   private final CloudTasksUtils cloudTasksUtils;
+  private final RdeMarshaller marshaller;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
   private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
       Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
 
-  private static final String EPP_RESOURCE_QUERY =
-      "SELECT id FROM %entity% "
-          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
-
-  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
-    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
-        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
-  }
+  // The field name of the EPP resource embedded in its corresponding history entry.
+  private static final ImmutableMap<Class<? extends HistoryEntry>, String> EPP_RESOURCE_FIELD_NAME =
+      ImmutableMap.of(
+          DomainHistory.class,
+          "domainContent",
+          ContactHistory.class,
+          "contactBase",
+          HostHistory.class,
+          "hostBase");
 
   @Inject
   RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
-    this.pendings = decodePendings(options.getPendings());
+    this.pendingDeposits = decodePendingDeposits(options.getPendings());
+    ImmutableSet<DateTime> potentialWatermarks =
+        pendingDeposits.stream()
+            .map(PendingDeposit::watermark)
+            .distinct()
+            .collect(toImmutableSet());
+    checkArgument(
+        potentialWatermarks.size() == 1,
+        String.format(
+            "RDE pipeline should only work on pending deposits "
+                + "with the same watermark, but %d were given: %s",
+            potentialWatermarks.size(), potentialWatermarks));
+    this.watermark = potentialWatermarks.asList().get(0);
     this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
     this.cloudTasksUtils = cloudTasksUtils;
+    this.marshaller = new RdeMarshaller(mode);
   }
 
   PipelineResult run() {
@@ -133,13 +231,46 @@ public class RdePipeline implements Serializable {
   }
 
   PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
-    return PCollectionList.of(processRegistrars(pipeline))
-        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
-        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
-        .and(processNonRegistrarEntities(pipeline, HostResource.class))
-        .apply(Flatten.pCollections())
+    PCollection<KV<PendingDeposit, DepositFragment>> registrarFragments =
+        processRegistrars(pipeline);
+
+    PCollection<KV<String, Long>> domainHistories =
+        getMostRecentHistoryEntries(pipeline, DomainHistory.class);
+
+    PCollection<KV<String, Long>> contactHistories =
+        getMostRecentHistoryEntries(pipeline, ContactHistory.class);
+
+    PCollection<KV<String, Long>> hostHistories =
+        getMostRecentHistoryEntries(pipeline, HostHistory.class);
+
+    PCollectionTuple processedDomainHistories = processDomainHistories(domainHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> domainFragments =
+        processedDomainHistories.get(DOMAIN_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> contactFragments =
+        processContactHistories(
+            processedDomainHistories.get(REFERENCED_CONTACTS), contactHistories);
+
+    PCollectionTuple processedHosts =
+        processHostHistories(processedDomainHistories.get(REFERENCED_HOSTS), hostHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> externalHostFragments =
+        processedHosts.get(EXTERNAL_HOST_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> subordinateHostFragments =
+        processSubordinateHosts(processedHosts.get(SUPERORDINATE_DOMAINS), domainHistories);
+
+    return PCollectionList.of(registrarFragments)
+        .and(domainFragments)
+        .and(contactFragments)
+        .and(externalHostFragments)
+        .and(subordinateHostFragments)
+        .apply(
+            "Combine PendingDeposit:DepositFragment pairs from all entities",
+            Flatten.pCollections())
         .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply("Group by PendingDeposit", GroupByKey.create());
+        .apply("Group DepositFragment by PendingDeposit", GroupByKey.create());
   }
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
@@ -154,127 +285,398 @@ public class RdePipeline implements Serializable {
             .build());
   }
 
-  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+  private PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    // Note that the namespace in the metric is not being used by Stackdriver, it just has to be
+    // non-empty.
+    // See:
+    // https://stackoverflow.com/questions/48530496/google-dataflow-custom-metrics-not-showing-on-stackdriver
+    Counter includedRegistrarCounter = Metrics.counter("RDE", "IncludedRegistrar");
+    Counter registrarFragmentCounter = Metrics.counter("RDE", "RegistrarFragment");
     return pipeline
         .apply(
-            "Read all production Registrar entities",
+            "Read all production Registrars",
             RegistryJpaIO.read(
                 "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
                 ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
                 String.class,
-                // TODO: consider adding coders for entities and pass them directly instead of using
-                // VKeys.
                 id -> VKey.createSql(Registrar.class, id)))
         .apply(
-            "Marshal Registrar into DepositFragment",
+            "Marshall Registrar into DepositFragment",
             FlatMapElements.into(
-                    TypeDescriptors.kvs(
+                    kvs(
                         TypeDescriptor.of(PendingDeposit.class),
                         TypeDescriptor.of(DepositFragment.class)))
                 .via(
                     (VKey<Registrar> key) -> {
+                      includedRegistrarCounter.inc();
                       Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
-                      DepositFragment fragment =
-                          new RdeMarshaller(mode).marshalRegistrar(registrar);
-                      return pendings.values().stream()
-                          .map(pending -> KV.of(pending, fragment))
-                          .collect(toImmutableSet());
+                      DepositFragment fragment = marshaller.marshalRegistrar(registrar);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          pendingDeposits.stream()
+                              .map(pending -> KV.of(pending, fragment))
+                              .collect(toImmutableSet());
+                      registrarFragmentCounter.inc(fragments.size());
+                      return fragments;
                     }));
   }
 
-  <T extends EppResource>
-      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
-          Pipeline pipeline, Class<T> clazz) {
-    return createInputs(pipeline, clazz)
-        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(
-            KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)));
+  /**
+   * Load the most recent history entry before the watermark for a given history entry type.
+   *
+   * <p>Note that deleted and non-production resources are not included.
+   *
+   * @return A KV pair of (repoId, revisionId), used to reconstruct the composite key for the
+   *     history entry.
+   */
+  private <T extends HistoryEntry> PCollection<KV<String, Long>> getMostRecentHistoryEntries(
+      Pipeline pipeline, Class<T> historyClass) {
+    String repoIdFieldName = HistoryEntryDao.REPO_ID_FIELD_NAMES.get(historyClass);
+    String resourceFieldName = EPP_RESOURCE_FIELD_NAME.get(historyClass);
+    return pipeline
+        .apply(
+            String.format("Load most recent %s", historyClass.getSimpleName()),
+            RegistryJpaIO.read(
+                ("SELECT %repoIdField%, id FROM %entity% WHERE (%repoIdField%, modificationTime)"
+                     + " IN (SELECT %repoIdField%, MAX(modificationTime) FROM %entity% WHERE"
+                     + " modificationTime <= :watermark GROUP BY %repoIdField%) AND"
+                     + " %resourceField%.deletionTime > :watermark AND"
+                     + " COALESCE(%resourceField%.creationClientId, '') NOT LIKE 'prober-%' AND"
+                     + " COALESCE(%resourceField%.currentSponsorClientId, '') NOT LIKE 'prober-%'"
+                     + " AND COALESCE(%resourceField%.lastEppUpdateClientId, '') NOT LIKE"
+                     + " 'prober-%' "
+                        + (historyClass == DomainHistory.class
+                            ? "AND %resourceField%.tld IN "
+                                + "(SELECT id FROM Tld WHERE tldType = 'REAL')"
+                            : ""))
+                    .replace("%entity%", historyClass.getSimpleName())
+                    .replace("%repoIdField%", repoIdFieldName)
+                    .replace("%resourceField%", resourceFieldName),
+                ImmutableMap.of("watermark", watermark),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1])))
+        .setCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));
   }
 
-  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
-    return pipeline.apply(
-        "Read all production " + clazz.getSimpleName() + " entities",
-        RegistryJpaIO.read(
-            createEppResourceQuery(clazz),
-            clazz.equals(DomainBase.class)
-                ? ImmutableMap.of("tlds", pendings.keySet())
-                : ImmutableMap.of(),
-            String.class,
-            // TODO: consider adding coders for entities and pass them directly instead of using
-            // VKeys.
-            x -> VKey.createSql(clazz, x)));
+  private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
+      Class<T> historyEntryClazz, String repoId, long revisionId) {
+    try {
+      Class<?> idClazz = historyEntryClazz.getAnnotation(IdClass.class).value();
+      Serializable idObject =
+          (Serializable)
+              idClazz.getConstructor(String.class, long.class).newInstance(repoId, revisionId);
+      return jpaTm()
+          .transact(() -> jpaTm().loadByKey(VKey.createSql(historyEntryClazz, idObject)))
+          .getResourceAtPointInTime()
+          .map(resource -> resource.cloneProjectedAtTime(watermark))
+          .get();
+    } catch (NoSuchMethodException
+        | InvocationTargetException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot load resource from %s with repoId %s and revisionId %s",
+              historyEntryClazz.getSimpleName(), repoId, revisionId),
+          e);
+    }
   }
 
-  <T extends EppResource>
-      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
-    return FlatMapElements.into(
-            TypeDescriptors.kvs(
-                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
-        .via(
-            (VKey<T> key) -> {
-              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
-              // The set of all TLDs to which this resource should be emitted.
-              ImmutableSet<String> tlds =
-                  clazz.equals(DomainBase.class)
-                      ? ImmutableSet.of(((DomainBase) resource).getTld())
-                      : pendings.keySet();
-              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
-              ImmutableSet<DateTime> dates =
-                  tlds.stream()
-                      .map(pendings::get)
-                      .flatMap(ImmutableSet::stream)
-                      .map(PendingDeposit::watermark)
-                      .collect(toImmutableSet());
-              // Launch asynchronous fetches of point-in-time representations of resource.
-              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
-                  ImmutableMap.copyOf(
-                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
-              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
-              // the result.
-              RdeFragmenter fragmenter =
-                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
-              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
-              for (String tld : tlds) {
-                for (PendingDeposit pending : pendings.get(tld)) {
-                  // Hosts and contacts don't get included in BRDA deposits.
-                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
-                    continue;
+  /**
+   * Remove unreferenced resources by joining the (repoId, pendingDeposit) pair with the (repoId,
+   * revisionId) on the repoId.
+   *
+   * <p>The (repoId, pendingDeposit) pairs denote resources (contact, host) that are referenced from
+   * a domain, that are to be included in the corresponding pending deposit.
+   *
+   * <p>The (repoId, revisionId) paris come from the most recent history entry query, which can be
+   * used to load the embedded resources themselves.
+   *
+   * @return a pair of (repoId, ([pendingDeposit], [revisionId])) where neither the pendingDeposit
+   *     nor the revisionId list is empty.
+   */
+  private static PCollection<KV<String, CoGbkResult>> removeUnreferencedResource(
+      PCollection<KV<String, PendingDeposit>> referencedResources,
+      PCollection<KV<String, Long>> historyEntries,
+      Class<? extends EppResource> resourceClazz) {
+    String resourceName = resourceClazz.getSimpleName();
+    Class<? extends HistoryEntry> historyEntryClazz =
+        RESOURCE_TYPES_TO_HISTORY_TYPES.get(resourceClazz);
+    String historyEntryName = historyEntryClazz.getSimpleName();
+    Counter referencedResourceCounter = Metrics.counter("RDE", "Referenced" + resourceName);
+    return KeyedPCollectionTuple.of(PENDING_DEPOSIT, referencedResources)
+        .and(REVISION_ID, historyEntries)
+        .apply(
+            String.format(
+                "Join PendingDeposit with %s revision ID on %s", historyEntryName, resourceName),
+            CoGroupByKey.create())
+        .apply(
+            String.format("Remove unreferenced %s", resourceName),
+            Filter.by(
+                (KV<String, CoGbkResult> kv) -> {
+                  boolean toInclude =
+                      // If a resource does not have corresponding pending deposit, it is not
+                      // referenced and should not be included.
+                      kv.getValue().getAll(PENDING_DEPOSIT).iterator().hasNext()
+                          // If a resource does not have revision id (this should not happen, as
+                          // every referenced resource must be valid at watermark time, therefore
+                          // be embedded in a history entry valid at watermark time, otherwise
+                          // the domain cannot reference it), there is no way for us to find the
+                          // history entry and load the embedded resource. So we ignore the resource
+                          // to keep the downstream process simple.
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedResourceCounter.inc();
                   }
-                  Optional<DepositFragment> fragment =
-                      fragmenter.marshal(pending.watermark(), pending.mode());
-                  fragment.ifPresent(
-                      depositFragment -> results.add(KV.of(pending, depositFragment)));
-                }
-              }
-              return results;
-            });
+                  return toInclude;
+                }));
+  }
+
+  private PCollectionTuple processDomainHistories(PCollection<KV<String, Long>> domainHistories) {
+    Counter activeDomainCounter = Metrics.counter("RDE", "ActiveDomainBase");
+    Counter domainFragmentCounter = Metrics.counter("RDE", "DomainFragment");
+    Counter referencedContactCounter = Metrics.counter("RDE", "ReferencedContactResource");
+    Counter referencedHostCounter = Metrics.counter("RDE", "ReferencedHostResource");
+    return domainHistories.apply(
+        "Map DomainHistory to DepositFragment "
+            + "and emit referenced ContactResource and HostResource",
+        ParDo.of(
+                new DoFn<KV<String, Long>, KV<PendingDeposit, DepositFragment>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Long> kv, MultiOutputReceiver receiver) {
+                    activeDomainCounter.inc();
+                    DomainBase domain =
+                        (DomainBase)
+                            loadResourceByHistoryEntryId(
+                                DomainHistory.class, kv.getKey(), kv.getValue());
+                    pendingDeposits.stream()
+                        .filter(pendingDeposit -> pendingDeposit.tld().equals(domain.getTld()))
+                        .forEach(
+                            pendingDeposit -> {
+                              // Domains are always deposited in both modes.
+                              domainFragmentCounter.inc();
+                              receiver
+                                  .get(DOMAIN_FRAGMENTS)
+                                  .output(
+                                      KV.of(
+                                          pendingDeposit,
+                                          marshaller.marshalDomain(domain, pendingDeposit.mode())));
+                              // Contacts and hosts are only deposited in RDE, not BRDA.
+                              if (pendingDeposit.mode() == RdeMode.FULL) {
+                                HashSet<Serializable> contacts = new HashSet<>();
+                                contacts.add(domain.getAdminContact().getSqlKey());
+                                contacts.add(domain.getTechContact().getSqlKey());
+                                contacts.add(domain.getRegistrant().getSqlKey());
+                                // Billing contact is not mandatory.
+                                if (domain.getBillingContact() != null) {
+                                  contacts.add(domain.getBillingContact().getSqlKey());
+                                }
+                                referencedContactCounter.inc(contacts.size());
+                                contacts.forEach(
+                                    contactRepoId ->
+                                        receiver
+                                            .get(REFERENCED_CONTACTS)
+                                            .output(KV.of((String) contactRepoId, pendingDeposit)));
+                                if (domain.getNsHosts() != null) {
+                                  referencedHostCounter.inc(domain.getNsHosts().size());
+                                  domain
+                                      .getNsHosts()
+                                      .forEach(
+                                          hostKey ->
+                                              receiver
+                                                  .get(REFERENCED_HOSTS)
+                                                  .output(
+                                                      KV.of(
+                                                          (String) hostKey.getSqlKey(),
+                                                          pendingDeposit)));
+                                }
+                              }
+                            });
+                  }
+                })
+            .withOutputTags(
+                DOMAIN_FRAGMENTS, TupleTagList.of(REFERENCED_CONTACTS).and(REFERENCED_HOSTS)));
+  }
+
+  private PCollection<KV<PendingDeposit, DepositFragment>> processContactHistories(
+      PCollection<KV<String, PendingDeposit>> referencedContacts,
+      PCollection<KV<String, Long>> contactHistories) {
+    Counter contactFragmentCounter = Metrics.counter("RDE", "ContactFragment");
+    return removeUnreferencedResource(referencedContacts, contactHistories, ContactResource.class)
+        .apply(
+            "Map ContactResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      ContactResource contact =
+                          (ContactResource)
+                              loadResourceByHistoryEntryId(
+                                  ContactHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      DepositFragment fragment = marshaller.marshalContact(contact);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same contact could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .map(pendingDeposit -> KV.of(pendingDeposit, fragment))
+                              .collect(toImmutableSet());
+                      contactFragmentCounter.inc(fragments.size());
+                      return fragments;
+                    }));
+  }
+
+  private PCollectionTuple processHostHistories(
+      PCollection<KV<String, PendingDeposit>> referencedHosts,
+      PCollection<KV<String, Long>> hostHistories) {
+    Counter subordinateHostCounter = Metrics.counter("RDE", "SubordinateHostResource");
+    Counter externalHostCounter = Metrics.counter("RDE", "ExternalHostResource");
+    Counter externalHostFragmentCounter = Metrics.counter("RDE", "ExternalHostFragment");
+    return removeUnreferencedResource(referencedHosts, hostHistories, HostResource.class)
+        .apply(
+            "Map external DomainResource to DepositFragment and process subordinate domains",
+            ParDo.of(
+                    new DoFn<KV<String, CoGbkResult>, KV<PendingDeposit, DepositFragment>>() {
+                      @ProcessElement
+                      public void processElement(
+                          @Element KV<String, CoGbkResult> kv, MultiOutputReceiver receiver) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    kv.getKey(),
+                                    kv.getValue().getOnly(REVISION_ID));
+                        // When a host is subordinate, we need to find it's superordinate domain and
+                        // include it in the deposit as well.
+                        if (host.isSubordinate()) {
+                          subordinateHostCounter.inc();
+                          receiver
+                              .get(SUPERORDINATE_DOMAINS)
+                              .output(
+                                  // The output are pairs of
+                                  // (superordinateDomainRepoId,
+                                  //   (subordinateHostRepoId, (pendingDeposit, revisionId))).
+                                  KV.of((String) host.getSuperordinateDomain().getSqlKey(), kv));
+                        } else {
+                          externalHostCounter.inc();
+                          DepositFragment fragment = marshaller.marshalExternalHost(host);
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same host could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .forEach(
+                                  pendingDeposit -> {
+                                    externalHostFragmentCounter.inc();
+                                    receiver
+                                        .get(EXTERNAL_HOST_FRAGMENTS)
+                                        .output(KV.of(pendingDeposit, fragment));
+                                  });
+                        }
+                      }
+                    })
+                .withOutputTags(EXTERNAL_HOST_FRAGMENTS, TupleTagList.of(SUPERORDINATE_DOMAINS)));
+  }
+
+  /**
+   * Process subordinate hosts by making a deposit fragment with pending transfer information
+   * obtained from its superordinate domain.
+   *
+   * @param superordinateDomains Pairs of (superordinateDomainRepoId, (subordinateHostRepoId,
+   *     (pendingDeposit, revisionId))). This collection maps the subordinate host and the pending
+   *     deposit to include it to its superordinate domain.
+   * @param domainHistories Pairs of (domainRepoId, revisionId). This collection helps us find the
+   *     historical superordinate domain from its history entry and is obtained from calling {@link
+   *     #getMostRecentHistoryEntries} for domains.
+   */
+  private PCollection<KV<PendingDeposit, DepositFragment>> processSubordinateHosts(
+      PCollection<KV<String, KV<String, CoGbkResult>>> superordinateDomains,
+      PCollection<KV<String, Long>> domainHistories) {
+    Counter subordinateHostFragmentCounter = Metrics.counter("RDE", "SubordinateHostFragment");
+    Counter referencedSubordinateHostCounter = Metrics.counter("RDE", "ReferencedSubordinateHost");
+    return KeyedPCollectionTuple.of(HOST_TO_PENDING_DEPOSIT, superordinateDomains)
+        .and(REVISION_ID, domainHistories)
+        .apply(
+            "Join HostResource:PendingDeposits with DomainHistory on DomainResource",
+            CoGroupByKey.create())
+        .apply(
+            " Remove unreferenced DomainResource",
+            Filter.by(
+                kv -> {
+                  boolean toInclude =
+                      kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT).iterator().hasNext()
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedSubordinateHostCounter.inc();
+                  }
+                  return toInclude;
+                }))
+        .apply(
+            "Map subordinate HostResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      DomainBase superordinateDomain =
+                          (DomainBase)
+                              loadResourceByHistoryEntryId(
+                                  DomainHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      ImmutableSet.Builder<KV<PendingDeposit, DepositFragment>> results =
+                          new ImmutableSet.Builder<>();
+                      for (KV<String, CoGbkResult> hostToPendingDeposits :
+                          kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT)) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    hostToPendingDeposits.getKey(),
+                                    hostToPendingDeposits.getValue().getOnly(REVISION_ID));
+                        DepositFragment fragment =
+                            marshaller.marshalSubordinateHost(host, superordinateDomain);
+                        Streams.stream(hostToPendingDeposits.getValue().getAll(PENDING_DEPOSIT))
+                            .distinct()
+                            .forEach(
+                                pendingDeposit -> {
+                                  subordinateHostFragmentCounter.inc();
+                                  results.add(KV.of(pendingDeposit, fragment));
+                                });
+                      }
+                      return results.build();
+                    }));
   }
 
   /**
    * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
-   * the original TLD to pending deposit map.
+   * the original pending deposit set.
    */
   @SuppressWarnings("unchecked")
-  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+  static ImmutableSet<PendingDeposit> decodePendingDeposits(String encodedPendingDeposits) {
     try (ObjectInputStream ois =
         new ObjectInputStream(
             new ByteArrayInputStream(
-                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
-      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+                BaseEncoding.base64Url().omitPadding().decode(encodedPendingDeposits)))) {
+      return (ImmutableSet<PendingDeposit>) ois.readObject();
     } catch (IOException | ClassNotFoundException e) {
       throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
     }
   }
 
   /**
-   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
-   * worker by the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
+   * the pipeline launcher as a pipeline option.
    */
-  public static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
-      oos.writeObject(pendings);
+      oos.writeObject(pendingDeposits);
       oos.flush();
       return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
     }
@@ -284,14 +686,40 @@ public class RdePipeline implements Serializable {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
     RdePipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
-    // RegistryPipelineWorkerInitializer only initializes before pipeline executions, after the
-    // main() function constructed the graph. We need the registry environment set up so that we
-    // can create a CloudTasksUtils which uses the environment-dependent config file.
-    options.getRegistryEnvironment().setup();
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
+  /**
+   * A utility class that contains {@link TupleTag}s when {@link PCollectionTuple}s and {@link
+   * CoGbkResult}s are used.
+   */
+  protected abstract static class TupleTags {
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> DOMAIN_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_CONTACTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_HOSTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, KV<String, CoGbkResult>>> SUPERORDINATE_DOMAINS =
+        new TupleTag<KV<String, KV<String, CoGbkResult>>>() {};
+
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> EXTERNAL_HOST_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<PendingDeposit> PENDING_DEPOSIT =
+        new TupleTag<PendingDeposit>() {};
+
+    protected static final TupleTag<KV<String, CoGbkResult>> HOST_TO_PENDING_DEPOSIT =
+        new TupleTag<KV<String, CoGbkResult>>() {};
+
+    protected static final TupleTag<Long> REVISION_ID = new TupleTag<Long>() {};
+  }
+
   @Singleton
   @Component(
       modules = {

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -61,7 +61,8 @@ import org.json.JSONObject;
 /**
  * Definition of a Dataflow Flex template, which generates a given month's spec11 report.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=spec11}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -557,11 +557,23 @@ public final class RegistryConfig {
       return config.registryPolicy.requireSslCertificates;
     }
 
+    /**
+     * Returns the GCE machine type that a CPU-demanding pipeline should use.
+     *
+     * @see google.registry.beam.rde.RdePipeline
+     */
+    @Provides
+    @Config("highPerformanceMachineType")
+    public static String provideHighPerformanceMachineType(RegistryConfigSettings config) {
+      return config.beam.highPerformanceMachineType;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
      * @see google.registry.beam.invoicing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
+     * @see google.registry.beam.invoicing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -133,6 +133,7 @@ public class RegistryConfigSettings {
   /** Configuration for Apache Beam (Cloud Dataflow). */
   public static class Beam {
     public String defaultJobRegion;
+    public String highPerformanceMachineType;
     public String stagingBucketUrl;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -419,6 +419,14 @@ misc:
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
   defaultJobRegion: us-east1
+  # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
+  # to check the VM CPU quota for the job region. In a massively parallel
+  # pipeline this quota can be easily reached and needs to be raised, otherwise
+  # the job will run very slowly. Also note that there is a separate quota for
+  # external IPv4 address in a region, which means that machine type with higher
+  # core count per machine may be preferable in order to preserve IP addresses.
+  # See: https://cloud.google.com/compute/quotas#cpu_quota
+  highPerformanceMachineType: n2-standard-4
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -14,18 +14,15 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
-import java.time.Duration;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import javax.inject.Inject;
 
 /** Action for fanning out cron tasks for each commit log bucket. */
@@ -38,25 +35,27 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  private static final Random random = new Random();
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
-  @Inject TaskQueueUtils taskQueueUtils;
   @Inject @Parameter("endpoint") String endpoint;
   @Inject @Parameter("queue") String queue;
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
+
+
   @Override
   public void run() {
-    Queue taskQueue = getQueue(queue);
     for (int bucketId : CommitLogBucket.getBucketIds()) {
-      long delay =
-          jitterSeconds.map(i -> random.nextInt((int) Duration.ofSeconds(i).toMillis())).orElse(0);
-      TaskOptions taskOptions =
-          TaskOptions.Builder.withUrl(endpoint)
-              .param(BUCKET_PARAM, Integer.toString(bucketId))
-              .countdownMillis(delay);
-      taskQueueUtils.enqueue(taskQueue, taskOptions);
+      cloudTasksUtils.enqueue(
+          queue,
+          CloudTasksUtils.createPostTask(
+              endpoint,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
+              clock,
+              jitterSeconds));
     }
   }
 }

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -157,6 +157,12 @@
     <url-pattern>/_dr/cron/readDnsQueue</url-pattern>
   </servlet-mapping>
 
+  <!-- Replicates SQL transactions to Datastore during the Registry 3.0 migration. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/cron/replicateToDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Publishes DNS updates. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -371,6 +371,10 @@ public final class DomainDeleteFlow implements TransactionalFlow {
             String.format(
                 "Domain %s was deleted by registry administrator with final deletion effective: %s",
                 existingDomain.getDomainName(), deletionTime))
+        .setResponseData(
+            ImmutableList.of(
+                DomainPendingActionNotificationResponse.create(
+                    existingDomain.getDomainName(), true, trid, now)))
         .build();
   }
 

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -16,6 +16,7 @@ package google.registry.flows.domain;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
 import static com.google.common.collect.Sets.symmetricDifference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.flows.FlowUtils.persistEntityChanges;
@@ -43,6 +44,9 @@ import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_UPDATE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
 import com.google.common.net.InternetDomainName;
 import google.registry.dns.DnsQueue;
 import google.registry.flows.EppException;
@@ -76,6 +80,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
+import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.model.tld.Registry;
 import java.util.Optional;
@@ -175,6 +180,9 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     Optional<BillingEvent.OneTime> statusUpdateBillingEvent =
         createBillingEventForStatusUpdates(existingDomain, newDomain, domainHistory, now);
     statusUpdateBillingEvent.ifPresent(entitiesToSave::add);
+    Optional<PollMessage.OneTime> serverStatusUpdatePollMessage =
+        createPollMessageForServerStatusUpdates(existingDomain, newDomain, domainHistory, now);
+    serverStatusUpdatePollMessage.ifPresent(entitiesToSave::add);
     EntityChanges entityChanges =
         flowCustomLogic.beforeSave(
             BeforeSaveParameters.newBuilder()
@@ -306,4 +314,50 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     }
     return Optional.empty();
   }
+
+  /** Enqueues a poll message iff a superuser is adding/removing server statuses. */
+  private Optional<PollMessage.OneTime> createPollMessageForServerStatusUpdates(
+      DomainBase existingDomain, DomainBase newDomain, DomainHistory historyEntry, DateTime now) {
+    if (registrarId.equals(existingDomain.getPersistedCurrentSponsorRegistrarId())) {
+      // Don't send a poll message when a superuser registrar is updating its own domain.
+      return Optional.empty();
+    }
+    ImmutableSortedSet<String> addedServerStatuses =
+        Sets.difference(newDomain.getStatusValues(), existingDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+    ImmutableSortedSet<String> removedServerStatuses =
+        Sets.difference(existingDomain.getStatusValues(), newDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+
+    String msg = "";
+    if (addedServerStatuses.size() > 0 && removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s and removed the status(es)"
+                  + " %s.",
+              addedServerStatuses, removedServerStatuses);
+    } else if (addedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s.", addedServerStatuses);
+    } else if (removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has removed the status(es) %s.", removedServerStatuses);
+    } else {
+      return Optional.empty();
+    }
+
+    return Optional.ofNullable(
+        new PollMessage.OneTime.Builder()
+            .setParent(historyEntry)
+            .setEventTime(now)
+            .setRegistrarId(existingDomain.getCurrentSponsorRegistrarId())
+            .setMsg(msg)
+            .build());
+  }
 }

core/src/main/java/google/registry/model/AppEngineEnvironment.java

@@ -43,10 +43,22 @@ public class AppEngineEnvironment {
 
   private Environment environment;
 
+  /**
+   * Constructor for use by tests.
+   *
+   * <p>All test suites must use the same appId for environments, since when tearing down we do not
+   * clear cached environments in spawned threads. See {@link #unsetEnvironmentForAllThreads} for
+   * more information.
+   */
   public AppEngineEnvironment() {
-    this("PlaceholderAppId");
+    /**
+     * Use AppEngineExtension's appId here so that ofy and sql entities can be compared with {@code
+     * Objects#equals()}. The choice of this value does not impact functional correctness.
+     */
+    this("test");
   }
 
+  /** Constructor for use by applications, e.g., BEAM pipelines. */
   public AppEngineEnvironment(String appId) {
     environment = createAppEngineEnvironment(appId);
   }
@@ -65,7 +77,17 @@ public class AppEngineEnvironment {
     ApiProxy.clearEnvironmentForCurrentThread();
   }
 
+  /**
+   * Unsets the test environment in all threads with best effort.
+   *
+   * <p>This method unsets the environment factory and clears the cached environment in the current
+   * thread (the main test runner thread). We do not clear the cache in spawned threads, even though
+   * they may be reused. This is not a problem as long as the appId stays the same: those threads
+   * are used only in AppEngine or BEAM tests, and expect the presence of an environment.
+   */
   public void unsetEnvironmentForAllThreads() {
+    unsetEnvironmentForCurrentThread();
+
     try {
       Method method = ApiProxy.class.getDeclaredMethod("clearEnvironmentFactory");
       method.setAccessible(true);

core/src/main/java/google/registry/model/BackupGroupRoot.java

@@ -14,6 +14,7 @@
 
 package google.registry.model;
 
+import google.registry.util.PreconditionsUtils;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.AttributeOverride;
@@ -30,7 +31,7 @@ import javax.xml.bind.annotation.XmlTransient;
  * that we can enforce strictly increasing timestamps.
  */
 @MappedSuperclass
-public abstract class BackupGroupRoot extends ImmutableObject {
+public abstract class BackupGroupRoot extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * An automatically managed timestamp of when this object was last written to Datastore.
@@ -49,4 +50,14 @@ public abstract class BackupGroupRoot extends ImmutableObject {
   public UpdateAutoTimestamp getUpdateTimestamp() {
     return updateTimestamp;
   }
+
+  /**
+   * Copies {@link #updateTimestamp} from another entity.
+   *
+   * <p>This method is for the few cases when {@code updateTimestamp} is copied between different
+   * types of entities. Use {@link #clone} for same-type copying.
+   */
+  protected void copyUpdateTimestamp(BackupGroupRoot other) {
+    this.updateTimestamp = PreconditionsUtils.checkArgumentNotNull(other, "other").updateTimestamp;
+  }
 }

core/src/main/java/google/registry/model/CreateAutoTimestamp.java

@@ -14,8 +14,20 @@
 
 package google.registry.model;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.OnLoad;
 import google.registry.model.translators.CreateAutoTimestampTranslatorFactory;
+import google.registry.util.DateTimeUtils;
+import java.time.ZonedDateTime;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.PostLoad;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -23,9 +35,37 @@ import org.joda.time.DateTime;
  *
  * @see CreateAutoTimestampTranslatorFactory
  */
-public class CreateAutoTimestamp extends ImmutableObject {
+@Embeddable
+public class CreateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
-  DateTime timestamp;
+  @Transient DateTime timestamp;
+
+  @Column(nullable = false)
+  @Ignore
+  ZonedDateTime creationTime;
+
+  @PrePersist
+  @PreUpdate
+  void setTimestamp() {
+    if (creationTime == null) {
+      timestamp = jpaTm().getTransactionTime();
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @OnLoad
+  void onLoad() {
+    if (timestamp != null) {
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (creationTime != null) {
+      timestamp = DateTimeUtils.toJodaDateTime(creationTime);
+    }
+  }
 
   /** Returns the timestamp. */
   @Nullable
@@ -36,6 +76,7 @@ public class CreateAutoTimestamp extends ImmutableObject {
   public static CreateAutoTimestamp create(@Nullable DateTime timestamp) {
     CreateAutoTimestamp instance = new CreateAutoTimestamp();
     instance.timestamp = timestamp;
+    instance.creationTime = (timestamp == null) ? null : DateTimeUtils.toZonedDateTime(timestamp);
     return instance;
   }
 }

core/src/main/java/google/registry/model/EntityClasses.java

@@ -43,7 +43,6 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.server.Lock;
 import google.registry.model.server.ServerSecret;
 import google.registry.model.tld.Registry;
-import google.registry.model.tmch.TmchCrl;
 
 /** Sets of classes of the Objectify-registered entities in use throughout the model. */
 public final class EntityClasses {
@@ -85,8 +84,7 @@ public final class EntityClasses {
           Registrar.class,
           RegistrarContact.class,
           Registry.class,
-          ServerSecret.class,
-          TmchCrl.class);
+          ServerSecret.class);
 
   private EntityClasses() {}
 }

core/src/main/java/google/registry/model/EppResource.java

@@ -50,6 +50,8 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -112,7 +114,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
    * resource fields.
    */
-  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  @AttributeOverrides({@AttributeOverride(name = "creationTime", column = @Column())})
+  @Index
+  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -85,6 +85,9 @@ public abstract class ImmutableObject implements Cloneable {
   @Target(FIELD)
   public @interface Insignificant {}
 
+  // Note: if this class is made to implement Serializable, this field must become 'transient' since
+  // hashing is not stable across executions. Also note that @XmlTransient is forbidden on transient
+  // fields and need to be removed if transient is added.
   @Ignore @XmlTransient protected Integer hashCode;
 
   private boolean equalsImmutableObject(ImmutableObject other) {

core/src/main/java/google/registry/model/UnsafeSerializable.java

@@ -0,0 +1,34 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import java.io.Serializable;
+
+/**
+ * Marker interface for Nomulus entities whose serialization are implemented in a fragile way. These
+ * entities are made {@link Serializable} so that they can be passed between JVMs. The intended use
+ * case is BEAM pipeline-based cross-database data validation between Datastore and Cloud SQL during
+ * the migration. Note that only objects loaded from the SQL database need serialization support.
+ * Objects exported from Datastore can already be serialized as protocol buffers.
+ *
+ * <p>All entities implementing this interface take advantage of the fact that all Java collection
+ * classes we use, either directly or indirectly, including those in Java libraries, Guava,
+ * Objectify, and Hibernate are {@code Serializable}.
+ *
+ * <p>The {@code serialVersionUID} field has also been omitted in the implementing classes, since
+ * they are not used for persistence.
+ */
+// TODO(b/203609782): either remove this interface or fix implementors post migration.
+public interface UnsafeSerializable extends Serializable {}

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -38,7 +38,7 @@ import org.joda.time.DateTime;
  * @see UpdateAutoTimestampTranslatorFactory
  */
 @Embeddable
-public class UpdateAutoTimestamp extends ImmutableObject {
+public class UpdateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
   // When set to true, database converters/translators should do the auto update.  When set to
   // false, auto update should be suspended (this exists to allow us to preserve the original value

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -27,7 +27,6 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -39,6 +38,7 @@ import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.TimeOfYear;
 import google.registry.model.domain.DomainBase;
@@ -72,18 +72,33 @@ import org.joda.time.DateTime;
 /** A billable event in a domain's lifecycle. */
 @MappedSuperclass
 public abstract class BillingEvent extends ImmutableObject
-    implements Buildable, TransferServerApproveEntity {
+    implements Buildable, TransferServerApproveEntity, UnsafeSerializable {
 
   /** The reason for the bill, which maps 1:1 to skus in go/registry-billing-skus. */
   public enum Reason {
-    CREATE,
+    CREATE(true),
     @Deprecated // TODO(b/31676071): remove this legacy value once old data is cleaned up.
-    ERROR,
-    FEE_EARLY_ACCESS,
-    RENEW,
-    RESTORE,
-    SERVER_STATUS,
-    TRANSFER
+    ERROR(false),
+    FEE_EARLY_ACCESS(true),
+    RENEW(true),
+    RESTORE(true),
+    SERVER_STATUS(false),
+    TRANSFER(true);
+
+    private final boolean requiresPeriod;
+
+    Reason(boolean requiresPeriod) {
+      this.requiresPeriod = requiresPeriod;
+    }
+
+    /**
+     * Returns whether billing events with this reason have a period years associated with them.
+     *
+     * <p>Note that this is an "if an only if" condition.
+     */
+    public boolean hasPeriodYears() {
+      return requiresPeriod;
+    }
   }
 
   /** Set of flags that can be applied to billing events. */
@@ -267,7 +282,7 @@ public abstract class BillingEvent extends ImmutableObject
     public T build() {
       T instance = getInstance();
       checkNotNull(instance.reason, "Reason must be set");
-      checkNotNull(instance.clientId, "Client ID must be set");
+      checkNotNull(instance.clientId, "Registrar ID must be set");
       checkNotNull(instance.eventTime, "Event time must be set");
       checkNotNull(instance.targetId, "Target ID must be set");
       checkNotNull(instance.parent, "Parent must be set");
@@ -461,17 +476,14 @@ public abstract class BillingEvent extends ImmutableObject
         checkNotNull(instance.billingTime);
         checkNotNull(instance.cost);
         checkState(!instance.cost.isNegative(), "Costs should be non-negative.");
-        ImmutableSet<Reason> reasonsWithPeriods =
-            Sets.immutableEnumSet(
-                Reason.CREATE,
-                Reason.FEE_EARLY_ACCESS,
-                Reason.RENEW,
-                Reason.RESTORE,
-                Reason.TRANSFER);
-        checkState(
-            reasonsWithPeriods.contains(instance.reason) == (instance.periodYears != null),
-            "Period years must be set if and only if reason is "
-                + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        // TODO(mcilwain): Enforce this check on all billing events (not just more recent ones)
+        //                 post-migration after we add the missing period years values in SQL.
+        if (instance.eventTime.isAfter(DateTime.parse("2019-01-01T00:00:00Z"))) {
+          checkState(
+              instance.reason.hasPeriodYears() == (instance.periodYears != null),
+              "Period years must be set if and only if reason is "
+                  + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        }
         checkState(
             instance.getFlags().contains(Flag.SYNTHETIC)
                 == (instance.syntheticCreationTime != null),

core/src/main/java/google/registry/model/bulkquery/BulkQueryEntities.java

@@ -0,0 +1,109 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+
+/**
+ * Utilities for managing an alternative JPA entity model optimized for bulk loading multi-level
+ * entities such as {@link DomainBase} and {@link DomainHistory}.
+ *
+ * <p>In a bulk query for a multi-level JPA entity type, the JPA framework only generates a bulk
+ * query (SELECT * FROM table) for the base table. Then, for each row in the base table, additional
+ * queries are issued to load associated rows in child tables. This can be very slow when an entity
+ * type has multiple child tables.
+ *
+ * <p>We have defined an alternative entity model for {@code DomainBase} and {@code DomainHistory},
+ * where the base table as well as the child tables are mapped to single-level entity types. The
+ * idea is to load each of these types using a bulk query, and assemble them into the target type in
+ * memory in a pipeline. The main use case is Datastore-Cloud SQL validation during the Registry
+ * database migration, where we will need the full database snapshots frequently.
+ */
+public class BulkQueryEntities {
+  /**
+   * The JPA entity classes in persistence.xml to replace when creating the {@link
+   * JpaTransactionManager} for bulk query.
+   */
+  public static final ImmutableMap<String, String> JPA_ENTITIES_REPLACEMENTS =
+      ImmutableMap.of(
+          DomainBase.class.getCanonicalName(),
+          DomainBaseLite.class.getCanonicalName(),
+          DomainHistory.class.getCanonicalName(),
+          DomainHistoryLite.class.getCanonicalName());
+
+  /* The JPA entity classes that are not included in persistence.xml and need to be added to
+   * the {@link JpaTransactionManager} for bulk query.*/
+  public static final ImmutableList<String> JPA_ENTITIES_NEW =
+      ImmutableList.of(
+          DomainHost.class.getCanonicalName(), DomainHistoryHost.class.getCanonicalName());
+
+  public static DomainBase assembleDomainBase(
+      DomainBaseLite domainBaseLite,
+      ImmutableSet<GracePeriod> gracePeriods,
+      ImmutableSet<DelegationSignerData> delegationSignerData,
+      ImmutableSet<VKey<HostResource>> nsHosts) {
+    DomainBase.Builder builder = new DomainBase.Builder();
+    builder.copyFrom(domainBaseLite);
+    builder.setGracePeriods(gracePeriods);
+    builder.setDsData(delegationSignerData);
+    builder.setNameservers(nsHosts);
+    return builder.build();
+  }
+
+  public static DomainHistory assembleDomainHistory(
+      DomainHistoryLite domainHistoryLite,
+      ImmutableSet<DomainDsDataHistory> dsDataHistories,
+      ImmutableSet<VKey<HostResource>> domainHistoryHosts,
+      ImmutableSet<GracePeriodHistory> gracePeriodHistories,
+      ImmutableSet<DomainTransactionRecord> transactionRecords) {
+    DomainHistory.Builder builder = new DomainHistory.Builder();
+    builder.copyFrom(domainHistoryLite);
+    DomainContent rawDomainContent = domainHistoryLite.domainContent;
+    if (rawDomainContent != null) {
+      DomainContent newDomainContent =
+          domainHistoryLite
+              .domainContent
+              .asBuilder()
+              .setNameservers(domainHistoryHosts)
+              .setGracePeriods(
+                  gracePeriodHistories.stream()
+                      .map(GracePeriod::createFromHistory)
+                      .collect(toImmutableSet()))
+              .setDsData(
+                  dsDataHistories.stream()
+                      .map(DelegationSignerData::create)
+                      .collect(toImmutableSet()))
+              .build();
+      builder.setDomain(newDomainContent);
+    }
+    return builder.buildAndAssemble(
+        dsDataHistories, domainHistoryHosts, gracePeriodHistories, transactionRecords);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java

@@ -0,0 +1,49 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+
+/**
+ * A 'light' version of {@link DomainBase} with only base table ("Domain") attributes, which allows
+ * fast bulk loading. They are used in in-memory assembly of {@code DomainBase} instances along with
+ * bulk-loaded child entities ({@code GracePeriod} etc). The in-memory assembly achieves much higher
+ * performance than loading {@code DomainBase} directly.
+ *
+ * <p>Please refer to {@link BulkQueryEntities} for more information.
+ */
+@Entity(name = "Domain")
+@WithStringVKey
+@Access(AccessType.FIELD)
+public class DomainBaseLite extends DomainContent implements SqlOnlyEntity {
+
+  @Override
+  @javax.persistence.Id
+  @Access(AccessType.PROPERTY)
+  public String getRepoId() {
+    return super.getRepoId();
+  }
+
+  public static VKey<DomainBaseLite> createVKey(String repoId) {
+    return VKey.createSql(DomainBaseLite.class, repoId);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java

@@ -0,0 +1,50 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.host.HostResource;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+
+/**
+ * A name server host referenced by a {@link google.registry.model.domain.DomainHistory} record.
+ * Please refer to {@link BulkQueryEntities} for usage.
+ */
+@Entity
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryHost.class)
+public class DomainHistoryHost implements Serializable, SqlOnlyEntity {
+
+  @Id private Long domainHistoryHistoryRevisionId;
+  @Id private String domainHistoryDomainRepoId;
+  @Id private String hostRepoId;
+
+  private DomainHistoryHost() {}
+
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(domainHistoryDomainRepoId, domainHistoryHistoryRevisionId);
+  }
+
+  public VKey<HostResource> getHostVKey() {
+    return VKey.create(HostResource.class, hostRepoId);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java

@@ -0,0 +1,122 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.Period;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
+
+/**
+ * A 'light' version of {@link DomainHistory} with only base table ("DomainHistory") attributes,
+ * which allows fast bulk loading. They are used in in-memory assembly of {@code DomainHistory}
+ * instances along with bulk-loaded child entities ({@code GracePeriodHistory} etc). The in-memory
+ * assembly achieves much higher performance than loading {@code DomainHistory} directly.
+ *
+ * <p>Please refer to {@link BulkQueryEntities} for more information.
+ *
+ * <p>This class is adapted from {@link DomainHistory} by removing the {@code dsDataHistories},
+ * {@code gracePeriodHistories}, and {@code nsHosts} fields and associated methods.
+ */
+@Entity(name = "DomainHistory")
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryId.class)
+public class DomainHistoryLite extends HistoryEntry implements SqlOnlyEntity {
+
+  // Store DomainContent instead of DomainBase so we don't pick up its @Id
+  // Nullable for the sake of pre-Registry-3.0 history objects
+  @Nullable DomainContent domainContent;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    parent = Key.create(DomainBase.class, domainRepoId);
+  }
+
+  @Override
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @AttributeOverrides({
+    @AttributeOverride(name = "unit", column = @Column(name = "historyPeriodUnit")),
+    @AttributeOverride(name = "value", column = @Column(name = "historyPeriodValue"))
+  })
+  public Period getPeriod() {
+    return super.getPeriod();
+  }
+
+  /**
+   * For transfers, the id of the other registrar.
+   *
+   * <p>For requests and cancels, the other registrar is the losing party (because the registrar
+   * sending the EPP transfer command is the gaining party). For approves and rejects, the other
+   * registrar is the gaining party.
+   */
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @Column(name = "historyOtherRegistrarId")
+  @Override
+  public String getOtherRegistrarId() {
+    return super.getOtherRegistrarId();
+  }
+
+  @Id
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
+  /** The key to the {@link DomainBase} this is based off of. */
+  public VKey<DomainBase> getParentVKey() {
+    return VKey.create(DomainBase.class, getDomainRepoId());
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (domainContent == null) {
+      return;
+    }
+    // See inline comments in DomainHistory.postLoad for reasons for the following lines.
+    if (domainContent.getDomainName() == null) {
+      domainContent = null;
+    } else if (domainContent.getRepoId() == null) {
+      domainContent.setRepoId(parent.getName());
+    }
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHost.java

@@ -0,0 +1,46 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.host.HostResource;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+
+/** A name server host of a domain. Please refer to {@link BulkQueryEntities} for usage. */
+@Entity
+@Access(AccessType.FIELD)
+@IdClass(DomainHost.class)
+public class DomainHost implements Serializable, SqlOnlyEntity {
+
+  @Id private String domainRepoId;
+
+  @Id private String hostRepoId;
+
+  DomainHost() {}
+
+  public String getDomainRepoId() {
+    return domainRepoId;
+  }
+
+  public VKey<HostResource> getHostVKey() {
+    return VKey.create(HostResource.class, hostRepoId);
+  }
+}

core/src/main/java/google/registry/model/common/Cursor.java

@@ -27,13 +27,13 @@ import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.common.Cursor.CursorId;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.model.tld.Registry;
 import google.registry.persistence.VKey;
-import java.io.Serializable;
 import java.util.List;
 import java.util.Optional;
 import javax.persistence.Column;
@@ -53,7 +53,7 @@ import org.joda.time.DateTime;
 @javax.persistence.Entity
 @IdClass(CursorId.class)
 @InCrossTld
-public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
+public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity, UnsafeSerializable {
 
   /** The scope of a global cursor. A global cursor is a cursor that is not specific to one tld. */
   public static final String GLOBAL = "GLOBAL";
@@ -283,7 +283,7 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
     return cursorTime;
   }
 
-  static class CursorId extends ImmutableObject implements Serializable {
+  public static class CursorId extends ImmutableObject implements UnsafeSerializable {
 
     public CursorType type;
     public String scope;

core/src/main/java/google/registry/model/common/TimeOfYear.java

@@ -28,6 +28,7 @@ import com.google.common.collect.Range;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.List;
 import javax.persistence.Embeddable;
 import org.joda.time.DateTime;
@@ -45,7 +46,7 @@ import org.joda.time.DateTime;
  */
 @Embed
 @Embeddable
-public class TimeOfYear extends ImmutableObject {
+public class TimeOfYear extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * The time as "month day millis" with all fields left-padded with zeroes so that lexographic

core/src/main/java/google/registry/model/common/TimedTransitionProperty.java

@@ -27,7 +27,9 @@ import com.google.common.collect.Maps;
 import com.google.common.collect.Ordering;
 import com.googlecode.objectify.mapper.Mapper;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.util.TypeUtils;
+import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -53,11 +55,12 @@ import org.joda.time.DateTime;
  * to use for storing the list of transitions. The user is given this choice of subclass so that the
  * field of the value type stored in the transition can be given a customized name.
  */
-public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedTransition<V>>
-    extends ForwardingMap<DateTime, T> {
+public class TimedTransitionProperty<
+        V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
+    extends ForwardingMap<DateTime, T> implements UnsafeSerializable {
 
   /**
-   * A transition to a value of type {@code V} at a certain time.  This superclass only has a field
+   * A transition to a value of type {@code V} at a certain time. This superclass only has a field
    * for the {@code DateTime}, which means that subclasses should supply the field of type {@code V}
    * and implementations of the abstract getter and setter methods to access that field. This design
    * is so that subclasses tagged with @Embed can define a custom field name for their value, for
@@ -65,11 +68,12 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    *
    * <p>The public visibility of this class exists only so that it can be subclassed; clients should
    * never call any methods on this class or attempt to access its members, but should instead treat
-   * it as a customizable implementation detail of {@code TimedTransitionProperty}.  However, note
+   * it as a customizable implementation detail of {@code TimedTransitionProperty}. However, note
    * that subclasses must also have public visibility so that they can be instantiated via
    * reflection in a call to {@code fromValueMap}.
    */
-  public abstract static class TimedTransition<V> extends ImmutableObject {
+  public abstract static class TimedTransition<V extends Serializable> extends ImmutableObject
+      implements UnsafeSerializable {
     /** The time at which this value becomes the active value. */
     private DateTime transitionTime;
 
@@ -89,16 +93,16 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
   }
 
   /**
-   * Converts the provided value map into the equivalent transition map, using transition objects
-   * of the given TimedTransition subclass.  The value map must be sorted according to the natural
+   * Converts the provided value map into the equivalent transition map, using transition objects of
+   * the given TimedTransition subclass. The value map must be sorted according to the natural
    * ordering of its DateTime keys, and keys cannot be earlier than START_OF_TIME.
    */
   // NB: The Class<T> parameter could be eliminated by getting the class via reflection, but then
   // the callsite cannot infer T, so unless you explicitly call this as .<V, T>fromValueMap() it
   // will default to using just TimedTransition<V>, which fails at runtime.
-  private static <V, T extends TimedTransition<V>> NavigableMap<DateTime, T> makeTransitionMap(
-      ImmutableSortedMap<DateTime, V> valueMap,
-      final Class<T> timedTransitionSubclass) {
+  private static <V extends Serializable, T extends TimedTransition<V>>
+      NavigableMap<DateTime, T> makeTransitionMap(
+          ImmutableSortedMap<DateTime, V> valueMap, final Class<T> timedTransitionSubclass) {
     checkArgument(
         Ordering.natural().equals(valueMap.comparator()),
         "Timed transition value map must have transition time keys in chronological order");
@@ -121,9 +125,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    *
    * <p>This method should be the normal method for constructing a {@link TimedTransitionProperty}.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> fromValueMap(
-      ImmutableSortedMap<DateTime, V> valueMap,
-      final Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> fromValueMap(
+          ImmutableSortedMap<DateTime, V> valueMap, final Class<T> timedTransitionSubclass) {
     return new TimedTransitionProperty<>(ImmutableSortedMap.copyOf(
         makeTransitionMap(valueMap, timedTransitionSubclass)));
   }
@@ -175,10 +179,10 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * @param allowedTransitions optional map of all possible state-to-state transitions
    * @param allowedTransitionMapName optional transition map description string for error messages
    * @param initialValue optional initial value; if present, the first transition must have this
-   *        value
+   *     value
    * @param badInitialValueErrorMessage option error message string if the initial value is wrong
    */
-  public static <V, T extends TimedTransitionProperty.TimedTransition<V>>
+  public static <V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
       TimedTransitionProperty<V, T> make(
           ImmutableSortedMap<DateTime, V> newTransitions,
           Class<T> transitionClass,
@@ -200,7 +204,7 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * Validates that a transition map is not null or empty, starts at START_OF_TIME, and has
    * transitions which move from one value to another in allowed ways.
    */
-  public static <V, T extends TimedTransitionProperty.TimedTransition<V>>
+  public static <V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
       void validateTimedTransitionMap(
           @Nullable NavigableMap<DateTime, V> transitionMap,
           ImmutableMultimap<V, V> allowedTransitions,
@@ -240,8 +244,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * annotation. The map for those fields must be mutable so that Objectify can load values from
    * Datastore into the map, but clients should still never mutate the field's map directly.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> forMapify(
-      ImmutableSortedMap<DateTime, V> valueMap, Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> forMapify(
+          ImmutableSortedMap<DateTime, V> valueMap, Class<T> timedTransitionSubclass) {
     return new TimedTransitionProperty<>(
         new TreeMap<>(makeTransitionMap(valueMap, timedTransitionSubclass)));
   }
@@ -254,8 +259,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * annotation. The map for those fields must be mutable so that Objectify can load values from
    * Datastore into the map, but clients should still never mutate the field's map directly.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> forMapify(
-      V valueAtStartOfTime, Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> forMapify(
+          V valueAtStartOfTime, Class<T> timedTransitionSubclass) {
     return forMapify(
         ImmutableSortedMap.of(START_OF_TIME, valueAtStartOfTime), timedTransitionSubclass);
   }

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -20,6 +20,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.contact.ContactHistory.ContactHistoryId;
 import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.SqlEntity;
@@ -59,7 +60,7 @@ import javax.persistence.PostLoad;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(ContactHistoryId.class)
-public class ContactHistory extends HistoryEntry implements SqlEntity {
+public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable {
 
   // Store ContactBase instead of ContactResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects

core/src/main/java/google/registry/model/contact/Disclose.java

@@ -20,7 +20,9 @@ import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.eppcommon.PresenceMarker;
+import java.io.Serializable;
 import java.util.List;
 import javax.persistence.Embeddable;
 import javax.persistence.Embedded;
@@ -31,7 +33,7 @@ import javax.xml.bind.annotation.XmlType;
 @Embed
 @Embeddable
 @XmlType(propOrder = {"name", "org", "addr", "voice", "fax", "email"})
-public class Disclose extends ImmutableObject {
+public class Disclose extends ImmutableObject implements UnsafeSerializable {
 
   List<PostalInfoChoice> name;
 
@@ -78,7 +80,7 @@ public class Disclose extends ImmutableObject {
 
   /** The "intLocType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
   @Embed
-  public static class PostalInfoChoice extends ImmutableObject {
+  public static class PostalInfoChoice extends ImmutableObject implements Serializable {
     @XmlAttribute
     PostalInfo.Type type;
 

core/src/main/java/google/registry/model/contact/PostalInfo.java

@@ -20,6 +20,7 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.Buildable;
 import google.registry.model.Buildable.Overlayable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.persistence.Embeddable;
 import javax.persistence.EnumType;
@@ -38,7 +39,8 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @Embed
 @Embeddable
 @XmlType(propOrder = {"name", "org", "address", "type"})
-public class PostalInfo extends ImmutableObject implements Overlayable<PostalInfo> {
+public class PostalInfo extends ImmutableObject
+    implements Overlayable<PostalInfo>, UnsafeSerializable {
 
   /** The type of the address, either localized or international. */
   public enum Type {

core/src/main/java/google/registry/model/domain/DesignatedContact.java

@@ -21,6 +21,7 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.contact.ContactResource;
 import google.registry.persistence.VKey;
 import javax.persistence.Embeddable;
@@ -46,7 +47,7 @@ import javax.xml.bind.annotation.XmlEnumValue;
  */
 @Embed
 @Embeddable
-public class DesignatedContact extends ImmutableObject {
+public class DesignatedContact extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * XML type for contact types. This can be either: {@code "admin"}, {@code "billing"}, or

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -24,6 +24,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import java.util.Set;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -164,6 +165,11 @@ public class DomainBase extends DomainContent
     return cloneDomainProjectedAtTime(this, now);
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();
@@ -189,6 +195,7 @@ public class DomainBase extends DomainContent
     }
 
     public Builder copyFrom(DomainContent domainContent) {
+      this.getInstance().copyUpdateTimestamp(domainContent);
       return this.setAuthInfo(domainContent.getAuthInfo())
           .setAutorenewPollMessage(domainContent.getAutorenewPollMessage())
           .setAutorenewBillingEvent(domainContent.getAutorenewBillingEvent())

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -33,6 +33,7 @@ import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.HashSet;
 import java.util.Optional;
@@ -216,6 +217,10 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     return super.getId();
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), getId());
+  }
+
   /** Returns keys to the {@link HostResource} that are the nameservers for the domain. */
   public Set<VKey<HostResource>> getNsHosts() {
     return nsHosts;
@@ -299,6 +304,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   public void beforeSqlSaveOnReplay() {
     if (domainContent == null) {
       domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
+      domainContent.fullyQualifiedDomainName =
+          DomainNameUtils.canonicalizeDomainName(domainContent.fullyQualifiedDomainName);
       fillAuxiliaryFieldsFromDomain(this);
     }
   }
@@ -314,6 +321,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
           nullToEmptyImmutableCopy(domainHistory.domainContent.getGracePeriods()).stream()
               .map(gracePeriod -> GracePeriodHistory.createFrom(domainHistory.id, gracePeriod))
               .collect(toImmutableSet());
+    } else {
+      domainHistory.nsHosts = ImmutableSet.of();
     }
   }
 
@@ -393,8 +402,16 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       if (domainContent == null) {
         return this;
       }
+      // TODO(b/203609982): if actual type of domainContent is DomainBase, convert to DomainContent
+      // Note: a DomainHistory fetched by JPA has DomainContent in this field. Allowing DomainBase
+      // in the setter makes equality checks messy.
       getInstance().domainContent = domainContent;
-      return super.setParent(domainContent);
+      if (domainContent instanceof DomainBase) {
+        super.setParent(domainContent);
+      } else {
+        super.setParent(Key.create(DomainBase.class, domainContent.getRepoId()));
+      }
+      return this;
     }
 
     public Builder setDomainRepoId(String domainRepoId) {
@@ -412,5 +429,19 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       fillAuxiliaryFieldsFromDomain(instance);
       return instance;
     }
+
+    public DomainHistory buildAndAssemble(
+        ImmutableSet<DomainDsDataHistory> dsDataHistories,
+        ImmutableSet<VKey<HostResource>> domainHistoryHosts,
+        ImmutableSet<GracePeriodHistory> gracePeriodHistories,
+        ImmutableSet<DomainTransactionRecord> transactionRecords) {
+      DomainHistory instance = super.build();
+      instance.dsDataHistories = dsDataHistories;
+      instance.nsHosts = domainHistoryHosts;
+      instance.gracePeriodHistories = gracePeriodHistories;
+      instance.domainTransactionRecords = transactionRecords;
+      instance.hashCode = null;
+      return instance;
+    }
   }
 }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.model.replay.SqlOnlyEntity;
@@ -203,7 +204,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
   /** Entity class to represent a historic {@link GracePeriod}. */
   @Entity(name = "GracePeriodHistory")
   @Table(indexes = @Index(columnList = "domainRepoId"))
-  static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity {
+  public static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity {
     @Id Long gracePeriodHistoryRevisionId;
 
     /** ID for the associated {@link DomainHistory} entity. */
@@ -215,6 +216,10 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
       return super.getGracePeriodId();
     }
 
+    public DomainHistoryId getDomainHistoryId() {
+      return new DomainHistoryId(getDomainRepoId(), domainHistoryRevisionId);
+    }
+
     static GracePeriodHistory createFrom(long historyRevisionId, GracePeriod gracePeriod) {
       GracePeriodHistory instance = new GracePeriodHistory();
       instance.gracePeriodHistoryRevisionId = allocateId();

core/src/main/java/google/registry/model/domain/GracePeriodBase.java

@@ -17,6 +17,7 @@ package google.registry.model.domain;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.persistence.BillingVKey.BillingEventVKey;
@@ -35,7 +36,7 @@ import org.joda.time.DateTime;
 @Embed
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public class GracePeriodBase extends ImmutableObject {
+public class GracePeriodBase extends ImmutableObject implements UnsafeSerializable {
 
   /** Unique id required for hibernate representation. */
   @Transient long gracePeriodId;

core/src/main/java/google/registry/model/domain/Period.java

@@ -16,6 +16,7 @@ package google.registry.model.domain;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -25,7 +26,7 @@ import javax.xml.bind.annotation.XmlValue;
 /** The "periodType" from <a href="http://tools.ietf.org/html/rfc5731">RFC5731</a>. */
 @Embed
 @javax.persistence.Embeddable
-public class Period extends ImmutableObject {
+public class Period extends ImmutableObject implements UnsafeSerializable {
 
   @Enumerated(EnumType.STRING)
   @XmlAttribute

core/src/main/java/google/registry/model/domain/RegistryLock.java

@@ -28,6 +28,8 @@ import google.registry.util.DateTimeUtils;
 import java.time.ZonedDateTime;
 import java.util.Optional;
 import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
@@ -100,7 +102,11 @@ public final class RegistryLock extends ImmutableObject implements Buildable, Sq
   private String registrarPocId;
 
   /** When the lock is first requested. */
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "lockRequestTime", nullable = false))
+  })
   private CreateAutoTimestamp lockRequestTime = CreateAutoTimestamp.create(null);
 
   /** When the unlock is first requested. */

core/src/main/java/google/registry/model/domain/launch/LaunchNotice.java

@@ -27,6 +27,7 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.persistence.Embedded;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -39,7 +40,7 @@ import org.joda.time.DateTime;
 @Embed
 @XmlType(propOrder = {"noticeId", "expirationTime", "acceptedTime"})
 @javax.persistence.Embeddable
-public class LaunchNotice extends ImmutableObject {
+public class LaunchNotice extends ImmutableObject implements UnsafeSerializable {
 
   /** An empty instance to use in place of null. */
   private static final NoticeIdType EMPTY_NOTICE_ID = new NoticeIdType();
@@ -47,14 +48,13 @@ public class LaunchNotice extends ImmutableObject {
   /** An id with a validator-id attribute. */
   @Embed
   @javax.persistence.Embeddable
-  public static class NoticeIdType extends ImmutableObject {
+  public static class NoticeIdType extends ImmutableObject implements UnsafeSerializable {
 
     /**
-     * The Trademark Claims Notice ID from
-     * {@link "http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3"}.
+     * The Trademark Claims Notice ID from <a
+     * href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">the RFC</a>.
      */
-    @XmlValue
-    String tcnId;
+    @XmlValue String tcnId;
 
     /** The identifier of the TMDB provider to use, defaulting to the TMCH. */
     @IgnoreSave(IfNull.class)

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataBase.java

@@ -17,6 +17,7 @@ package google.registry.model.domain.secdns;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.MappedSuperclass;
@@ -31,7 +32,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @Embed
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public abstract class DomainDsDataBase extends ImmutableObject {
+public abstract class DomainDsDataBase extends ImmutableObject implements UnsafeSerializable {
 
   @Ignore @XmlTransient @Transient String domainRepoId;
 

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java

@@ -16,7 +16,9 @@ package google.registry.model.domain.secdns;
 
 import static google.registry.model.IdService.allocateId;
 
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.replay.SqlOnlyEntity;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -26,7 +28,8 @@ import javax.persistence.Id;
 
 /** Entity class to represent a historic {@link DelegationSignerData}. */
 @Entity
-public class DomainDsDataHistory extends DomainDsDataBase implements SqlOnlyEntity {
+public class DomainDsDataHistory extends DomainDsDataBase
+    implements SqlOnlyEntity, UnsafeSerializable {
 
   @Id Long dsDataHistoryRevisionId;
 
@@ -53,6 +56,10 @@ public class DomainDsDataHistory extends DomainDsDataBase implements SqlOnlyEnti
     return instance;
   }
 
+  public DomainHistory.DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), domainHistoryRevisionId);
+  }
+
   @Override
   @Access(AccessType.PROPERTY)
   public String getDomainRepoId() {

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -122,7 +122,6 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @Nullable @Index String domainName;
 
   /** When this token was created. */
-  @Column(nullable = false)
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /** Allowed registrar client IDs for this token, or null if all registrars are allowed. */

core/src/main/java/google/registry/model/eppcommon/Address.java

@@ -27,6 +27,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -56,7 +57,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public class Address extends ImmutableObject implements Jsonifiable {
+public class Address extends ImmutableObject implements Jsonifiable, UnsafeSerializable {
 
   /** The schema validation will enforce that this has 3 lines at most. */
   // TODO(b/177569726): Remove this field after migration. We need to figure out how to generate

core/src/main/java/google/registry/model/eppcommon/AuthInfo.java

@@ -16,6 +16,7 @@ package google.registry.model.eppcommon;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Embeddable;
 import javax.persistence.Embedded;
 import javax.persistence.MappedSuperclass;
@@ -35,7 +36,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public abstract class AuthInfo extends ImmutableObject {
+public abstract class AuthInfo extends ImmutableObject implements UnsafeSerializable {
 
   @Embedded protected PasswordAuth pw;
 
@@ -47,7 +48,7 @@ public abstract class AuthInfo extends ImmutableObject {
   @Embed
   @XmlType(namespace = "urn:ietf:params:xml:ns:eppcom-1.0")
   @Embeddable
-  public static class PasswordAuth extends ImmutableObject {
+  public static class PasswordAuth extends ImmutableObject implements UnsafeSerializable {
     @XmlValue
     @XmlJavaTypeAdapter(NormalizedStringAdapter.class)
     String value;

core/src/main/java/google/registry/model/eppcommon/PhoneNumber.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Embeddable;
 import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -49,7 +50,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public class PhoneNumber extends ImmutableObject {
+public class PhoneNumber extends ImmutableObject implements UnsafeSerializable {
 
   @XmlValue
   @XmlJavaTypeAdapter(CollapsedStringAdapter.class)

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -112,7 +112,6 @@ public enum StatusValue implements EppEnum {
    */
   PENDING_UPDATE(AllowedOn.NONE),
 
-
   /** A non-client-settable status that prevents deletes of EPP resources. */
   SERVER_DELETE_PROHIBITED(AllowedOn.ALL),
 
@@ -162,6 +161,10 @@ public enum StatusValue implements EppEnum {
     return xmlName.startsWith("client");
   }
 
+  public boolean isServerSettable() {
+    return xmlName.startsWith("server");
+  }
+
   public boolean isChargedStatus() {
     return xmlName.startsWith("server") && xmlName.endsWith("Prohibited");
   }

core/src/main/java/google/registry/model/eppcommon/Trid.java

@@ -18,6 +18,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import javax.xml.bind.annotation.XmlElement;
@@ -32,7 +33,7 @@ import javax.xml.bind.annotation.XmlType;
 @Embed
 @XmlType(propOrder = {"clientTransactionId", "serverTransactionId"})
 @javax.persistence.Embeddable
-public class Trid extends ImmutableObject {
+public class Trid extends ImmutableObject implements UnsafeSerializable {
 
   /** The server transaction id. */
   @XmlElement(name = "svTRID", namespace = "urn:ietf:params:xml:ns:epp-1.0")

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -20,11 +20,13 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.host.HostHistory.HostHistoryId;
 import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -60,7 +62,7 @@ import javax.persistence.PostLoad;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(HostHistoryId.class)
-public class HostHistory extends HistoryEntry implements SqlEntity {
+public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable {
 
   // Store HostBase instead of HostResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects
@@ -140,6 +142,8 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
   public void beforeSqlSaveOnReplay() {
     if (hostBase == null) {
       hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
+      hostBase.fullyQualifiedHostName =
+          DomainNameUtils.canonicalizeDomainName(hostBase.fullyQualifiedHostName);
     }
   }
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -22,6 +22,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 
@@ -51,6 +52,11 @@ public class HostResource extends HostBase
     return VKey.create(HostResource.class, getRepoId(), Key.create(this));
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedHostName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedHostName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();

core/src/main/java/google/registry/model/poll/PendingActionNotificationResponse.java

@@ -17,6 +17,7 @@ package google.registry.model.poll;
 import com.google.common.annotations.VisibleForTesting;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
 import javax.persistence.Embeddable;
@@ -31,12 +32,13 @@ import org.joda.time.DateTime;
 /** The {@link ResponseData} returned when completing a pending action on a domain. */
 @XmlTransient
 @Embeddable
-public class PendingActionNotificationResponse extends ImmutableObject implements ResponseData {
+public class PendingActionNotificationResponse extends ImmutableObject
+    implements ResponseData, UnsafeSerializable {
 
   /** The inner name type that contains a name and the result boolean. */
   @Embed
   @Embeddable
-  static class NameOrId extends ImmutableObject {
+  static class NameOrId extends ImmutableObject implements UnsafeSerializable {
     @XmlValue
     String value;
 

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -33,6 +33,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
@@ -98,7 +99,7 @@ import org.joda.time.DateTime;
       @javax.persistence.Index(columnList = "eventTime")
     })
 public abstract class PollMessage extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity {
+    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity, UnsafeSerializable {
 
   /** Entity id. */
   @Id

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -72,6 +72,7 @@ import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
@@ -116,7 +117,7 @@ import org.joda.time.DateTime;
     })
 @InCrossTld
 public class Registrar extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity, Jsonifiable {
+    implements Buildable, DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable {
 
   /** Represents the type of a registrar entity. */
   public enum Type {
@@ -404,7 +405,7 @@ public class Registrar extends ImmutableObject
 
   /** A billing account entry for this registrar, consisting of a currency and an account Id. */
   @Embed
-  public static class BillingAccountEntry extends ImmutableObject {
+  public static class BillingAccountEntry extends ImmutableObject implements UnsafeSerializable {
 
     CurrencyUnit currency;
     String accountId;

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -46,6 +46,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registrar.RegistrarContact.RegistrarPocId;
@@ -82,7 +83,7 @@ import javax.persistence.Transient;
 @IdClass(RegistrarPocId.class)
 @InCrossTld
 public class RegistrarContact extends ImmutableObject
-    implements DatastoreAndSqlEntity, Jsonifiable {
+    implements DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable {
 
   @Parent @Transient Key<Registrar> parent;
 

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -169,7 +169,7 @@ public class ReplicateToDatastoreAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire ReplicateToDatastoreAction lock, aborting.";
@@ -194,7 +194,7 @@ public class ReplicateToDatastoreAction implements Runnable {
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 

core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java

@@ -22,6 +22,8 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -45,11 +47,10 @@ import org.joda.time.DateTime;
 @Embed
 @Entity
 public class DomainTransactionRecord extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity {
+    implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable {
 
   @Id
   @Ignore
-  @ImmutableObject.DoNotCompare
   @GeneratedValue(strategy = GenerationType.IDENTITY)
   @ImmutableObject.Insignificant
   Long id;
@@ -58,6 +59,14 @@ public class DomainTransactionRecord extends ImmutableObject
   @Column(nullable = false)
   String tld;
 
+  // The following two fields are exposed in this entity to support bulk-loading in Cloud SQL by the
+  // Datastore-SQL validation. They are excluded from equality check since they are not set in
+  // Datastore.
+  // TODO(b/203609782): post migration, decide whether to keep these two fields.
+  @Ignore @ImmutableObject.Insignificant String domainRepoId;
+
+  @Ignore @ImmutableObject.Insignificant Long historyRevisionId;
+
   /**
    * The time this Transaction takes effect (counting grace periods and other nuances).
    *
@@ -174,6 +183,10 @@ public class DomainTransactionRecord extends ImmutableObject
     }
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(domainRepoId, historyRevisionId);
+  }
+
   public DateTime getReportingTime() {
     return reportingTime;
   }

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -31,6 +31,7 @@ import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactBase;
 import google.registry.model.contact.ContactHistory;
@@ -83,7 +84,8 @@ import org.joda.time.DateTime;
 @Entity
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public class HistoryEntry extends ImmutableObject implements Buildable, DatastoreEntity {
+public class HistoryEntry extends ImmutableObject
+    implements Buildable, DatastoreEntity, UnsafeSerializable {
 
   /** Represents the type of history entry. */
   public enum Type {

core/src/main/java/google/registry/model/server/Lock.java

@@ -32,6 +32,8 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.persistence.transaction.TransactionManager;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
 import java.io.Serializable;
@@ -212,6 +214,47 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
       Duration leaseLength,
       RequestStatusChecker requestStatusChecker,
       boolean checkThreadRunning) {
+    return acquireWithTransactionManager(
+        resourceName, tld, leaseLength, requestStatusChecker, checkThreadRunning, tm());
+  }
+
+  /**
+   * Try to acquire a lock in SQL. Returns absent if it can't be acquired.
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   */
+  public static Optional<Lock> acquireSql(
+      String resourceName,
+      @Nullable String tld,
+      Duration leaseLength,
+      RequestStatusChecker requestStatusChecker,
+      boolean checkThreadRunning) {
+    return acquireWithTransactionManager(
+        resourceName, tld, leaseLength, requestStatusChecker, checkThreadRunning, jpaTm());
+  }
+
+  /** Release the lock. */
+  public void release() {
+    releaseWithTransactionManager(tm());
+  }
+
+  /**
+   * Release the lock from SQL.
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   */
+  public void releaseSql() {
+    releaseWithTransactionManager(jpaTm());
+  }
+
+  /** Try to acquire a lock. Returns absent if it can't be acquired. */
+  private static Optional<Lock> acquireWithTransactionManager(
+      String resourceName,
+      @Nullable String tld,
+      Duration leaseLength,
+      RequestStatusChecker requestStatusChecker,
+      boolean checkThreadRunning,
+      TransactionManager transactionManager) {
     String scope = (tld != null) ? tld : GLOBAL;
     String lockId = makeLockId(resourceName, scope);
     // It's important to use transactNew rather than transact, because a Lock can be used to control
@@ -219,11 +262,12 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
     // must be definitively acquired before it is used, even when called inside another transaction.
     Supplier<AcquireResult> lockAcquirer =
         () -> {
-          DateTime now = tm().getTransactionTime();
+          DateTime now = transactionManager.getTransactionTime();
 
           // Checking if an unexpired lock still exists - if so, the lock can't be acquired.
           Lock lock =
-              tm().loadByKeyIfPresent(
+              transactionManager
+                  .loadByKeyIfPresent(
                       VKey.create(
                           Lock.class,
                           new LockId(resourceName, scope),
@@ -249,13 +293,15 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
               create(resourceName, scope, requestStatusChecker.getLogId(), now, leaseLength);
           // Locks are not parented under an EntityGroupRoot (so as to avoid write
           // contention) and don't need to be backed up.
-          tm().putIgnoringReadOnly(newLock);
+          transactionManager.putIgnoringReadOnly(newLock);
 
           return AcquireResult.create(now, lock, newLock, lockState);
         };
     // In ofy, backup is determined per-action, but in SQL it's determined per-transaction
     AcquireResult acquireResult =
-        tm().isOfy() ? tm().transactNew(lockAcquirer) : jpaTm().transactWithoutBackup(lockAcquirer);
+        transactionManager.isOfy()
+            ? transactionManager.transactNew(lockAcquirer)
+            : ((JpaTransactionManager) transactionManager).transactWithoutBackup(lockAcquirer);
 
     logAcquireResult(acquireResult);
     lockMetrics.recordAcquire(resourceName, scope, acquireResult.lockState());
@@ -263,7 +309,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
   }
 
   /** Release the lock. */
-  public void release() {
+  private void releaseWithTransactionManager(TransactionManager transactionManager) {
     // Just use the default clock because we aren't actually doing anything that will use the clock.
     Supplier<Void> lockReleaser =
         () -> {
@@ -274,15 +320,17 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
           VKey<Lock> key =
               VKey.create(
                   Lock.class, new LockId(resourceName, tld), Key.create(Lock.class, lockId));
-          Lock loadedLock = tm().loadByKeyIfPresent(key).orElse(null);
+          Lock loadedLock = transactionManager.loadByKeyIfPresent(key).orElse(null);
           if (Lock.this.equals(loadedLock)) {
             // Use deleteIgnoringReadOnly() so that we don't create a commit log entry for deleting
             // the lock.
             logger.atInfo().log("Deleting lock: %s", lockId);
-            tm().deleteIgnoringReadOnly(key);
+            transactionManager.deleteIgnoringReadOnly(key);
 
             lockMetrics.recordRelease(
-                resourceName, tld, new Duration(acquiredTime, tm().getTransactionTime()));
+                resourceName,
+                tld,
+                new Duration(acquiredTime, transactionManager.getTransactionTime()));
           } else {
             logger.atSevere().log(
                 "The lock we acquired was transferred to someone else before we"
@@ -294,11 +342,12 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
           }
           return null;
         };
+
     // In ofy, backup is determined per-action, but in SQL it's determined per-transaction
-    if (tm().isOfy()) {
-      tm().transact(lockReleaser);
+    if (transactionManager.isOfy()) {
+      transactionManager.transact(lockReleaser);
     } else {
-      jpaTm().transactWithoutBackup(lockReleaser);
+      ((JpaTransactionManager) transactionManager).transactWithoutBackup(lockReleaser);
     }
   }
 

core/src/main/java/google/registry/model/tld/Registry.java

@@ -52,6 +52,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
@@ -88,7 +89,8 @@ import org.joda.time.Duration;
 @Entity
 @javax.persistence.Entity(name = "Tld")
 @InCrossTld
-public class Registry extends ImmutableObject implements Buildable, DatastoreAndSqlEntity {
+public class Registry extends ImmutableObject
+    implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable {
 
   @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
@@ -308,7 +310,7 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
    * <p>There must be at least one entry in this set.
    *
    * <p>All entries of this list must be valid keys for the map of {@code DnsWriter}s injected by
-   * <code>@Inject Map<String, DnsWriter></code>
+   * {@code @Inject Map<String, DnsWriter>}
    */
   @Column(nullable = false)
   Set<String> dnsWriters;

core/src/main/java/google/registry/model/tmch/ClaimsList.java

@@ -27,6 +27,8 @@ import google.registry.model.replay.SqlOnlyEntity;
 import google.registry.model.tld.label.ReservedList.ReservedListEntry;
 import java.util.Map;
 import java.util.Optional;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
@@ -56,7 +58,11 @@ public class ClaimsList extends ImmutableObject implements SqlOnlyEntity {
   @GeneratedValue(strategy = GenerationType.IDENTITY)
   Long revisionId;
 
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "creationTimestamp", nullable = false))
+  })
   CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
 
   /**

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -16,25 +16,18 @@ package google.registry.model.tmch;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
-import com.googlecode.objectify.annotation.Entity;
-import google.registry.model.annotations.NotBackedUp;
-import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
-import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
 import javax.persistence.Column;
 import org.joda.time.DateTime;
 
-/** Datastore singleton for ICANN's TMCH CA certificate revocation list (CRL). */
-@Entity
+/** Singleton for ICANN's TMCH CA certificate revocation list (CRL). */
 @javax.persistence.Entity
 @Immutable
-@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEntity {
+public final class TmchCrl extends CrossTldSingleton implements SqlOnlyEntity {
 
   @Column(name = "certificateRevocations", nullable = false)
   String crl;
@@ -47,25 +40,23 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
 
   /** Returns the singleton instance of this entity, without memoization. */
   public static Optional<TmchCrl> get() {
-    return tm().transact(() -> tm().loadSingleton(TmchCrl.class));
+    return jpaTm().transact(() -> jpaTm().loadSingleton(TmchCrl.class));
   }
 
   /**
-   * Change the Datastore singleton to a new ASCII-armored X.509 CRL.
+   * Change the singleton to a new ASCII-armored X.509 CRL.
    *
    * <p>Please do not call this function unless your CRL is properly formatted, signed by the root,
    * and actually newer than the one currently in Datastore.
-   *
-   * <p>During the dual-write period, we write to both Datastore and SQL
    */
   public static void set(final String crl, final String url) {
-    tm().transact(
+    jpaTm()
+        .transact(
             () -> {
               TmchCrl tmchCrl = new TmchCrl();
-              tmchCrl.updated = tm().getTransactionTime();
+              tmchCrl.updated = jpaTm().getTransactionTime();
               tmchCrl.crl = checkNotNull(crl, "crl");
               tmchCrl.url = checkNotNull(url, "url");
-              ofyTm().transactNew(() -> ofyTm().putWithoutBackup(tmchCrl));
               jpaTm().transactNew(() -> jpaTm().putWithoutBackup(tmchCrl));
             });
   }
@@ -80,7 +71,7 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
     return crl;
   }
 
-  /** Time we last updated the Datastore with a newer ICANN CRL. */
+  /** Time we last updated the Database with a newer ICANN CRL. */
   public final DateTime getUpdated() {
     return updated;
   }

core/src/main/java/google/registry/model/transfer/BaseTransferObject.java

@@ -16,6 +16,7 @@ package google.registry.model.transfer;
 
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
@@ -27,7 +28,7 @@ import org.joda.time.DateTime;
 /** Fields common to {@link TransferData} and {@link TransferResponse}. */
 @XmlTransient
 @MappedSuperclass
-public abstract class BaseTransferObject extends ImmutableObject {
+public abstract class BaseTransferObject extends ImmutableObject implements UnsafeSerializable {
   /**
    * The status of the current or last transfer. Can be null if never transferred. Note that we
    * leave IgnoreSave off this field so that we can ensure that TransferData loaded from Objectify

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -60,6 +60,7 @@ import google.registry.export.sheet.SheetModule;
 import google.registry.export.sheet.SyncRegistrarsSheetAction;
 import google.registry.flows.FlowComponent;
 import google.registry.mapreduce.MapreduceModule;
+import google.registry.model.replay.ReplicateToDatastoreAction;
 import google.registry.monitoring.whitebox.WhiteboxModule;
 import google.registry.rdap.UpdateRegistrarRdapBaseUrlsAction;
 import google.registry.rde.BrdaCopyAction;
@@ -190,6 +191,8 @@ interface BackendRequestComponent {
 
   ReplayCommitLogsToSqlAction replayCommitLogsToSqlAction();
 
+  ReplicateToDatastoreAction replicateToDatastoreAction();
+
   ResaveAllEppResourcesAction resaveAllEppResourcesAction();
 
   ResaveEntityAction resaveEntityAction();

core/src/main/java/google/registry/persistence/BulkQueryJpaFactory.java

@@ -0,0 +1,65 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.util.Clock;
+import java.util.List;
+import javax.persistence.EntityManagerFactory;
+import org.hibernate.jpa.boot.internal.ParsedPersistenceXmlDescriptor;
+import org.hibernate.jpa.boot.spi.Bootstrap;
+
+/**
+ * Defines factory method for instantiating the bulk-query optimized {@link JpaTransactionManager}.
+ */
+public final class BulkQueryJpaFactory {
+
+  private BulkQueryJpaFactory() {}
+
+  static EntityManagerFactory createBulkQueryEntityManagerFactory(
+      ImmutableMap<String, String> cloudSqlConfigs) {
+    ParsedPersistenceXmlDescriptor descriptor =
+        PersistenceXmlUtility.getParsedPersistenceXmlDescriptor();
+
+    List<String> updatedManagedClasses =
+        Streams.concat(
+                descriptor.getManagedClassNames().stream(),
+                BulkQueryEntities.JPA_ENTITIES_NEW.stream())
+            .map(
+                name -> {
+                  if (BulkQueryEntities.JPA_ENTITIES_REPLACEMENTS.containsKey(name)) {
+                    return BulkQueryEntities.JPA_ENTITIES_REPLACEMENTS.get(name);
+                  }
+                  return name;
+                })
+            .collect(ImmutableList.toImmutableList());
+
+    descriptor.getManagedClassNames().clear();
+    descriptor.getManagedClassNames().addAll(updatedManagedClasses);
+
+    return Bootstrap.getEntityManagerFactoryBuilder(descriptor, cloudSqlConfigs).build();
+  }
+
+  public static JpaTransactionManager createBulkQueryJpaTransactionManager(
+      ImmutableMap<String, String> cloudSqlConfigs, Clock clock) {
+    return new JpaTransactionManagerImpl(
+        createBulkQueryEntityManagerFactory(cloudSqlConfigs), clock);
+  }
+}

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -152,13 +152,36 @@ public abstract class PersistenceModule {
   @Singleton
   @BeamPipelineCloudSqlConfigs
   static ImmutableMap<String, String> provideBeamPipelineCloudSqlConfigs(
-      @Config("beamCloudSqlJdbcUrl") String jdbcUrl,
-      @Config("beamCloudSqlInstanceConnectionName") String instanceConnectionName,
-      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs,
+      SqlCredentialStore credentialStore,
+      @Config("instanceConnectionNameOverride")
+          Optional<Provider<String>> instanceConnectionNameOverride,
       @Config("beamIsolationOverride")
-          Optional<Provider<TransactionIsolationLevel>> isolationOverride) {
-    return createPartialSqlConfigs(
-        jdbcUrl, instanceConnectionName, defaultConfigs, isolationOverride);
+          Optional<Provider<TransactionIsolationLevel>> isolationOverride,
+      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs) {
+    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
+    // TODO(b/175700623): make sql username configurable from config file.
+    SqlCredential credential = credentialStore.getCredential(new RobotUser(RobotId.NOMULUS));
+    overrides.put(Environment.USER, credential.login());
+    overrides.put(Environment.PASS, credential.password());
+    // Override the default minimum which is tuned for the Registry server. A worker VM should
+    // release all connections if it no longer interacts with the database.
+    overrides.put(HIKARI_MINIMUM_IDLE, "0");
+    /**
+     * Disable Hikari's maxPoolSize limit check by setting it to an absurdly large number. The
+     * effective (and desirable) limit is the number of pipeline threads on the pipeline worker,
+     * which can be configured using pipeline options. See {@link RegistryPipelineOptions} for more
+     * information.
+     */
+    overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(Integer.MAX_VALUE));
+    instanceConnectionNameOverride
+        .map(Provider::get)
+        .ifPresent(
+            instanceConnectionName ->
+                overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName));
+    isolationOverride
+        .map(Provider::get)
+        .ifPresent(isolation -> overrides.put(Environment.ISOLATION, isolation.name()));
+    return ImmutableMap.copyOf(overrides);
   }
 
   @VisibleForTesting
@@ -230,37 +253,17 @@ public abstract class PersistenceModule {
   @Singleton
   @BeamJpaTm
   static JpaTransactionManager provideBeamJpaTm(
-      SqlCredentialStore credentialStore,
-      @Config("instanceConnectionNameOverride")
-          Optional<Provider<String>> instanceConnectionNameOverride,
-      @Config("beamIsolationOverride")
-          Optional<Provider<TransactionIsolationLevel>> isolationOverride,
-      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
-      Clock clock) {
-    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
-    // TODO(b/175700623): make sql username configurable from config file.
-    SqlCredential credential = credentialStore.getCredential(new RobotUser(RobotId.NOMULUS));
-    overrides.put(Environment.USER, credential.login());
-    overrides.put(Environment.PASS, credential.password());
-    // Override the default minimum which is tuned for the Registry server. A worker VM should
-    // release all connections if it no longer interacts with the database.
-    overrides.put(HIKARI_MINIMUM_IDLE, "0");
-    /**
-     * Disable Hikari's maxPoolSize limit check by setting it to an absurdly large number. The
-     * effective (and desirable) limit is the number of pipeline threads on the pipeline worker,
-     * which can be configured using pipeline options. See {@link RegistryPipelineOptions} for more
-     * information.
-     */
-    overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(Integer.MAX_VALUE));
-    instanceConnectionNameOverride
-        .map(Provider::get)
-        .ifPresent(
-            instanceConnectionName ->
-                overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName));
-    isolationOverride
-        .map(Provider::get)
-        .ifPresent(isolation -> overrides.put(Environment.ISOLATION, isolation.name()));
-    return new JpaTransactionManagerImpl(create(overrides), clock);
+      @BeamPipelineCloudSqlConfigs ImmutableMap<String, String> beamCloudSqlConfigs, Clock clock) {
+    return new JpaTransactionManagerImpl(create(beamCloudSqlConfigs), clock);
+  }
+
+  @Provides
+  @Singleton
+  @BeamBulkQueryJpaTm
+  static JpaTransactionManager provideBeamBulkQueryJpaTm(
+      @BeamPipelineCloudSqlConfigs ImmutableMap<String, String> beamCloudSqlConfigs, Clock clock) {
+    return new JpaTransactionManagerImpl(
+        BulkQueryJpaFactory.createBulkQueryEntityManagerFactory(beamCloudSqlConfigs), clock);
   }
 
   @Provides
@@ -346,6 +349,17 @@ public abstract class PersistenceModule {
     }
   }
 
+  /** Types of {@link JpaTransactionManager JpaTransactionManagers}. */
+  public enum JpaTransactionManagerType {
+    /** The regular {@link JpaTransactionManager} for general use. */
+    REGULAR,
+    /**
+     * The {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities. Please
+     * see {@link google.registry.model.bulkquery.BulkQueryEntities} for more information.
+     */
+    BULK_QUERY
+  }
+
   /** Dagger qualifier for JDBC {@link Connection} with schema management privilege. */
   @Qualifier
   @Documented
@@ -357,11 +371,18 @@ public abstract class PersistenceModule {
   @interface AppEngineJpaTm {}
 
   /** Dagger qualifier for {@link JpaTransactionManager} used inside BEAM pipelines. */
-  // Note: @SocketFactoryJpaTm will be phased out in favor of this qualifier.
   @Qualifier
   @Documented
   public @interface BeamJpaTm {}
 
+  /**
+   * Dagger qualifier for {@link JpaTransactionManager} that uses an alternative entity model for
+   * faster bulk queries.
+   */
+  @Qualifier
+  @Documented
+  public @interface BeamBulkQueryJpaTm {}
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for Nomulus tool. */
   @Qualifier
   @Documented

core/src/main/java/google/registry/persistence/VKey.java

@@ -18,10 +18,13 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.ImmutableObject;
 import google.registry.model.translators.VKeyTranslatorFactory;
+import google.registry.util.SerializeUtils;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -36,6 +39,15 @@ public class VKey<T> extends ImmutableObject implements Serializable {
 
   private static final long serialVersionUID = -5291472863840231240L;
 
+  // Info that's stored in in vkey string generated via stringify().
+  private static final String SQL_LOOKUP_KEY = "sql";
+  private static final String OFY_LOOKUP_KEY = "ofy";
+  private static final String CLASS_TYPE = "kind";
+
+  // Web safe delimiters that won't be used in base 64.
+  private static final String KV_SEPARATOR = ":";
+  private static final String DELIMITER = "@";
+
   // The SQL key for the referenced entity.
   Serializable sqlKey;
 
@@ -114,6 +126,47 @@ public class VKey<T> extends ImmutableObject implements Serializable {
     return new VKey<T>(kind, Key.create(kind, name), name);
   }
 
+  /**
+   * Constructs a {@link VKey} from the string representation of a vkey.
+   *
+   * <p>There are two types of string representations: 1) existing ofy key string handled by
+   * fromWebsafeKey() and 2) string encoded via stringify() where @ separates the substrings and
+   * each of the substrings contains a look up key, ":", and its corresponding value. The key info
+   * is encoded via Base64. The string begins with "kind:" and it must contains at least ofy key or
+   * sql key.
+   *
+   * <p>Example of a Vkey string by fromWebsafeKey(): "agR0ZXN0chYLEgpEb21haW5CYXNlIgZST0lELTEM"
+   *
+   * <p>Example of a vkey string by stringify(): "google.registry.testing.TestObject@sql:rO0ABX" +
+   * "QAA2Zvbw@ofy:agR0ZXN0cjELEg9FbnRpdHlHcm91cFJvb3QiCWNyb3NzLXRsZAwLEgpUZXN0T2JqZWN0IgNmb28M",
+   * where sql key and ofy key are values are encoded in Base64.
+   */
+  public static <T> VKey<T> create(String keyString) throws Exception {
+    if (!keyString.startsWith(CLASS_TYPE + KV_SEPARATOR)) {
+      // to handle the existing ofy key string
+      return fromWebsafeKey(keyString);
+    } else {
+      ImmutableMap<String, String> kvs =
+          ImmutableMap.copyOf(
+              Splitter.on(DELIMITER).withKeyValueSeparator(KV_SEPARATOR).split(keyString));
+      Class classType = Class.forName(kvs.get(CLASS_TYPE));
+
+      if (kvs.containsKey(SQL_LOOKUP_KEY) && kvs.containsKey(OFY_LOOKUP_KEY)) {
+        return VKey.create(
+            classType,
+            SerializeUtils.parse(Serializable.class, kvs.get(SQL_LOOKUP_KEY)),
+            Key.create(kvs.get(OFY_LOOKUP_KEY)));
+      } else if (kvs.containsKey(SQL_LOOKUP_KEY)) {
+        return VKey.createSql(
+            classType, SerializeUtils.parse(Serializable.class, kvs.get(SQL_LOOKUP_KEY)));
+      } else if (kvs.containsKey(OFY_LOOKUP_KEY)) {
+        return VKey.createOfy(classType, Key.create(kvs.get(OFY_LOOKUP_KEY)));
+      } else {
+        throw new IllegalArgumentException(String.format("Cannot parse key string: %s", keyString));
+      }
+    }
+  }
+
   /**
    * Returns a clone with an ofy key restored from {@code ancestors}.
    *
@@ -233,4 +286,29 @@ public class VKey<T> extends ImmutableObject implements Serializable {
   public static <T> VKey<T> fromWebsafeKey(String ofyKeyRepr) {
     return from(Key.create(ofyKeyRepr));
   }
+
+  /**
+   * Constructs the string representation of a {@link VKey}.
+   *
+   * <p>The string representation of a vkey contains its type, and sql key or ofy key, or both. Each
+   * of the keys is first serialized into a byte array then encoded via Base64 into a web safe
+   * string.
+   *
+   * <p>The string representation of a vkey contains key values pairs separated by delimiter "@".
+   * Another delimiter ":" is put in between each key and value. The following is the complete
+   * format of the string: "kind:class_name@sql:encoded_sqlKey@ofy:encoded_ofyKey", where kind is
+   * required. The string representation may contain an encoded ofy key, or an encoded sql key, or
+   * both.
+   */
+  public String stringify() {
+    // class type is required to create a vkey
+    String key = CLASS_TYPE + KV_SEPARATOR + getKind().getName();
+    if (maybeGetSqlKey().isPresent()) {
+      key += DELIMITER + SQL_LOOKUP_KEY + KV_SEPARATOR + SerializeUtils.stringify(getSqlKey());
+    }
+    if (maybeGetOfyKey().isPresent()) {
+      key += DELIMITER + OFY_LOOKUP_KEY + KV_SEPARATOR + getOfyKey().getString();
+    }
+    return key;
+  }
 }

core/src/main/java/google/registry/persistence/converter/CreateAutoTimestampConverter.java

@@ -1,53 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package google.registry.persistence.converter;
-
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-
-import google.registry.model.CreateAutoTimestamp;
-import google.registry.util.DateTimeUtils;
-import java.sql.Timestamp;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-import javax.annotation.Nullable;
-import javax.persistence.AttributeConverter;
-import javax.persistence.Converter;
-import org.joda.time.DateTime;
-
-/** JPA converter to for storing/retrieving CreateAutoTimestamp objects. */
-@Converter(autoApply = true)
-public class CreateAutoTimestampConverter
-    implements AttributeConverter<CreateAutoTimestamp, Timestamp> {
-
-  @Override
-  @Nullable
-  public Timestamp convertToDatabaseColumn(@Nullable CreateAutoTimestamp entity) {
-    if (entity == null) {
-      return null;
-    }
-    DateTime dateTime = firstNonNull(entity.getTimestamp(), jpaTm().getTransactionTime());
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(dateTime).toInstant());
-  }
-
-  @Override
-  @Nullable
-  public CreateAutoTimestamp convertToEntityAttribute(@Nullable Timestamp columnValue) {
-    if (columnValue == null) {
-      return null;
-    }
-    ZonedDateTime zdt = ZonedDateTime.ofInstant(columnValue.toInstant(), ZoneOffset.UTC);
-    return CreateAutoTimestamp.create(DateTimeUtils.toJodaDateTime(zdt));
-  }
-}

core/src/main/java/google/registry/persistence/converter/TimedTransitionPropertyConverterBase.java

@@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.persistence.converter.StringMapDescriptor.StringMap;
+import java.io.Serializable;
 import java.util.Map;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
@@ -30,7 +31,8 @@ import org.joda.time.DateTime;
  * Base JPA converter for {@link TimedTransitionProperty} objects that are stored in a column with
  * data type of hstore in the database.
  */
-public abstract class TimedTransitionPropertyConverterBase<K, V extends TimedTransition<K>>
+public abstract class TimedTransitionPropertyConverterBase<
+        K extends Serializable, V extends TimedTransition<K>>
     implements AttributeConverter<TimedTransitionProperty<K, V>, StringMap> {
 
   abstract Map.Entry<String, String> convertToDatabaseMapEntry(Map.Entry<DateTime, V> entry);

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManager.java

@@ -24,7 +24,33 @@ import javax.persistence.criteria.CriteriaQuery;
 /** Sub-interface of {@link TransactionManager} which defines JPA related methods. */
 public interface JpaTransactionManager extends TransactionManager {
 
-  /** Returns the {@link EntityManager} for the current request. */
+  /**
+   * Returns a long-lived {@link EntityManager} not bound to a particular transaction.
+   *
+   * <p>Caller is responsible for closing the returned instance.
+   */
+  EntityManager getStandaloneEntityManager();
+
+  /**
+   * Specifies a database snapshot exported by another transaction to use in the current
+   * transaction.
+   *
+   * <p>This is a Postgresql-specific feature. This method must be called before any other SQL
+   * commands in a transaction.
+   *
+   * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to
+   * avoid exhausting predicate locks at the SERIALIZABLE level.
+   *
+   * @see google.registry.beam.common.DatabaseSnapshot
+   */
+  // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+  JpaTransactionManager setDatabaseSnapshot(String snapshotId);
+
+  /**
+   * Returns the {@link EntityManager} for the current request.
+   *
+   * <p>The returned instance is closed when the current transaction completes.
+   */
   EntityManager getEntityManager();
 
   /**

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -120,6 +120,11 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     emf.close();
   }
 
+  @Override
+  public EntityManager getStandaloneEntityManager() {
+    return emf.createEntityManager();
+  }
+
   @Override
   public EntityManager getEntityManager() {
     EntityManager entityManager = transactionInfo.get().entityManager;
@@ -131,6 +136,22 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     return entityManager;
   }
 
+  @Override
+  public JpaTransactionManager setDatabaseSnapshot(String snapshotId) {
+    // Postgresql-specific: 'set transaction' command must be called inside a transaction
+    assertInTransaction();
+
+    EntityManager entityManager = getEntityManager();
+    // Isolation is hardcoded to REPEATABLE READ, as specified by parent's Javadoc.
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+    entityManager
+        .createNativeQuery(String.format("SET TRANSACTION SNAPSHOT '%s'", snapshotId))
+        .executeUpdate();
+    return this;
+  }
+
   @Override
   public <T> TypedQuery<T> query(String sqlString, Class<T> resultClass) {
     return new DetachingTypedQuery<>(getEntityManager().createQuery(sqlString, resultClass));
@@ -548,7 +569,6 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     }
     EntityType<?> entityType = getEntityType(key.getKind());
     ImmutableSet<EntityId> entityIds = getEntityIdsFromSqlKey(entityType, key.getSqlKey());
-    // TODO(b/179158393): use Criteria for query to leave not doubt about sql injection risk.
     String sql =
         String.format("DELETE FROM %s WHERE %s", entityType.getName(), getAndClause(entityIds));
     Query query = query(sql);

core/src/main/java/google/registry/privileges/secretmanager/SqlUser.java

@@ -54,7 +54,10 @@ public abstract class SqlUser {
      * Credential for RegistryTool. This is temporary, and will be removed when tool users are
      * assigned their personal credentials.
      */
-    TOOL
+    TOOL,
+
+    /** The Postgres admin account. */
+    POSTGRES
   }
 
   /** Information of a RobotUser for privilege management purposes. */

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -14,12 +14,14 @@
 
 package google.registry.rde;
 
+import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
 import static google.registry.beam.BeamUtils.createJobName;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.xml.ValidationMode.LENIENT;
 import static google.registry.xml.ValidationMode.STRICT;
+import static java.util.function.Function.identity;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -29,6 +31,7 @@ import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
 import com.google.common.base.Ascii;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -48,6 +51,7 @@ import google.registry.model.EppResource;
 import google.registry.model.common.Cursor;
 import google.registry.model.common.Cursor.CursorType;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.rde.RdeMode;
@@ -67,14 +71,17 @@ import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /**
- * MapReduce that idempotently stages escrow deposit XML files on GCS for RDE/BRDA for all TLDs.
+ * Action that kicks off either a MapReduce (for Datastore) or Dataflow (for Cloud SQL) job to stage
+ * escrow deposit XML files on GCS for RDE/BRDA for all TLDs.
  *
- * <h3>MapReduce Operation</h3>
+ * <h3>Pending Deposits</h3>
  *
  * <p>This task starts by asking {@link PendingDepositChecker} which deposits need to be generated.
  * If there's nothing to deposit, we return 204 No Content; otherwise, we fire off a MapReduce job
  * and redirect to its status GUI. The task can also be run in manual operation, as described below.
  *
+ * <h3>MapReduce</h3>
+ *
  * <p>The mapreduce job scans every {@link EppResource} in Datastore. It maps a point-in-time
  * representation of each entity to the escrow XML files in which it should appear.
  *
@@ -88,11 +95,26 @@ import org.joda.time.Duration;
  * <p>{@link Registrar} entities, both active and inactive, are included in all deposits. They are
  * not rewinded point-in-time.
  *
+ * <h3>Dataflow</h3>
+ *
+ * The Dataflow job finds the most recent history entry on or before watermark for each resource
+ * type and loads the embedded resource from it, which is then projected to watermark time to
+ * account for things like pending transfer.
+ *
+ * <p>Only {@link ContactResource}s and {@link HostResource}s that are referenced by an included
+ * {@link DomainBase} will be included in the corresponding pending deposit.
+ *
+ * <p>{@link Registrar} entities, both active and inactive, are included in all deposits. They are
+ * not rewinded point-in-time.
+ *
+ * <h3>Afterward</h3>
+ *
  * <p>The XML deposit files generated by this job are humongous. A tiny XML report file is generated
  * for each deposit, telling us how much of what it contains.
  *
- * <p>Once a deposit is successfully generated, an {@link RdeUploadAction} is enqueued which will
- * upload it via SFTP to the third-party escrow provider.
+ * <p>Once a deposit is successfully generated, For RDE an {@link RdeUploadAction} is enqueued which
+ * will upload it via SFTP to the third-party escrow provider; for BRDA an {@link BrdaCopyAction} is
+ * enqueued which will copy it to a GCS bucket and be rsynced to a third-party escrow provider.
  *
  * <p>To generate escrow deposits manually and locally, use the {@code nomulus} tool command {@code
  * GenerateEscrowDepositCommand}.
@@ -106,7 +128,7 @@ import org.joda.time.Duration;
  *
  * <p>Valid model objects might not be valid to the RDE XML schema. A single invalid object will
  * cause the whole deposit to fail. You need to check the logs, find out which entities are broken,
- * and perform Datastore surgery.
+ * and perform database surgery.
  *
  * <p>If a deposit fails, an error is emitted to the logs for each broken entity. It tells you the
  * key and shows you its representation in lenient XML.
@@ -137,33 +159,40 @@ import org.joda.time.Duration;
  * <p>The deposit and report are encrypted using {@link Ghostryde}. Administrators can use the
  * {@code GhostrydeCommand} command in the {@code nomulus} tool to view them.
  *
- * <p>Unencrypted XML fragments are stored temporarily between the map and reduce steps. The
- * ghostryde encryption on the full archived deposits makes life a little more difficult for an
- * attacker. But security ultimately depends on the bucket.
+ * <p>Unencrypted XML fragments are stored temporarily between the map and reduce steps and between
+ * Dataflow transforms. The ghostryde encryption on the full archived deposits makes life a little
+ * more difficult for an attacker. But security ultimately depends on the bucket.
  *
  * <h3>Idempotency</h3>
  *
- * <p>We lock the reduce tasks. This is necessary because: a) App Engine tasks might get double
- * executed; and b) Cloud Storage file handles get committed on close <i>even if our code throws an
- * exception.</i>
+ * <p>We lock the reduce tasks for the MapReduce job. This is necessary because: a) App Engine tasks
+ * might get double executed; and b) Cloud Storage file handles get committed on close <i>even if
+ * our code throws an exception.</i>
+ *
+ * <p>For the Dataflow job we do not employ a lock because it is difficult to span a lock across
+ * three subsequent transforms (save to GCS, roll forward cursor, enqueue next action). Instead, we
+ * get around the issue by saving the deposit to a unique folder named after the job name so there
+ * is no possibility of overwriting.
  *
  * <p>Deposits are generated serially for a given (watermark, mode) pair. A deposit is never started
  * beyond the cursor. Once a deposit is completed, its cursor is rolled forward transactionally.
  * Duplicate jobs may exist {@code <=cursor}. So a transaction will not bother changing the cursor
  * if it's already been rolled forward.
  *
- * <p>Enqueuing {@code RdeUploadAction} is also part of the cursor transaction. This is necessary
- * because the first thing the upload task does is check the staging cursor to verify it's been
- * completed, so we can't enqueue before we roll. We also can't enqueue after the roll, because then
- * if enqueuing fails, the upload might never be enqueued.
+ * <p>Enqueuing {@code RdeUploadAction} or {@code BrdaCopyAction} is also part of the cursor
+ * transaction. This is necessary because the first thing the upload task does is check the staging
+ * cursor to verify it's been completed, so we can't enqueue before we roll. We also can't enqueue
+ * after the roll, because then if enqueuing fails, the upload might never be enqueued.
  *
  * <h3>Determinism</h3>
  *
  * <p>The filename of an escrow deposit is determistic for a given (TLD, watermark, {@linkplain
  * RdeMode mode}) triplet. Its generated contents is deterministic in all the ways that we care
- * about. Its view of the database is strongly consistent.
+ * about. Its view of the database is strongly consistent in Cloud SQL automatically by nature of
+ * the initial query for the history entry running at {@code READ_COMMITTED} transaction isolation
+ * level.
  *
- * <p>This is because:
+ * <p>This is also true in Datastore because:
  *
  * <ol>
  *   <li>{@code EppResource} queries are strongly consistent thanks to {@link EppResourceIndex}
@@ -226,6 +255,11 @@ public final class RdeStagingAction implements Runnable {
   @Inject MapreduceRunner mrRunner;
   @Inject @Config("projectId") String projectId;
   @Inject @Config("defaultJobRegion") String jobRegion;
+
+  @Inject
+  @Config("highPerformanceMachineType")
+  String machineType;
+
   @Inject @Config("transactionCooldown") Duration transactionCooldown;
   @Inject @Config("beamStagingBucketUrl") String stagingBucketUrl;
   @Inject @Config("rdeBucket") String rdeBucket;
@@ -270,43 +304,65 @@ public final class RdeStagingAction implements Runnable {
                   new NullInput<>(), EppResourceInputs.createEntityInput(EppResource.class)))
           .sendLinkToMapreduceConsole(response);
     } else {
-      try {
-        LaunchFlexTemplateParameter parameter =
-            new LaunchFlexTemplateParameter()
-                .setJobName(createJobName("rde", clock))
-                .setContainerSpecGcsPath(
-                    String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-                .setParameters(
-                    ImmutableMap.of(
-                        "pendings",
-                        RdePipeline.encodePendings(pendings),
-                        "validationMode",
-                        validationMode.name(),
-                        "rdeStagingBucket",
-                        rdeBucket,
-                        "stagingKey",
-                        BaseEncoding.base64Url().omitPadding().encode(stagingKeyBytes),
-                        "registryEnvironment",
-                        RegistryEnvironment.get().name()));
-        LaunchFlexTemplateResponse launchResponse =
-            dataflow
-                .projects()
-                .locations()
-                .flexTemplates()
-                .launch(
-                    projectId,
-                    jobRegion,
-                    new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
-                .execute();
-        logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
-        response.setStatus(SC_OK);
-        response.setPayload(
-            String.format("Launched RDE pipeline: %s", launchResponse.getJob().getId()));
-      } catch (IOException e) {
-        logger.atWarning().withCause(e).log("Pipeline Launch failed");
-        response.setStatus(SC_INTERNAL_SERVER_ERROR);
-        response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
-      }
+      ImmutableList.Builder<String> jobNameBuilder = new ImmutableList.Builder<>();
+      pendings.values().stream()
+          .collect(toImmutableSetMultimap(PendingDeposit::watermark, identity()))
+          .asMap()
+          .forEach(
+              (watermark, pendingDeposits) -> {
+                try {
+                  LaunchFlexTemplateParameter parameter =
+                      new LaunchFlexTemplateParameter()
+                          .setJobName(
+                              createJobName(
+                                  String.format(
+                                      "rde-%s", watermark.toString("yyyy-MM-dd't'HH-mm-ss'z'")),
+                                  clock))
+                          .setContainerSpecGcsPath(
+                              String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+                          .setParameters(
+                              new ImmutableMap.Builder<String, String>()
+                                  .put(
+                                      "pendings",
+                                      RdePipeline.encodePendingDeposits(
+                                          ImmutableSet.copyOf(pendingDeposits)))
+                                  .put("validationMode", validationMode.name())
+                                  .put("rdeStagingBucket", rdeBucket)
+                                  .put(
+                                      "stagingKey",
+                                      BaseEncoding.base64Url()
+                                          .omitPadding()
+                                          .encode(stagingKeyBytes))
+                                  .put("registryEnvironment", RegistryEnvironment.get().name())
+                                  .put("workerMachineType", machineType)
+                                  // TODO (jianglai): Investigate turning off public IPs (for which
+                                  // there is a quota) in order to increase the total number of
+                                  // workers allowed (also under quota).
+                                  // See:
+                                  // https://cloud.google.com/dataflow/docs/guides/routes-firewall
+                                  .put("usePublicIps", "true")
+                                  .build());
+                  LaunchFlexTemplateResponse launchResponse =
+                      dataflow
+                          .projects()
+                          .locations()
+                          .flexTemplates()
+                          .launch(
+                              projectId,
+                              jobRegion,
+                              new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
+                          .execute();
+                  logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
+                  jobNameBuilder.add(launchResponse.getJob().getId());
+                } catch (IOException e) {
+                  logger.atWarning().withCause(e).log("Pipeline Launch failed");
+                  response.setStatus(SC_INTERNAL_SERVER_ERROR);
+                  response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
+                }
+              });
+      response.setStatus(SC_OK);
+      response.setPayload(
+          String.format("Launched RDE pipeline: %s", Joiner.on(", ").join(jobNameBuilder.build())));
     }
   }
 
@@ -349,7 +405,7 @@ public final class RdeStagingAction implements Runnable {
       throw new BadRequestException("Directory must not start with a slash");
     }
     String directoryWithTrailingSlash =
-        directory.get().endsWith("/") ? directory.get() : (directory.get() + '/');
+        directory.get().endsWith("/") ? directory.get() : directory.get() + '/';
 
     if (modeStrings.isEmpty()) {
       throw new BadRequestException("Mode parameter required in manual operation");
@@ -381,7 +437,7 @@ public final class RdeStagingAction implements Runnable {
       }
     }
 
-    if (revision.isPresent() && (revision.get() < 0)) {
+    if (revision.isPresent() && revision.get() < 0) {
       throw new BadRequestException("Revision must be greater than or equal to zero");
     }
 
@@ -394,11 +450,7 @@ public final class RdeStagingAction implements Runnable {
           pendingsBuilder.put(
               tld,
               PendingDeposit.createInManualOperation(
-                  tld,
-                  watermark,
-                  mode,
-                  directoryWithTrailingSlash,
-                  revision.orElse(null)));
+                  tld, watermark, mode, directoryWithTrailingSlash, revision.orElse(null)));
         }
       }
     }

core/src/main/java/google/registry/reporting/ReportingModule.java

@@ -14,6 +14,7 @@
 
 package google.registry.reporting;
 
+import static google.registry.request.RequestParameters.extractOptionalBooleanParameter;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 
@@ -55,6 +56,12 @@ public class ReportingModule {
   /** The request parameter specifying the jobId for a running Dataflow pipeline. */
   public static final String PARAM_JOB_ID = "jobId";
 
+  /**
+   * The request parameter name used by actions to indicate if an email should be sent. This
+   * parameter defaults to true if not specified.
+   */
+  public static final String SEND_EMAIL = "email";
+
   /** Provides the Cloud Dataflow jobId for a pipeline. */
   @Provides
   @Parameter(PARAM_JOB_ID)
@@ -62,6 +69,13 @@ public class ReportingModule {
     return extractRequiredParameter(req, PARAM_JOB_ID);
   }
 
+  /** Provides the boolean value indicating if emails should be sent. */
+  @Provides
+  @Parameter(SEND_EMAIL)
+  static boolean provideSendEmail(HttpServletRequest req) {
+    return extractOptionalBooleanParameter(req, SEND_EMAIL).orElse(true);
+  }
+
   /** Extracts an optional YearMonth in yyyy-MM format from the request. */
   @Provides
   @Parameter(PARAM_YEAR_MONTH)

core/src/main/java/google/registry/reporting/billing/PublishInvoicesAction.java

@@ -23,16 +23,17 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 
 import com.google.api.services.dataflow.Dataflow;
 import com.google.api.services.dataflow.model.Job;
-import com.google.appengine.api.taskqueue.QueueFactory;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.reporting.ReportingModule;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
+import google.registry.util.CloudTasksUtils;
 import java.io.IOException;
 import javax.inject.Inject;
 import org.joda.time.YearMonth;
@@ -65,6 +66,7 @@ public class PublishInvoicesAction implements Runnable {
   private final Dataflow dataflow;
   private final Response response;
   private final YearMonth yearMonth;
+  private final CloudTasksUtils cloudTasksUtils;
 
   @Inject
   PublishInvoicesAction(
@@ -74,7 +76,8 @@ public class PublishInvoicesAction implements Runnable {
       BillingEmailUtils emailUtils,
       Dataflow dataflow,
       Response response,
-      YearMonth yearMonth) {
+      YearMonth yearMonth,
+      CloudTasksUtils cloudTasksUtils) {
     this.projectId = projectId;
     this.jobRegion = jobRegion;
     this.jobId = jobId;
@@ -82,6 +85,7 @@ public class PublishInvoicesAction implements Runnable {
     this.dataflow = dataflow;
     this.response = response;
     this.yearMonth = yearMonth;
+    this.cloudTasksUtils = cloudTasksUtils;
   }
 
   static final String PATH = "/_dr/task/publishInvoices";
@@ -119,10 +123,11 @@ public class PublishInvoicesAction implements Runnable {
   }
 
   private void enqueueCopyDetailReportsTask() {
-    TaskOptions copyDetailTask =
-        TaskOptions.Builder.withUrl(CopyDetailReportsAction.PATH)
-            .method(TaskOptions.Method.POST)
-            .param(PARAM_YEAR_MONTH, yearMonth.toString());
-    QueueFactory.getQueue(BillingModule.CRON_QUEUE).add(copyDetailTask);
+    cloudTasksUtils.enqueue(
+        BillingModule.CRON_QUEUE,
+        CloudTasksUtils.createPostTask(
+            CopyDetailReportsAction.PATH,
+            Service.BACKEND.toString(),
+            ImmutableMultimap.of(PARAM_YEAR_MONTH, yearMonth.toString())));
   }
 }

core/src/main/java/google/registry/reporting/icann/DnsCountQueryCoordinator.java

@@ -35,14 +35,19 @@ public interface DnsCountQueryCoordinator {
    * <p>If your report query requires any additional parameters, add them here.
    */
   class Params {
+
     public BigqueryConnection bigquery;
 
     /** The Google Cloud project id. */
     public String projectId;
 
-    public Params(BigqueryConnection bigquery, String projectId) {
+    /** The BigQuery dataset from which to query. */
+    public String icannReportingDataSet;
+
+    public Params(BigqueryConnection bigquery, String projectId, String icannReportingDataSet) {
       this.bigquery = bigquery;
       this.projectId = projectId;
+      this.icannReportingDataSet = icannReportingDataSet;
     }
   }
 

core/src/main/java/google/registry/reporting/icann/DnsCountQueryCoordinatorModule.java

@@ -14,6 +14,7 @@
 
 package google.registry.reporting.icann;
 
+import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
 import static google.registry.util.TypeUtils.getClassFromString;
 import static google.registry.util.TypeUtils.instantiate;
 
@@ -21,6 +22,7 @@ import dagger.Module;
 import dagger.Provides;
 import google.registry.bigquery.BigqueryConnection;
 import google.registry.config.RegistryConfig.Config;
+import javax.inject.Named;
 
 /** Dagger module to provide the DnsCountQueryCoordinator. */
 @Module
@@ -30,9 +32,10 @@ public class DnsCountQueryCoordinatorModule {
   static DnsCountQueryCoordinator provideDnsCountQueryCoordinator(
       @Config("dnsCountQueryCoordinatorClass") String customClass,
       BigqueryConnection bigquery,
-      @Config("projectId") String projectId) {
+      @Config("projectId") String projectId,
+      @Named(ICANN_REPORTING_DATA_SET) String icannReportingDataSet) {
     DnsCountQueryCoordinator.Params params =
-        new DnsCountQueryCoordinator.Params(bigquery, projectId);
+        new DnsCountQueryCoordinator.Params(bigquery, projectId, icannReportingDataSet);
     return instantiate(getClassFromString(customClass, DnsCountQueryCoordinator.class), params);
   }
 }

core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java

@@ -14,6 +14,7 @@
 
 package google.registry.reporting.icann;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
 import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
 import static google.registry.reporting.icann.QueryBuilderUtils.getQueryFromFile;
@@ -71,102 +72,180 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
     DateTime latestReportTime = earliestReportTime.plusMonths(1).minusMillis(1);
 
     ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
-    String registrarIanaIdQuery =
-        SqlTemplate.create(getQueryFromFile("registrar_iana_id.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .build();
+    String registrarIanaIdQuery;
+    if (tm().isOfy()) {
+      registrarIanaIdQuery =
+          SqlTemplate.create(getQueryFromFile("registrar_iana_id.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .build();
+    } else {
+      registrarIanaIdQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_registrar_iana_id.sql"))
+              .put("PROJECT_ID", projectId)
+              .build();
+    }
     queriesBuilder.put(getTableName(REGISTRAR_IANA_ID, yearMonth), registrarIanaIdQuery);
 
-    String totalDomainsQuery =
-        SqlTemplate.create(getQueryFromFile("total_domains.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("DOMAINBASE_TABLE", "DomainBase")
-            .put("REGISTRAR_TABLE", "Registrar")
-            .build();
+    String totalDomainsQuery;
+    if (tm().isOfy()) {
+      totalDomainsQuery =
+          SqlTemplate.create(getQueryFromFile("total_domains.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("DOMAINBASE_TABLE", "DomainBase")
+              .put("REGISTRAR_TABLE", "Registrar")
+              .build();
+    } else {
+      totalDomainsQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_total_domains.sql"))
+              .put("PROJECT_ID", projectId)
+              .build();
+    }
     queriesBuilder.put(getTableName(TOTAL_DOMAINS, yearMonth), totalDomainsQuery);
 
     DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
-    String totalNameserversQuery =
-        SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("HOSTRESOURCE_TABLE", "HostResource")
-            .put("DOMAINBASE_TABLE", "DomainBase")
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
-            .build();
+    String totalNameserversQuery;
+    if (tm().isOfy()) {
+      totalNameserversQuery =
+          SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("HOSTRESOURCE_TABLE", "HostResource")
+              .put("DOMAINBASE_TABLE", "DomainBase")
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    } else {
+      totalNameserversQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_total_nameservers.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    }
     queriesBuilder.put(getTableName(TOTAL_NAMESERVERS, yearMonth), totalNameserversQuery);
 
-    String transactionCountsQuery =
-        SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("HISTORYENTRY_TABLE", "HistoryEntry")
-            .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
-            .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
-            .put("CLIENT_ID", "clientId")
-            .put("OTHER_CLIENT_ID", "otherClientId")
-            .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_GAINING_SUCCESSFUL")
-            .put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED")
-            .put("DEFAULT_FIELD", "field")
-            .build();
+    String transactionCountsQuery;
+    if (tm().isOfy()) {
+      transactionCountsQuery =
+          SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("HISTORYENTRY_TABLE", "HistoryEntry")
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .put("CLIENT_ID", "clientId")
+              .put("OTHER_CLIENT_ID", "otherClientId")
+              .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_GAINING_SUCCESSFUL")
+              .put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED")
+              .put("DEFAULT_FIELD", "field")
+              .build();
+    } else {
+      transactionCountsQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_transaction_counts.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    }
     queriesBuilder.put(getTableName(TRANSACTION_COUNTS, yearMonth), transactionCountsQuery);
 
-    String transactionTransferLosingQuery =
-        SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("HISTORYENTRY_TABLE", "HistoryEntry")
-            .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
-            .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
-            // Roles are reversed for losing queries
-            .put("CLIENT_ID", "otherClientId")
-            .put("OTHER_CLIENT_ID", "clientId")
-            .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_LOSING_SUCCESSFUL")
-            .put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED")
-            .put("DEFAULT_FIELD", "NULL")
-            .build();
+    String transactionTransferLosingQuery;
+    if (tm().isOfy()) {
+      transactionTransferLosingQuery =
+          SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("HISTORYENTRY_TABLE", "HistoryEntry")
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              // Roles are reversed for losing queries
+              .put("CLIENT_ID", "otherClientId")
+              .put("OTHER_CLIENT_ID", "clientId")
+              .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_LOSING_SUCCESSFUL")
+              .put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED")
+              .put("DEFAULT_FIELD", "NULL")
+              .build();
+    } else {
+      transactionTransferLosingQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_transaction_transfer_losing.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    }
     queriesBuilder.put(
         getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth), transactionTransferLosingQuery);
 
     // App Engine log table suffixes use YYYYMMDD format
     DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
-    String attemptedAddsQuery =
-        SqlTemplate.create(getQueryFromFile("attempted_adds.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
-            .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
-            .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
-            .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
-            // All metadata logs for reporting come from google.registry.flows.FlowReporter.
-            .put(
-                "METADATA_LOG_PREFIX",
-                "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
-            .build();
+    String attemptedAddsQuery;
+    if (tm().isOfy()) {
+      attemptedAddsQuery =
+          SqlTemplate.create(getQueryFromFile("attempted_adds.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
+              .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
+              .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
+              .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
+              // All metadata logs for reporting come from google.registry.flows.FlowReporter.
+              .put(
+                  "METADATA_LOG_PREFIX",
+                  "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
+              .build();
+    } else {
+      attemptedAddsQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_attempted_adds.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
+              .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
+              .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
+              .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
+              // All metadata logs for reporting come from google.registry.flows.FlowReporter.
+              .put(
+                  "METADATA_LOG_PREFIX",
+                  "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
+              .build();
+    }
     queriesBuilder.put(getTableName(ATTEMPTED_ADDS, yearMonth), attemptedAddsQuery);
 
-    String aggregateQuery =
-        SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRY_TABLE", "Registry")
-            .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
-            .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID, yearMonth))
-            .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS, yearMonth))
-            .put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS, yearMonth))
-            .put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS, yearMonth))
-            .put(
-                "TRANSACTION_TRANSFER_LOSING_TABLE",
-                getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth))
-            .put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS, yearMonth))
-            .build();
+    String aggregateQuery;
+    if (tm().isOfy()) {
+      aggregateQuery =
+          SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRY_TABLE", "Registry")
+              .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
+              .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID, yearMonth))
+              .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS, yearMonth))
+              .put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS, yearMonth))
+              .put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS, yearMonth))
+              .put(
+                  "TRANSACTION_TRANSFER_LOSING_TABLE",
+                  getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth))
+              .put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS, yearMonth))
+              .build();
+    } else {
+      aggregateQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_transactions_report_aggregation.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
+              .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID, yearMonth))
+              .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS, yearMonth))
+              .put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS, yearMonth))
+              .put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS, yearMonth))
+              .put(
+                  "TRANSACTION_TRANSFER_LOSING_TABLE",
+                  getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth))
+              .put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS, yearMonth))
+              .build();
+    }
     queriesBuilder.put(getTableName(TRANSACTIONS_REPORT_AGGREGATION, yearMonth), aggregateQuery);
 
     return queriesBuilder.build();

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql

@@ -0,0 +1,73 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Determine the number of attempted adds each registrar made.
+
+  -- Since the specification requests all 'attempted' adds, we regex the
+  -- monthly App Engine logs, searching for all create commands and associating
+  -- them with their corresponding registrars.
+
+  -- Example log generated by FlowReporter in App Engine logs:
+  --google.registry.flows.FlowReporter
+  -- recordToLogs: FLOW-LOG-SIGNATURE-METADATA:
+  --{"serverTrid":"oNwL2J2eRya7bh7c9oHIzg==-2360a","clientId":"ipmirror"
+  -- ,"commandType":"hello", "resourceType":"","flowClassName":"HelloFlow"
+  -- ,"targetId":"","targetIds":[],"tld":"",
+  -- "tlds":[],"icannActivityReportField":""}
+
+-- This outer select just converts the registrar's clientId to their name.
+SELECT
+  tld,
+  registrar_table.registrar_name AS registrar_name,
+  'ATTEMPTED_ADDS' AS metricName,
+  count AS metricValue
+FROM (
+  SELECT
+    JSON_EXTRACT_SCALAR(json, '$.tld') AS tld,
+    JSON_EXTRACT_SCALAR(json, '$.clientId') AS clientId,
+    COUNT(json) AS count
+  FROM (
+      -- Extract JSON metadata package from monthly logs
+    SELECT
+      REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+          AS json
+    FROM (
+      SELECT
+        protoPayload.resource AS requestPath,
+        ARRAY(
+          SELECT logMessage
+          FROM UNNEST(protoPayload.line)) AS logMessage
+      FROM
+        `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
+      WHERE _TABLE_SUFFIX
+      BETWEEN '%FIRST_DAY_OF_MONTH%'
+      AND '%LAST_DAY_OF_MONTH%')
+    JOIN UNNEST(logMessage) AS logMessages
+    -- Look for metadata logs from epp and registrar console requests
+    WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
+    AND STARTS_WITH(logMessages, "%METADATA_LOG_PREFIX%")
+    -- Look for domain creates
+    AND REGEXP_CONTAINS(
+       logMessages, r'"commandType":"create","resourceType":"domain"')
+    -- Filter prober data
+    AND NOT REGEXP_CONTAINS(
+        logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') )
+  GROUP BY tld, clientId ) AS logs_table
+JOIN
+  EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+              '''SELECT registrar_id, registrar_name FROM "Registrar";''') AS registrar_table
+  ON
+      logs_table.clientId = registrar_table.registrar_id
+  ORDER BY tld, registrar_name

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_registrar_iana_id.sql

@@ -0,0 +1,37 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Gather a list of all tld-registrar pairs, with their IANA IDs.
+
+  -- This establishes which registrars will appear in the reports.
+
+SELECT
+  allowedTlds AS tld,
+  registrar_name,
+  iana_identifier AS iana_id
+FROM
+  EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+  '''SELECT
+    allowedTlds,
+    registrar_name,
+    iana_identifier
+   FROM
+    "Registrar" AS r,
+    UNNEST(allowed_tlds) AS allowedTlds
+   WHERE
+    r.type='REAL' OR r.type='INTERNAL';''')
+-- Filter out prober data
+WHERE NOT ENDS_WITH(allowedTlds, "test")
+ORDER BY tld, registrar_name

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_total_domains.sql

@@ -0,0 +1,43 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Determine the number of domains each registrar sponsors per tld.
+
+  -- This is just the number of fullyQualifiedDomainNames under each
+  -- tld-registrar pair.
+
+SELECT
+  tld,
+  registrar_name,
+  'TOTAL_DOMAINS' as metricName,
+  COUNT(domain_name) as metricValue
+FROM
+  EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+  '''SELECT
+    registrar_name,
+    registrar_id
+  FROM "Registrar" AS r
+  WHERE r.type='REAL' OR r.type='INTERNAL';''')
+JOIN
+  EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+  '''SELECT
+    tld,
+    current_sponsor_registrar_id,
+    domain_name
+  FROM "Domain" AS d;''')
+ON
+  current_sponsor_registrar_id = registrar_id
+GROUP BY tld, registrar_name
+ORDER BY tld, registrar_name

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_total_nameservers.sql

@@ -0,0 +1,80 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Determine the number of referenced nameservers for a registrar's domains.
+
+  -- We count the number of unique hosts under each tld-registrar combo by
+  -- collecting all domains' listed hosts that were still valid at the
+  -- end of the reporting month.
+
+SELECT
+    tld,
+    registrar_name,
+    'TOTAL_NAMESERVERS' AS metricName,
+    COUNT(host_name) AS metricValue
+FROM
+    EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+    '''SELECT
+        host_name,
+        current_sponsor_registrar_id,
+        creation_time AS host_creation_time,
+        CASE
+          WHEN deletion_time > to_timestamp('9999-12-31 23:59:59', 'YYYY-MM-DD HH24:MI:SS')
+            THEN to_timestamp('9999-12-31 23:59:59', 'YYYY-MM-DD HH24:MI:SS')
+          ELSE
+            deletion_time
+          END as host_deletion_time
+    FROM "Host";''')
+JOIN (
+    SELECT
+    registrar_id,
+    registrar_name
+    FROM
+        EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql", '''SELECT registrar_id, registrar_name FROM "Registrar" AS r WHERE r.type='REAL' OR r.type='INTERNAL';'''))
+ON current_sponsor_registrar_id = registrar_id
+JOIN (
+    SELECT
+        tld,
+        referencedHostName
+    FROM
+        EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+        '''SELECT
+                repo_id,
+                tld,
+                creation_time AS domain_creation_time,
+                CASE
+                  WHEN deletion_time > to_timestamp('9999-12-31 23:59:59', 'YYYY-MM-DD HH24:MI:SS')
+                    THEN to_timestamp('9999-12-31 23:59:59', 'YYYY-MM-DD HH24:MI:SS')
+                  ELSE
+                    deletion_time
+                  END as domain_deletion_time
+            FROM "Domain";''')
+        JOIN
+        EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql", '''SELECT domain_repo_id, host_repo_id FROM "DomainHost";''')
+        ON
+        repo_id = domain_repo_id
+        JOIN
+        EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql", '''SELECT repo_id as host_table_repo_id, host_name AS referencedHostName FROM "Host";''')
+        ON
+        host_table_repo_id = host_repo_id
+        WHERE
+        domain_creation_time <= TIMESTAMP("%LATEST_REPORT_TIME%")
+        AND domain_deletion_time > TIMESTAMP("%LATEST_REPORT_TIME%"))
+    ON host_name = referencedHostName
+WHERE
+    host_creation_time <= TIMESTAMP("%LATEST_REPORT_TIME%")
+    AND host_deletion_time > TIMESTAMP("%LATEST_REPORT_TIME%")
+GROUP BY tld, registrar_name
+ORDER BY tld, registrar_name

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_transaction_counts.sql

@@ -0,0 +1,72 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Counts the number of mutating transactions each registrar made.
+
+  -- We populate the fields through explicit logging of
+  -- DomainTransactionRecords, which contain all necessary information for
+  -- reporting (such as reporting time, report field, report amount, etc.
+
+SELECT
+    tld,
+    registrar_table.registrar_name AS registrar_name,
+    metricName,
+    metricValue
+FROM (
+    SELECT
+        tld,
+        clientId,
+        CASE WHEN field = 'TRANSFER_SUCCESSFUL' THEN 'TRANSFER_GAINING_SUCCESSFUL'
+        WHEN field = 'TRANSFER_NACKED' THEN 'TRANSFER_GAINING_NACKED'
+        ELSE field
+        END AS metricName,
+        SUM(amount) AS metricValue
+    FROM (
+        SELECT
+            CASE
+                -- Explicit transfer acks (approve) and nacks (reject) are done
+                -- by the opposing registrar. Thus, for these specific actions,
+                -- we swap the 'history_other_registrar_id' with the
+                -- 'history_registrar_id' to properly account for this reversal.
+                WHEN (history_type = 'DOMAIN_TRANSFER_APPROVE'
+                OR history_type = 'DOMAIN_TRANSFER_REJECT')
+                THEN history_other_registrar_id
+                ELSE history_registrar_id
+            END AS clientId,
+            tld,
+            report_field AS field,
+            report_amount AS amount,
+            reporting_time AS reportingTime
+            FROM EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+            ''' SELECT history_type, history_other_registrar_id, history_registrar_id, domain_repo_id, history_revision_id FROM "DomainHistory";''') AS dh
+            JOIN
+                EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+                '''SELECT domain_repo_id, history_revision_id, reporting_time, tld, report_field, report_amount FROM "DomainTransactionRecord";''') AS dtr
+            ON
+            dh.domain_repo_id = dtr.domain_repo_id AND dh.history_revision_id = dtr.history_revision_id
+        )
+    -- Only look at this month's data
+    WHERE reportingTime
+    BETWEEN TIMESTAMP('%EARLIEST_REPORT_TIME%')
+    AND TIMESTAMP('%LATEST_REPORT_TIME%')
+    GROUP BY
+        tld,
+        clientId,
+        field ) AS counts_table
+JOIN
+     EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+                '''SELECT registrar_id, registrar_name FROM "Registrar";''') AS registrar_table
+ON
+    counts_table.clientId = registrar_table.registrar_id

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_transaction_transfer_losing.sql

@@ -0,0 +1,72 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Counts the number of mutating transactions each registrar made.
+
+  -- We populate the fields through explicit logging of
+  -- DomainTransactionRecords, which contain all necessary information for
+  -- reporting (such as reporting time, report field, report amount, etc.
+
+SELECT
+    tld,
+    registrar_table.registrar_name AS registrar_name,
+    metricName,
+    metricValue
+FROM (
+    SELECT
+        tld,
+        clientId,
+        CASE WHEN field = 'TRANSFER_SUCCESSFUL' THEN 'TRANSFER_LOSING_SUCCESSFUL'
+        WHEN field = 'TRANSFER_NACKED' THEN 'TRANSFER_LOSING_NACKED'
+        ELSE NULL
+        END AS metricName,
+        SUM(amount) AS metricValue
+    FROM (
+        SELECT
+            CASE
+                -- Explicit transfer acks (approve) and nacks (reject) are done
+                -- by the opposing registrar. Thus, for these specific actions,
+                -- we swap the 'history_other_registrar_id' with the
+                -- 'history_registrar_id' to properly account for this reversal.
+                WHEN (history_type = 'DOMAIN_TRANSFER_APPROVE'
+                OR history_type = 'DOMAIN_TRANSFER_REJECT')
+                THEN history_registrar_id
+                ELSE history_other_registrar_id
+            END AS clientId,
+            tld,
+            report_field AS field,
+            report_amount AS amount,
+            reporting_time AS reportingTime
+            FROM EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+            ''' SELECT history_type, history_other_registrar_id, history_registrar_id, domain_repo_id, history_revision_id FROM "DomainHistory";''') AS dh
+            JOIN
+                EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+                '''SELECT domain_repo_id, history_revision_id, reporting_time, tld, report_field, report_amount FROM "DomainTransactionRecord";''') AS dtr
+            ON
+            dh.domain_repo_id = dtr.domain_repo_id AND dh.history_revision_id = dtr.history_revision_id
+        )
+    -- Only look at this month's data
+    WHERE reportingTime
+    BETWEEN TIMESTAMP('%EARLIEST_REPORT_TIME%')
+    AND TIMESTAMP('%LATEST_REPORT_TIME%')
+    GROUP BY
+        tld,
+        clientId,
+        field ) AS counts_table
+JOIN
+     EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+                '''SELECT registrar_id, registrar_name FROM "Registrar";''') AS registrar_table
+ON
+    counts_table.clientId = registrar_table.registrar_id

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_transactions_report_aggregation.sql

@@ -0,0 +1,97 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Construct the transaction reports' rows from the intermediary data views.
+
+  -- This query pulls from all intermediary tables to create the activity
+  -- report csv, via a table transpose and sum over all activity report fields.
+
+SELECT
+  tlds.tld_name as tld,
+  -- Surround registrar names with quotes to handle names containing a comma.
+  FORMAT("\"%s\"", registrars.registrar_name) as registrar_name,
+  registrars.iana_id as iana_id,
+  SUM(IF(metrics.metricName = 'TOTAL_DOMAINS', metrics.metricValue, 0)) AS total_domains,
+  SUM(IF(metrics.metricName = 'TOTAL_NAMESERVERS', metrics.metricValue, 0)) AS total_nameservers,
+  SUM(IF(metrics.metricName = 'NET_ADDS_1_YR', metrics.metricValue, 0)) AS net_adds_1_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_2_YR', metrics.metricValue, 0)) AS net_adds_2_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_3_YR', metrics.metricValue, 0)) AS net_adds_3_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_4_YR', metrics.metricValue, 0)) AS net_adds_4_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_5_YR', metrics.metricValue, 0)) AS net_adds_5_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_6_YR', metrics.metricValue, 0)) AS net_adds_6_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_7_YR', metrics.metricValue, 0)) AS net_adds_7_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_8_YR', metrics.metricValue, 0)) AS net_adds_8_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_9_YR', metrics.metricValue, 0)) AS net_adds_9_yr,
+  SUM(IF(metrics.metricName = 'NET_ADDS_10_Yr', metrics.metricValue, 0)) AS net_adds_10_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_1_YR', metrics.metricValue, 0)) AS net_renews_1_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_2_YR', metrics.metricValue, 0)) AS net_renews_2_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_3_YR', metrics.metricValue, 0)) AS net_renews_3_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_4_YR', metrics.metricValue, 0)) AS net_renews_4_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_5_YR', metrics.metricValue, 0)) AS net_renews_5_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_6_YR', metrics.metricValue, 0)) AS net_renews_6_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_7_YR', metrics.metricValue, 0)) AS net_renews_7_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_8_YR', metrics.metricValue, 0)) AS net_renews_8_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_9_YR', metrics.metricValue, 0)) AS net_renews_9_yr,
+  SUM(IF(metrics.metricName = 'NET_RENEWS_10_YR', metrics.metricValue, 0)) AS net_renews_10_yr,
+  SUM(IF(metrics.metricName = 'TRANSFER_GAINING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_gaining_successful,
+  SUM(IF(metrics.metricName = 'TRANSFER_GAINING_NACKED', metrics.metricValue, 0)) AS transfer_gaining_nacked,
+  SUM(IF(metrics.metricName = 'TRANSFER_LOSING_SUCCESSFUL', metrics.metricValue, 0)) AS transfer_losing_successful,
+  SUM(IF(metrics.metricName = 'TRANSFER_LOSING_NACKED', metrics.metricValue, 0)) AS transfer_losing_nacked,
+  -- We don't interact with transfer disputes
+  0 AS transfer_disputed_won,
+  0 AS transfer_disputed_lost,
+  0 AS transfer_disputed_nodecision,
+  SUM(IF(metrics.metricName = 'DELETED_DOMAINS_GRACE', metrics.metricValue, 0)) AS deleted_domains_grace,
+  SUM(IF(metrics.metricName = 'DELETED_DOMAINS_NOGRACE', metrics.metricValue, 0)) AS deleted_domains_nograce,
+  SUM(IF(metrics.metricName = 'RESTORED_DOMAINS', metrics.metricValue, 0)) AS restored_domains,
+  -- We don't require restore reports
+  0 AS restored_noreport,
+  -- We don't enforce AGP limits right now
+  0 AS agp_exemption_requests,
+  0 AS agp_exemptions_granted,
+  0 AS agp_exempted_domains,
+  SUM(IF(metrics.metricName = 'ATTEMPTED_ADDS', metrics.metricValue, 0)) AS attempted_adds
+FROM
+ -- Only produce reports for real TLDs
+    EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql", '''SELECT "tld_name" FROM "Tld" AS t WHERE t.tld_type='REAL';''') AS tlds
+JOIN
+(SELECT *
+  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%REGISTRAR_IANA_ID_TABLE%`)
+  AS registrars
+ON tlds.tld_name = registrars.tld
+-- We LEFT JOIN to produce reports even if the registrar made no transactions
+LEFT OUTER JOIN (
+  -- Gather all intermediary data views
+  SELECT *
+  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TOTAL_DOMAINS_TABLE%`
+  UNION ALL
+  SELECT *
+  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TOTAL_NAMESERVERS_TABLE%`
+  UNION ALL
+  SELECT *
+  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TRANSACTION_COUNTS_TABLE%`
+  UNION ALL
+  SELECT *
+  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%TRANSACTION_TRANSFER_LOSING_TABLE%`
+  UNION ALL
+  SELECT *
+  FROM `%PROJECT_ID%.%ICANN_REPORTING_DATA_SET%.%ATTEMPTED_ADDS_TABLE%` ) AS metrics
+-- Join on tld and registrar name
+ON registrars.tld = metrics.tld
+AND registrars.registrar_name = metrics.registrar_name
+GROUP BY
+tld, registrar_name, iana_id
+ORDER BY
+tld, registrar_name

core/src/main/java/google/registry/reporting/spec11/GenerateSpec11ReportAction.java

@@ -72,6 +72,7 @@ public class GenerateSpec11ReportAction implements Runnable {
   private final Response response;
   private final Dataflow dataflow;
   private final PrimaryDatabase database;
+  private final boolean sendEmail;
 
   @Inject
   GenerateSpec11ReportAction(
@@ -82,6 +83,7 @@ public class GenerateSpec11ReportAction implements Runnable {
       @Key("safeBrowsingAPIKey") String apiKey,
       @Parameter(ReportingModule.PARAM_DATE) LocalDate date,
       @Parameter(RequestParameters.PARAM_DATABASE) PrimaryDatabase database,
+      @Parameter(ReportingModule.SEND_EMAIL) boolean sendEmail,
       Clock clock,
       Response response,
       Dataflow dataflow) {
@@ -98,6 +100,7 @@ public class GenerateSpec11ReportAction implements Runnable {
     this.clock = clock;
     this.response = response;
     this.dataflow = dataflow;
+    this.sendEmail = sendEmail;
   }
 
   @Override
@@ -136,7 +139,9 @@ public class GenerateSpec11ReportAction implements Runnable {
       Map<String, String> beamTaskParameters =
           ImmutableMap.of(
               ReportingModule.PARAM_JOB_ID, jobId, ReportingModule.PARAM_DATE, date.toString());
-      enqueueBeamReportingTask(PublishSpec11ReportAction.PATH, beamTaskParameters);
+      if (sendEmail) {
+        enqueueBeamReportingTask(PublishSpec11ReportAction.PATH, beamTaskParameters);
+      }
       response.setStatus(SC_OK);
       response.setPayload(String.format("Launched Spec11 pipeline: %s", jobId));
     } catch (IOException e) {

core/src/main/java/google/registry/request/lock/LockHandler.java

@@ -22,8 +22,8 @@ import org.joda.time.Duration;
 /**
  * Code execution locked on some shared resource.
  *
- * <p>Locks are either specific to a tld or global to the entire system, in which case a tld of
- * null is used.
+ * <p>Locks are either specific to a tld or global to the entire system, in which case a tld of null
+ * is used.
  */
 public interface LockHandler extends Serializable {
 
@@ -42,4 +42,22 @@ public interface LockHandler extends Serializable {
       @Nullable String tld,
       Duration leaseLength,
       String... lockNames);
+
+  /**
+   * Acquire one or more locks using only Cloud SQL and execute a Void {@link Callable}.
+   *
+   * <p>Runs on a thread that will be killed if it doesn't complete before the lease expires.
+   *
+   * <p>Note that locks are specific either to a given tld or to the entire system (in which case
+   * tld should be passed as null).
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   *
+   * @return true if all locks were acquired and the callable was run; false otherwise.
+   */
+  boolean executeWithSqlLocks(
+      final Callable<Void> callable,
+      @Nullable String tld,
+      Duration leaseLength,
+      String... lockNames);
 }

core/src/main/java/google/registry/request/lock/LockHandlerImpl.java

@@ -73,12 +73,42 @@ public class LockHandlerImpl implements LockHandler {
       @Nullable String tld,
       Duration leaseLength,
       String... lockNames) {
+    return executeWithLockAcquirer(callable, tld, leaseLength, this::acquire, lockNames);
+  }
+
+  /**
+   * Acquire one or more locks using only Cloud SQL and execute a Void {@link Callable}.
+   *
+   * <p>Thread will be killed if it doesn't complete before the lease expires.
+   *
+   * <p>Note that locks are specific either to a given tld or to the entire system (in which case
+   * tld should be passed as null).
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   *
+   * @return whether all locks were acquired and the callable was run.
+   */
+  @Override
+  public boolean executeWithSqlLocks(
+      final Callable<Void> callable,
+      @Nullable String tld,
+      Duration leaseLength,
+      String... lockNames) {
+    return executeWithLockAcquirer(callable, tld, leaseLength, this::acquireSql, lockNames);
+  }
+
+  private boolean executeWithLockAcquirer(
+      final Callable<Void> callable,
+      @Nullable String tld,
+      Duration leaseLength,
+      LockAcquirer lockAcquirer,
+      String... lockNames) {
     DateTime startTime = clock.nowUtc();
     String sanitizedTld = Strings.emptyToNull(tld);
     try {
       return AppEngineTimeLimiter.create()
           .callWithTimeout(
-              new LockingCallable(callable, sanitizedTld, leaseLength, lockNames),
+              new LockingCallable(callable, lockAcquirer, sanitizedTld, leaseLength, lockNames),
               leaseLength.minus(LOCK_TIMEOUT_FUDGE).getMillis(),
               TimeUnit.MILLISECONDS);
     } catch (ExecutionException | UncheckedExecutionException e) {
@@ -108,17 +138,32 @@ public class LockHandlerImpl implements LockHandler {
     return Lock.acquire(lockName, tld, leaseLength, requestStatusChecker, true);
   }
 
+  @VisibleForTesting
+  Optional<Lock> acquireSql(String lockName, @Nullable String tld, Duration leaseLength) {
+    return Lock.acquireSql(lockName, tld, leaseLength, requestStatusChecker, true);
+  }
+
+  private interface LockAcquirer {
+    Optional<Lock> acquireLock(String lockName, @Nullable String tld, Duration leaseLength);
+  }
+
   /** A {@link Callable} that acquires and releases a lock around a delegate {@link Callable}. */
-  private class LockingCallable implements Callable<Boolean> {
+  private static class LockingCallable implements Callable<Boolean> {
     final Callable<Void> delegate;
+    final LockAcquirer lockAcquirer;
     @Nullable final String tld;
     final Duration leaseLength;
     final Set<String> lockNames;
 
     LockingCallable(
-        Callable<Void> delegate, String tld, Duration leaseLength, String... lockNames) {
+        Callable<Void> delegate,
+        LockAcquirer lockAcquirer,
+        String tld,
+        Duration leaseLength,
+        String... lockNames) {
       checkArgument(leaseLength.isLongerThan(LOCK_TIMEOUT_FUDGE));
       this.delegate = delegate;
+      this.lockAcquirer = lockAcquirer;
       this.tld = tld;
       this.leaseLength = leaseLength;
       // Make sure we join locks in a fixed (lexicographical) order to avoid deadlock.
@@ -130,7 +175,7 @@ public class LockHandlerImpl implements LockHandler {
       Set<Lock> acquiredLocks = new HashSet<>();
       try {
         for (String lockName : lockNames) {
-          Optional<Lock> lock = acquire(lockName, tld, leaseLength);
+          Optional<Lock> lock = lockAcquirer.acquireLock(lockName, tld, leaseLength);
           if (!lock.isPresent()) {
             logger.atInfo().log("Couldn't acquire lock named: %s for TLD %s.", lockName, tld);
             return false;

core/src/main/java/google/registry/tools/javascrap/CreateSyntheticHistoryEntriesAction.java

@@ -108,6 +108,16 @@ public class CreateSyntheticHistoryEntriesAction implements Runnable {
     return jpaTm()
         .transact(
             () -> {
+              // Use READ COMMITTED isolation level so that any long-living queries don't cause
+              // collection of predicate locks to spiral out of control (as would happen with a
+              // SERIALIZABLE isolation level)
+              //
+              // NB: setting the isolation level inside the transaction only works for Postgres and
+              // will be reverted to the default once the transaction is committed.
+              jpaTm()
+                  .getEntityManager()
+                  .createNativeQuery("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
+                  .executeUpdate();
               // The class we're searching from is based on which parent type (e.g. Domain) we have
               Class<? extends HistoryEntry> historyClass =
                   getHistoryClassFromParent(resource.getClass());