Provide useful error messages on flows run during read-only mode (#1425)

We want to keep the read-only-mode-exception as an unchecked exception,
so we introduce a temporary check in the EppController that provides a
specific error message for this situation (rather than letting it fall
through to the generic "command failed" messaging

build.gradle

@@ -501,7 +501,8 @@ task javadoc(type: Javadoc) {
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
   options.tags = ["type:a:Generic Type",
-                  "error:a:Expected Error"]
+                  "error:a:Expected Error",
+                  "invariant:a:Guaranteed Property"]
 }
 
 tasks.build.dependsOn(tasks.javadoc)

config/presubmits.py

@@ -119,9 +119,10 @@ PRESUBMITS = {
       "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
+    # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
-        "java", {}):
+        "java", {"DatabaseSnapshotTest.java"}):
       "PostgreSQLContainer instantiation must specify docker tag.",
 
     # Various Soy linting checks
@@ -186,51 +187,6 @@ PRESUBMITS = {
         {"/node_modules/", "google/registry/ui/js/util.js", "registrar_bin."},
     ):
         "JavaScript files should not include console logging.",
-    # SQL injection protection rule for java source file:
-    # The sql template passed to createQuery/createNativeQuery methods must be
-    # a variable name in UPPER_CASE_UNDERSCORE format, i.e., a static final
-    # String variable. This forces the use of parameter-binding on all queries
-    # that take parameters.
-    # The rule would forbid invocation of createQuery(Criteria). However, this
-    # can be handled by adding a helper method in an exempted class to make
-    # the calls.
-    # TODO(b/179158393): enable the 'ConstantName' Java style check to ensure
-    # that non-final variables do not use the UPPER_CASE_UNDERSCORE format.
-    PresubmitCheck(
-        # Line 1: the method names we check and the opening parenthesis, which
-        #    marks the beginning of the first parameter
-        # Line 2: The first parameter is a match if is NOT any of the following:
-        #    - final variable name: \s*([A-Z_]+
-        #    - string literal: "([^"]|\\")*"
-        #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
-        # Line 3: , or the closing parenthesis, marking the end of the first
-        #    parameter
-        r'.*\.(query|createQuery|createNativeQuery)\('
-        r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
-        r'(,|\s*\))))',
-        "java",
-        # ActivityReportingQueryBuilder deals with Dremel queries
-        {"src/test", "ActivityReportingQueryBuilder.java",
-         # This class contains helper method to make queries in Beam.
-         "RegistryJpaIO.java",
-         "CreateSyntheticHistoryEntriesAction.java",
-         # TODO(b/179158393): Remove everything below, which should be done
-         # using Criteria
-         "JpaTransactionManager.java",
-         "JpaTransactionManagerImpl.java",
-         # CriteriaQueryBuilder is a false positive
-         "CriteriaQueryBuilder.java",
-         "RdapDomainSearchAction.java",
-         "RdapNameserverSearchAction.java",
-         "ReadOnlyCheckingEntityManager.java",
-         "RegistryQuery",
-         },
-    ):
-        "The first String parameter to EntityManager.create(Native)Query "
-        "methods must be one of the following:\n"
-        "  - A String literal\n"
-        "  - Concatenation of String literals only\n"
-        "  - The name of a static final String variable"
 }
 
 # Note that this regex only works for one kind of Flyway file.  If we want to

core/build.gradle

@@ -470,7 +470,7 @@ task soyToJava {
 
     outputs.each { file ->
       exec {
-        commandLine 'sed', '-i', 's/@link/LINK/g', file.getCanonicalPath()
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
       }
     }
   }

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -41,11 +41,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnly().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -14,21 +14,21 @@
 
 package google.registry.backup;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogCheckpointRoot;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -56,7 +56,8 @@ public final class CommitLogCheckpointAction implements Runnable {
 
   @Inject Clock clock;
   @Inject CommitLogCheckpointStrategy strategy;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
+
   @Inject CommitLogCheckpointAction() {}
 
   @Override
@@ -73,16 +74,20 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveWithoutBackup()
+                  .saveIgnoringReadOnly()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.
-              taskQueueUtils.enqueue(
-                  getQueue(QUEUE_NAME),
-                  withUrl(ExportCommitLogDiffAction.PATH)
-                      .param(LOWER_CHECKPOINT_TIME_PARAM, lastWrittenTime.toString())
-                      .param(
-                          UPPER_CHECKPOINT_TIME_PARAM, checkpoint.getCheckpointTime().toString()));
+              cloudTasksUtils.enqueue(
+                  QUEUE_NAME,
+                  CloudTasksUtils.createPostTask(
+                      ExportCommitLogDiffAction.PATH,
+                      Service.BACKEND.toString(),
+                      ImmutableMultimap.of(
+                          LOWER_CHECKPOINT_TIME_PARAM,
+                          lastWrittenTime.toString(),
+                          UPPER_CHECKPOINT_TIME_PARAM,
+                          checkpoint.getCheckpointTime().toString())));
             });
   }
 }

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -112,7 +112,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";
@@ -140,7 +140,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,10 +24,8 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
@@ -84,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -96,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -114,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -129,14 +125,13 @@ public final class AsyncTaskEnqueuer {
       String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingRegistrarId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))

core/src/main/java/google/registry/batch/BatchModule.java

@@ -79,6 +79,7 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
+  // TODO(b/207363014): figure out if this needs to be modified for vkey string replacement
   static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
     return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
   }

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -523,18 +523,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -353,8 +353,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
       VKey<HostResource> hostKey =
-          VKey.fromWebsafeKey(
-              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
           tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
               .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -76,13 +76,15 @@ public class ResaveEntityAction implements Runnable {
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
     tm().transact(
             () -> {
+              // TODO(/207363014): figure out if this should modified for vkey string replacement
               ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
               tm().put(
                       (entity instanceof EppResource)
                           ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
                           : entity);
               if (!resaveTimes.isEmpty()) {
-                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.from(resourceKey), requestedTime, resaveTimes);
               }
             });
     response.setPayload("Entity re-saved.");

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -113,9 +113,11 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   }
 
   /**
-   * Returns a list of registrars that should receive expiring notification emails. There are two
-   * certificates that should be considered (the main certificate and failOver certificate). The
-   * registrars should receive notifications if one of the certificate checks returns true.
+   * Returns a list of registrars that should receive expiring notification emails.
+   *
+   * <p>There are two certificates that should be considered (the main certificate and failOver
+   * certificate). The registrars should receive notifications if one of the certificate checks
+   * returns true.
    */
   @VisibleForTesting
   ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
@@ -157,15 +159,17 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
     }
     try {
       ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      ImmutableSet<InternetAddress> ccs = getEmailAddresses(registrar, Type.ADMIN);
       Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
       logger.atInfo().log(
-          "Registrar %s should receive an email that its %s SSL certificate will expire on %s.",
-          registrar.getRegistrarName(),
+          " %s SSL certificate of registrar '%s' will expire on %s.",
           certificateType.getDisplayName(),
+          registrar.getRegistrarName(),
           expirationDate.toString());
-      if (recipients.isEmpty()) {
+      if (recipients.isEmpty() && ccs.isEmpty()) {
         logger.atWarning().log(
-            "Registrar %s contains no email addresses to receive notification email.",
+            "Registrar %s contains no TECH nor ADMIN email addresses to receive notification"
+                + " email.",
             registrar.getRegistrarName());
         return false;
       }
@@ -180,7 +184,7 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
                       expirationDate,
                       registrar.getRegistrarId()))
               .setRecipients(recipients)
-              .setCcs(getEmailAddresses(registrar, Type.ADMIN))
+              .setCcs(ccs)
               .build());
       /*
        * A duration time offset is used here to ensure that date comparison between two
@@ -249,30 +253,32 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   /** Sends notification emails to registrars with expiring certificates. */
   @VisibleForTesting
   int sendNotificationEmails() {
-    int emailsSent = 0;
+    int numEmailsSent = 0;
     for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
       Registrar registrar = registrarInfo.registrar();
-      if (registrarInfo.isCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringCertNotificationSentDate(),
-            CertificateType.PRIMARY,
-            registrar.getClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringCertNotificationSentDate(),
+              CertificateType.PRIMARY,
+              registrar.getClientCertificate())) {
+        numEmailsSent++;
       }
-      if (registrarInfo.isFailOverCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringFailoverCertNotificationSentDate(),
-            CertificateType.FAILOVER,
-            registrar.getFailoverClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isFailOverCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringFailoverCertNotificationSentDate(),
+              CertificateType.FAILOVER,
+              registrar.getFailoverClientCertificate())) {
+        numEmailsSent++;
       }
     }
-    return emailsSent;
+    return numEmailsSent;
   }
 
-  /** Returns a list of email addresses of the registrar that should receive a notification email */
+  /**
+   * Returns a list of email addresses of the registrar that should receive a notification email.
+   */
   @VisibleForTesting
   ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
     ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -0,0 +1,88 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import java.util.List;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+
+/**
+ * A database snapshot shareable by concurrent queries from multiple database clients. A snapshot is
+ * uniquely identified by its {@link #getSnapshotId snapshotId}, and must stay open until all
+ * concurrent queries to this snapshot have attached to it by calling {@link
+ * google.registry.persistence.transaction.JpaTransactionManager#setDatabaseSnapshot}. However, it
+ * can be closed before those queries complete.
+ *
+ * <p>This feature is <em>Postgresql-only</em>.
+ *
+ * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to avoid
+ * exhausting predicate locks at the SERIALIZABLE level.
+ */
+// TODO(b/193662898): vendor-independent support for richer transaction semantics.
+public class DatabaseSnapshot implements AutoCloseable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private String snapshotId;
+  private EntityManager entityManager;
+  private EntityTransaction transaction;
+
+  private DatabaseSnapshot() {}
+
+  public String getSnapshotId() {
+    checkState(entityManager != null, "Snapshot not opened yet.");
+    checkState(entityManager.isOpen(), "Snapshot already closed.");
+    return snapshotId;
+  }
+
+  private DatabaseSnapshot open() {
+    entityManager = jpaTm().getStandaloneEntityManager();
+    transaction = entityManager.getTransaction();
+    transaction.setRollbackOnly();
+    transaction.begin();
+
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+
+    List<?> snapshotIds =
+        entityManager.createNativeQuery("SELECT pg_export_snapshot();").getResultList();
+    checkState(snapshotIds.size() == 1, "Unexpected number of snapshots: %s", snapshotIds.size());
+    snapshotId = (String) snapshotIds.get(0);
+    return this;
+  }
+
+  @Override
+  public void close() {
+    if (transaction != null && transaction.isActive()) {
+      try {
+        transaction.rollback();
+      } catch (Exception e) {
+        logger.atWarning().withCause(e).log("Failed to close a Database Snapshot");
+      }
+    }
+    if (entityManager != null && entityManager.isOpen()) {
+      entityManager.close();
+    }
+  }
+
+  public static DatabaseSnapshot createSnapshot() {
+    return new DatabaseSnapshot().open();
+  }
+}

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -138,6 +138,9 @@ public final class RegistryJpaIO {
 
     abstract Coder<T> coder();
 
+    @Nullable
+    abstract String snapshotId();
+
     abstract Builder<R, T> toBuilder();
 
     @Override
@@ -145,7 +148,9 @@ public final class RegistryJpaIO {
     public PCollection<T> expand(PBegin input) {
       return input
           .apply("Starting " + name(), Create.of((Void) null))
-          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
+          .apply(
+              "Run query for " + name(),
+              ParDo.of(new QueryRunner<>(query(), resultMapper(), snapshotId())))
           .setCoder(coder())
           .apply("Reshuffle", Reshuffle.viaRandomKey());
     }
@@ -162,6 +167,18 @@ public final class RegistryJpaIO {
       return toBuilder().coder(coder).build();
     }
 
+    /**
+     * Specifies the database snapshot to use for this query.
+     *
+     * <p>This feature is <em>Postgresql-only</em>. User is responsible for keeping the snapshot
+     * available until all JVM workers have started using it by calling {@link
+     * JpaTransactionManager#setDatabaseSnapshot}.
+     */
+    // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+    public Read<R, T> withSnapshot(String snapshotId) {
+      return toBuilder().snapshotId(snapshotId).build();
+    }
+
     static <R, T> Builder<R, T> builder() {
       return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
           .name(DEFAULT_NAME)
@@ -179,6 +196,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<R, T> coder(Coder coder);
 
+      abstract Builder<R, T> snapshotId(@Nullable String sharedSnapshotId);
+
       abstract Read<R, T> build();
 
       Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
@@ -201,17 +220,28 @@ public final class RegistryJpaIO {
     static class QueryRunner<R, T> extends DoFn<Void, T> {
       private final RegistryQuery<R> query;
       private final SerializableFunction<R, T> resultMapper;
+      // java.util.Optional is not serializable. Use of Guava Optional is discouraged.
+      @Nullable private final String snapshotId;
 
-      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+      QueryRunner(
+          RegistryQuery<R> query,
+          SerializableFunction<R, T> resultMapper,
+          @Nullable String snapshotId) {
         this.query = query;
         this.resultMapper = resultMapper;
+        this.snapshotId = snapshotId;
       }
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
         jpaTm()
             .transactNoRetry(
-                () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
+                () -> {
+                  if (snapshotId != null) {
+                    jpaTm().setDatabaseSnapshot(snapshotId);
+                  }
+                  query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
+                });
       }
     }
   }

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -21,6 +21,7 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -45,9 +46,19 @@ public interface RegistryPipelineComponent {
   @Config("projectId")
   String getProjectId();
 
+  /** Returns the regular {@link JpaTransactionManager} for general use. */
   @BeamJpaTm
   Lazy<JpaTransactionManager> getJpaTransactionManager();
 
+  /**
+   * Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
+   * ({@link google.registry.model.domain.DomainBase} and {@link
+   * google.registry.model.domain.DomainHistory}). Please refer to {@link
+   * google.registry.model.bulkquery.BulkQueryEntities} for more information.
+   */
+  @BeamBulkQueryJpaTm
+  Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -44,6 +45,12 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setIsolationOverride(TransactionIsolationLevel isolationOverride);
 
+  @Description("The JPA Transaction Manager to use.")
+  @Default.Enum(value = "REGULAR")
+  JpaTransactionManagerType getJpaTransactionManagerType();
+
+  void setJpaTransactionManagerType(JpaTransactionManagerType jpaTransactionManagerType);
+
   @Description("The number of entities to write to the SQL database in one operation.")
   @Default.Integer(20)
   int getSqlWriteBatchSize();

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -49,10 +49,19 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
     }
     logger.atInfo().log("Setting up RegistryEnvironment %s.", environment);
     environment.setup();
-    Lazy<JpaTransactionManager> transactionManagerLazy =
-        toRegistryPipelineComponent(registryOptions).getJpaTransactionManager();
+    RegistryPipelineComponent registryPipelineComponent =
+        toRegistryPipelineComponent(registryOptions);
+    Lazy<JpaTransactionManager> transactionManagerLazy;
+    switch (registryOptions.getJpaTransactionManagerType()) {
+      case BULK_QUERY:
+        transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
+        break;
+      case REGULAR:
+      default:
+        transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
+    }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
-    // Masquarade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
     // loads all ofy entities.
     new AppEngineEnvironment("Beam").setEnvironmentForAllThreads();
     // Set the system property so that we can call IdService.allocateId() without access to

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -262,8 +262,8 @@ public final class Transforms {
 
   // Production data repair configs go below. See b/185954992.
 
-  // Prober domains in bad state, without associated contacts, hosts, billings, and history.
-  // They can be safely ignored.
+  // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized
+  // history. They can be safely ignored.
   private static final ImmutableSet<String> IGNORED_DOMAINS =
       ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT");
 
@@ -299,7 +299,7 @@ public final class Transforms {
       return !IGNORED_HOSTS.contains(roid);
     }
     if (entity.getKind().equals("HistoryEntry")) {
-      // Remove production bad data: History of the contacts to be ignored:
+      // Remove production bad data: Histories of ignored EPP resources:
       com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent();
       if (parentKey.getKind().equals("ContactResource")) {
         String contactRoid = parentKey.getName();
@@ -309,6 +309,10 @@ public final class Transforms {
         String hostRoid = parentKey.getName();
         return !IGNORED_HOSTS.contains(hostRoid);
       }
+      if (parentKey.getKind().equals("DomainBase")) {
+        String domainRoid = parentKey.getName();
+        return !IGNORED_DOMAINS.contains(domainRoid);
+      }
     }
     // End of production-specific checks.
 

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -57,7 +57,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=invoicing}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -75,6 +75,8 @@ public class RdeIO {
   abstract static class Write
       extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
 
+    private static final long serialVersionUID = 3334807737227087760L;
+
     abstract GcsUtils gcsUtils();
 
     abstract CloudTasksUtils cloudTasksUtils();
@@ -113,8 +115,9 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()))
-          .apply("Enqueue upload action", ParDo.of(new UploadEnqueuer(cloudTasksUtils())));
+          .apply(
+              "Update cursor and enqueue next action",
+              ParDo.of(new CursorUpdater(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -123,6 +126,7 @@ public class RdeIO {
       extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5496375923068400382L;
 
     private final GcsUtils gcsUtils;
     private final String rdeBucket;
@@ -169,7 +173,7 @@ public class RdeIO {
         checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
         prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
       } else {
-        prefix = prefix + "/" + basename;
+        prefix = prefix + '/' + basename;
       }
       BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
       // This file will contain the byte length (ASCII) of the raw unencrypted XML.
@@ -250,12 +254,20 @@ public class RdeIO {
     }
   }
 
-  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, PendingDeposit> {
+  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5822176227753327224L;
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private CursorUpdater(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
 
     @ProcessElement
     public void processElement(
-        @Element KV<PendingDeposit, Integer> input, OutputReceiver<PendingDeposit> outputReceiver) {
+        @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -282,46 +294,32 @@ public class RdeIO {
                 logger.atInfo().log(
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                if (key.mode() == RdeMode.FULL) {
+                  cloudTasksUtils.enqueue(
+                      RDE_UPLOAD_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          RdeUploadAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                } else {
+                  cloudTasksUtils.enqueue(
+                      BRDA_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          BrdaCopyAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_WATERMARK,
+                              key.watermark().toString(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                }
               });
-      outputReceiver.output(input.getKey());
-    }
-  }
-
-  private static class UploadEnqueuer extends DoFn<PendingDeposit, Void> {
-
-    private final CloudTasksUtils cloudTasksUtils;
-
-    private UploadEnqueuer(CloudTasksUtils cloudTasksUtils) {
-      this.cloudTasksUtils = cloudTasksUtils;
-    }
-
-    @ProcessElement
-    public void processElement(@Element PendingDeposit input, PipelineOptions options) {
-      if (input.mode() == RdeMode.FULL) {
-        cloudTasksUtils.enqueue(
-            RDE_UPLOAD_QUEUE,
-            CloudTasksUtils.createPostTask(
-                RdeUploadAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      } else {
-        cloudTasksUtils.enqueue(
-            BRDA_QUEUE,
-            CloudTasksUtils.createPostTask(
-                BrdaCopyAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_WATERMARK,
-                    input.watermark().toString(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      }
     }
   }
 }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -14,36 +14,50 @@
 
 package google.registry.beam.rde;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.beam.rde.RdePipeline.TupleTags.DOMAIN_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.EXTERNAL_HOST_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.HOST_TO_PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_CONTACTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
+import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
+import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
 import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
-import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.UtilsModule;
@@ -54,74 +68,158 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.persistence.Entity;
+import javax.persistence.IdClass;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.DateTime;
 
 /**
  * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=rde}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
+ * <p>This pipeline only works for pending deposits with the same watermark, the {@link
+ * google.registry.rde.RdeStagingAction} will batch such pending deposits together and launch
+ * multiple pipelines if multiple watermarks exist.
+ *
+ * <p>The pipeline is broadly divided into two parts -- creating the {@link DepositFragment}s, and
+ * processing them.
+ *
+ * <h1>Creating {@link DepositFragment}</h1>
+ *
+ * <h2>{@link Registrar}</h2>
+ *
+ * Non-test registrar entities are loaded from Cloud SQL and marshalled into deposit fragments. They
+ * are <b>NOT</b> rewound to the watermark.
+ *
+ * <h2>{@link EppResource}</h2>
+ *
+ * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
+ * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
+ * revision ID) from the SQL query.
+ *
+ * <h3>{@link DomainBase}</h3>
+ *
+ * After the most recent (live) domain resources are loaded from the corresponding history objects,
+ * we marshall them to deposit fragments and emit the (pending deposit: deposit fragment) pairs for
+ * further processing. We also find all the contacts and hosts referenced by a given domain and emit
+ * pairs of (contact/host repo ID: pending deposit) for all RDE pending deposits for further
+ * processing.
+ *
+ * <h3>{@link ContactResource}</h3>
+ *
+ * We first join most recent contact histories, represented by (contact repo ID: contact history
+ * revision ID) pairs, with referenced contacts, represented by (contact repo ID: pending deposit)
+ * pairs, on the contact repo ID, to remove unreferenced contact histories. Contact resources are
+ * then loaded from the remaining referenced contact histories, and marshalled into (pending
+ * deposit: deposit fragment) pairs.
+ *
+ * <h3>{@link HostResource}</h3>
+ *
+ * Similar to {@link ContactResource}, we join the most recent host history with referenced hosts to
+ * find most recent referenced hosts. For external hosts we do the same treatment as we did on
+ * contacts and obtain the (pending deposit: deposit fragment) pairs. For subordinate hosts, we need
+ * to find the superordinate domain in order to properly handle pending transfer in the deposit as
+ * well. So we first find the superordinate domain repo ID from the host and join the (superordinate
+ * domain repo ID: (subordinate host repo ID: (pending deposit: revision ID))) pair with the (domain
+ * repo ID: revision ID) pair obtained from the domain history query in order to map the host at
+ * watermark to the domain at watermark. We then proceed to create the (pending deposit: deposit
+ * fragment) pair for subordinate hosts using the added domain information.
+ *
+ * <h1>Processing {@link DepositFragment}</h1>
+ *
+ * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
+ * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
+ * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
+ * pending deposit is then rolled forward, and the next action is enqueued. The latter two
+ * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
+ *
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
 @Singleton
 public class RdePipeline implements Serializable {
 
+  private static final long serialVersionUID = -4866795928854754666L;
   private final transient RdePipelineOptions options;
   private final ValidationMode mode;
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final ImmutableSet<PendingDeposit> pendingDeposits;
+  private final DateTime watermark;
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
   private final CloudTasksUtils cloudTasksUtils;
+  private final RdeMarshaller marshaller;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
   private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
       Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
 
-  private static final String EPP_RESOURCE_QUERY =
-      "SELECT id FROM %entity% "
-          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
-
-  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
-    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
-        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
-  }
+  // The field name of the EPP resource embedded in its corresponding history entry.
+  private static final ImmutableMap<Class<? extends HistoryEntry>, String> EPP_RESOURCE_FIELD_NAME =
+      ImmutableMap.of(
+          DomainHistory.class,
+          "domainContent",
+          ContactHistory.class,
+          "contactBase",
+          HostHistory.class,
+          "hostBase");
 
   @Inject
   RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
-    this.pendings = decodePendings(options.getPendings());
+    this.pendingDeposits = decodePendingDeposits(options.getPendings());
+    ImmutableSet<DateTime> potentialWatermarks =
+        pendingDeposits.stream()
+            .map(PendingDeposit::watermark)
+            .distinct()
+            .collect(toImmutableSet());
+    checkArgument(
+        potentialWatermarks.size() == 1,
+        String.format(
+            "RDE pipeline should only work on pending deposits "
+                + "with the same watermark, but %d were given: %s",
+            potentialWatermarks.size(), potentialWatermarks));
+    this.watermark = potentialWatermarks.asList().get(0);
     this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
     this.cloudTasksUtils = cloudTasksUtils;
+    this.marshaller = new RdeMarshaller(mode);
   }
 
   PipelineResult run() {
@@ -133,13 +231,46 @@ public class RdePipeline implements Serializable {
   }
 
   PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
-    return PCollectionList.of(processRegistrars(pipeline))
-        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
-        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
-        .and(processNonRegistrarEntities(pipeline, HostResource.class))
-        .apply(Flatten.pCollections())
+    PCollection<KV<PendingDeposit, DepositFragment>> registrarFragments =
+        processRegistrars(pipeline);
+
+    PCollection<KV<String, Long>> domainHistories =
+        getMostRecentHistoryEntries(pipeline, DomainHistory.class);
+
+    PCollection<KV<String, Long>> contactHistories =
+        getMostRecentHistoryEntries(pipeline, ContactHistory.class);
+
+    PCollection<KV<String, Long>> hostHistories =
+        getMostRecentHistoryEntries(pipeline, HostHistory.class);
+
+    PCollectionTuple processedDomainHistories = processDomainHistories(domainHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> domainFragments =
+        processedDomainHistories.get(DOMAIN_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> contactFragments =
+        processContactHistories(
+            processedDomainHistories.get(REFERENCED_CONTACTS), contactHistories);
+
+    PCollectionTuple processedHosts =
+        processHostHistories(processedDomainHistories.get(REFERENCED_HOSTS), hostHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> externalHostFragments =
+        processedHosts.get(EXTERNAL_HOST_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> subordinateHostFragments =
+        processSubordinateHosts(processedHosts.get(SUPERORDINATE_DOMAINS), domainHistories);
+
+    return PCollectionList.of(registrarFragments)
+        .and(domainFragments)
+        .and(contactFragments)
+        .and(externalHostFragments)
+        .and(subordinateHostFragments)
+        .apply(
+            "Combine PendingDeposit:DepositFragment pairs from all entities",
+            Flatten.pCollections())
         .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply("Group by PendingDeposit", GroupByKey.create());
+        .apply("Group DepositFragment by PendingDeposit", GroupByKey.create());
   }
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
@@ -154,127 +285,398 @@ public class RdePipeline implements Serializable {
             .build());
   }
 
-  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+  private PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    // Note that the namespace in the metric is not being used by Stackdriver, it just has to be
+    // non-empty.
+    // See:
+    // https://stackoverflow.com/questions/48530496/google-dataflow-custom-metrics-not-showing-on-stackdriver
+    Counter includedRegistrarCounter = Metrics.counter("RDE", "IncludedRegistrar");
+    Counter registrarFragmentCounter = Metrics.counter("RDE", "RegistrarFragment");
     return pipeline
         .apply(
-            "Read all production Registrar entities",
+            "Read all production Registrars",
             RegistryJpaIO.read(
                 "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
                 ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
                 String.class,
-                // TODO: consider adding coders for entities and pass them directly instead of using
-                // VKeys.
                 id -> VKey.createSql(Registrar.class, id)))
         .apply(
-            "Marshal Registrar into DepositFragment",
+            "Marshall Registrar into DepositFragment",
             FlatMapElements.into(
-                    TypeDescriptors.kvs(
+                    kvs(
                         TypeDescriptor.of(PendingDeposit.class),
                         TypeDescriptor.of(DepositFragment.class)))
                 .via(
                     (VKey<Registrar> key) -> {
+                      includedRegistrarCounter.inc();
                       Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
-                      DepositFragment fragment =
-                          new RdeMarshaller(mode).marshalRegistrar(registrar);
-                      return pendings.values().stream()
-                          .map(pending -> KV.of(pending, fragment))
-                          .collect(toImmutableSet());
+                      DepositFragment fragment = marshaller.marshalRegistrar(registrar);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          pendingDeposits.stream()
+                              .map(pending -> KV.of(pending, fragment))
+                              .collect(toImmutableSet());
+                      registrarFragmentCounter.inc(fragments.size());
+                      return fragments;
                     }));
   }
 
-  <T extends EppResource>
-      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
-          Pipeline pipeline, Class<T> clazz) {
-    return createInputs(pipeline, clazz)
-        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(
-            KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)));
+  /**
+   * Load the most recent history entry before the watermark for a given history entry type.
+   *
+   * <p>Note that deleted and non-production resources are not included.
+   *
+   * @return A KV pair of (repoId, revisionId), used to reconstruct the composite key for the
+   *     history entry.
+   */
+  private <T extends HistoryEntry> PCollection<KV<String, Long>> getMostRecentHistoryEntries(
+      Pipeline pipeline, Class<T> historyClass) {
+    String repoIdFieldName = HistoryEntryDao.REPO_ID_FIELD_NAMES.get(historyClass);
+    String resourceFieldName = EPP_RESOURCE_FIELD_NAME.get(historyClass);
+    return pipeline
+        .apply(
+            String.format("Load most recent %s", historyClass.getSimpleName()),
+            RegistryJpaIO.read(
+                ("SELECT %repoIdField%, id FROM %entity% WHERE (%repoIdField%, modificationTime)"
+                     + " IN (SELECT %repoIdField%, MAX(modificationTime) FROM %entity% WHERE"
+                     + " modificationTime <= :watermark GROUP BY %repoIdField%) AND"
+                     + " %resourceField%.deletionTime > :watermark AND"
+                     + " COALESCE(%resourceField%.creationClientId, '') NOT LIKE 'prober-%' AND"
+                     + " COALESCE(%resourceField%.currentSponsorClientId, '') NOT LIKE 'prober-%'"
+                     + " AND COALESCE(%resourceField%.lastEppUpdateClientId, '') NOT LIKE"
+                     + " 'prober-%' "
+                        + (historyClass == DomainHistory.class
+                            ? "AND %resourceField%.tld IN "
+                                + "(SELECT id FROM Tld WHERE tldType = 'REAL')"
+                            : ""))
+                    .replace("%entity%", historyClass.getSimpleName())
+                    .replace("%repoIdField%", repoIdFieldName)
+                    .replace("%resourceField%", resourceFieldName),
+                ImmutableMap.of("watermark", watermark),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1])))
+        .setCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));
   }
 
-  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
-    return pipeline.apply(
-        "Read all production " + clazz.getSimpleName() + " entities",
-        RegistryJpaIO.read(
-            createEppResourceQuery(clazz),
-            clazz.equals(DomainBase.class)
-                ? ImmutableMap.of("tlds", pendings.keySet())
-                : ImmutableMap.of(),
-            String.class,
-            // TODO: consider adding coders for entities and pass them directly instead of using
-            // VKeys.
-            x -> VKey.createSql(clazz, x)));
+  private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
+      Class<T> historyEntryClazz, String repoId, long revisionId) {
+    try {
+      Class<?> idClazz = historyEntryClazz.getAnnotation(IdClass.class).value();
+      Serializable idObject =
+          (Serializable)
+              idClazz.getConstructor(String.class, long.class).newInstance(repoId, revisionId);
+      return jpaTm()
+          .transact(() -> jpaTm().loadByKey(VKey.createSql(historyEntryClazz, idObject)))
+          .getResourceAtPointInTime()
+          .map(resource -> resource.cloneProjectedAtTime(watermark))
+          .get();
+    } catch (NoSuchMethodException
+        | InvocationTargetException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot load resource from %s with repoId %s and revisionId %s",
+              historyEntryClazz.getSimpleName(), repoId, revisionId),
+          e);
+    }
   }
 
-  <T extends EppResource>
-      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
-    return FlatMapElements.into(
-            TypeDescriptors.kvs(
-                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
-        .via(
-            (VKey<T> key) -> {
-              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
-              // The set of all TLDs to which this resource should be emitted.
-              ImmutableSet<String> tlds =
-                  clazz.equals(DomainBase.class)
-                      ? ImmutableSet.of(((DomainBase) resource).getTld())
-                      : pendings.keySet();
-              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
-              ImmutableSet<DateTime> dates =
-                  tlds.stream()
-                      .map(pendings::get)
-                      .flatMap(ImmutableSet::stream)
-                      .map(PendingDeposit::watermark)
-                      .collect(toImmutableSet());
-              // Launch asynchronous fetches of point-in-time representations of resource.
-              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
-                  ImmutableMap.copyOf(
-                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
-              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
-              // the result.
-              RdeFragmenter fragmenter =
-                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
-              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
-              for (String tld : tlds) {
-                for (PendingDeposit pending : pendings.get(tld)) {
-                  // Hosts and contacts don't get included in BRDA deposits.
-                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
-                    continue;
+  /**
+   * Remove unreferenced resources by joining the (repoId, pendingDeposit) pair with the (repoId,
+   * revisionId) on the repoId.
+   *
+   * <p>The (repoId, pendingDeposit) pairs denote resources (contact, host) that are referenced from
+   * a domain, that are to be included in the corresponding pending deposit.
+   *
+   * <p>The (repoId, revisionId) paris come from the most recent history entry query, which can be
+   * used to load the embedded resources themselves.
+   *
+   * @return a pair of (repoId, ([pendingDeposit], [revisionId])) where neither the pendingDeposit
+   *     nor the revisionId list is empty.
+   */
+  private static PCollection<KV<String, CoGbkResult>> removeUnreferencedResource(
+      PCollection<KV<String, PendingDeposit>> referencedResources,
+      PCollection<KV<String, Long>> historyEntries,
+      Class<? extends EppResource> resourceClazz) {
+    String resourceName = resourceClazz.getSimpleName();
+    Class<? extends HistoryEntry> historyEntryClazz =
+        RESOURCE_TYPES_TO_HISTORY_TYPES.get(resourceClazz);
+    String historyEntryName = historyEntryClazz.getSimpleName();
+    Counter referencedResourceCounter = Metrics.counter("RDE", "Referenced" + resourceName);
+    return KeyedPCollectionTuple.of(PENDING_DEPOSIT, referencedResources)
+        .and(REVISION_ID, historyEntries)
+        .apply(
+            String.format(
+                "Join PendingDeposit with %s revision ID on %s", historyEntryName, resourceName),
+            CoGroupByKey.create())
+        .apply(
+            String.format("Remove unreferenced %s", resourceName),
+            Filter.by(
+                (KV<String, CoGbkResult> kv) -> {
+                  boolean toInclude =
+                      // If a resource does not have corresponding pending deposit, it is not
+                      // referenced and should not be included.
+                      kv.getValue().getAll(PENDING_DEPOSIT).iterator().hasNext()
+                          // If a resource does not have revision id (this should not happen, as
+                          // every referenced resource must be valid at watermark time, therefore
+                          // be embedded in a history entry valid at watermark time, otherwise
+                          // the domain cannot reference it), there is no way for us to find the
+                          // history entry and load the embedded resource. So we ignore the resource
+                          // to keep the downstream process simple.
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedResourceCounter.inc();
                   }
-                  Optional<DepositFragment> fragment =
-                      fragmenter.marshal(pending.watermark(), pending.mode());
-                  fragment.ifPresent(
-                      depositFragment -> results.add(KV.of(pending, depositFragment)));
-                }
-              }
-              return results;
-            });
+                  return toInclude;
+                }));
+  }
+
+  private PCollectionTuple processDomainHistories(PCollection<KV<String, Long>> domainHistories) {
+    Counter activeDomainCounter = Metrics.counter("RDE", "ActiveDomainBase");
+    Counter domainFragmentCounter = Metrics.counter("RDE", "DomainFragment");
+    Counter referencedContactCounter = Metrics.counter("RDE", "ReferencedContactResource");
+    Counter referencedHostCounter = Metrics.counter("RDE", "ReferencedHostResource");
+    return domainHistories.apply(
+        "Map DomainHistory to DepositFragment "
+            + "and emit referenced ContactResource and HostResource",
+        ParDo.of(
+                new DoFn<KV<String, Long>, KV<PendingDeposit, DepositFragment>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Long> kv, MultiOutputReceiver receiver) {
+                    activeDomainCounter.inc();
+                    DomainBase domain =
+                        (DomainBase)
+                            loadResourceByHistoryEntryId(
+                                DomainHistory.class, kv.getKey(), kv.getValue());
+                    pendingDeposits.stream()
+                        .filter(pendingDeposit -> pendingDeposit.tld().equals(domain.getTld()))
+                        .forEach(
+                            pendingDeposit -> {
+                              // Domains are always deposited in both modes.
+                              domainFragmentCounter.inc();
+                              receiver
+                                  .get(DOMAIN_FRAGMENTS)
+                                  .output(
+                                      KV.of(
+                                          pendingDeposit,
+                                          marshaller.marshalDomain(domain, pendingDeposit.mode())));
+                              // Contacts and hosts are only deposited in RDE, not BRDA.
+                              if (pendingDeposit.mode() == RdeMode.FULL) {
+                                HashSet<Serializable> contacts = new HashSet<>();
+                                contacts.add(domain.getAdminContact().getSqlKey());
+                                contacts.add(domain.getTechContact().getSqlKey());
+                                contacts.add(domain.getRegistrant().getSqlKey());
+                                // Billing contact is not mandatory.
+                                if (domain.getBillingContact() != null) {
+                                  contacts.add(domain.getBillingContact().getSqlKey());
+                                }
+                                referencedContactCounter.inc(contacts.size());
+                                contacts.forEach(
+                                    contactRepoId ->
+                                        receiver
+                                            .get(REFERENCED_CONTACTS)
+                                            .output(KV.of((String) contactRepoId, pendingDeposit)));
+                                if (domain.getNsHosts() != null) {
+                                  referencedHostCounter.inc(domain.getNsHosts().size());
+                                  domain
+                                      .getNsHosts()
+                                      .forEach(
+                                          hostKey ->
+                                              receiver
+                                                  .get(REFERENCED_HOSTS)
+                                                  .output(
+                                                      KV.of(
+                                                          (String) hostKey.getSqlKey(),
+                                                          pendingDeposit)));
+                                }
+                              }
+                            });
+                  }
+                })
+            .withOutputTags(
+                DOMAIN_FRAGMENTS, TupleTagList.of(REFERENCED_CONTACTS).and(REFERENCED_HOSTS)));
+  }
+
+  private PCollection<KV<PendingDeposit, DepositFragment>> processContactHistories(
+      PCollection<KV<String, PendingDeposit>> referencedContacts,
+      PCollection<KV<String, Long>> contactHistories) {
+    Counter contactFragmentCounter = Metrics.counter("RDE", "ContactFragment");
+    return removeUnreferencedResource(referencedContacts, contactHistories, ContactResource.class)
+        .apply(
+            "Map ContactResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      ContactResource contact =
+                          (ContactResource)
+                              loadResourceByHistoryEntryId(
+                                  ContactHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      DepositFragment fragment = marshaller.marshalContact(contact);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same contact could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .map(pendingDeposit -> KV.of(pendingDeposit, fragment))
+                              .collect(toImmutableSet());
+                      contactFragmentCounter.inc(fragments.size());
+                      return fragments;
+                    }));
+  }
+
+  private PCollectionTuple processHostHistories(
+      PCollection<KV<String, PendingDeposit>> referencedHosts,
+      PCollection<KV<String, Long>> hostHistories) {
+    Counter subordinateHostCounter = Metrics.counter("RDE", "SubordinateHostResource");
+    Counter externalHostCounter = Metrics.counter("RDE", "ExternalHostResource");
+    Counter externalHostFragmentCounter = Metrics.counter("RDE", "ExternalHostFragment");
+    return removeUnreferencedResource(referencedHosts, hostHistories, HostResource.class)
+        .apply(
+            "Map external DomainResource to DepositFragment and process subordinate domains",
+            ParDo.of(
+                    new DoFn<KV<String, CoGbkResult>, KV<PendingDeposit, DepositFragment>>() {
+                      @ProcessElement
+                      public void processElement(
+                          @Element KV<String, CoGbkResult> kv, MultiOutputReceiver receiver) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    kv.getKey(),
+                                    kv.getValue().getOnly(REVISION_ID));
+                        // When a host is subordinate, we need to find it's superordinate domain and
+                        // include it in the deposit as well.
+                        if (host.isSubordinate()) {
+                          subordinateHostCounter.inc();
+                          receiver
+                              .get(SUPERORDINATE_DOMAINS)
+                              .output(
+                                  // The output are pairs of
+                                  // (superordinateDomainRepoId,
+                                  //   (subordinateHostRepoId, (pendingDeposit, revisionId))).
+                                  KV.of((String) host.getSuperordinateDomain().getSqlKey(), kv));
+                        } else {
+                          externalHostCounter.inc();
+                          DepositFragment fragment = marshaller.marshalExternalHost(host);
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same host could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .forEach(
+                                  pendingDeposit -> {
+                                    externalHostFragmentCounter.inc();
+                                    receiver
+                                        .get(EXTERNAL_HOST_FRAGMENTS)
+                                        .output(KV.of(pendingDeposit, fragment));
+                                  });
+                        }
+                      }
+                    })
+                .withOutputTags(EXTERNAL_HOST_FRAGMENTS, TupleTagList.of(SUPERORDINATE_DOMAINS)));
+  }
+
+  /**
+   * Process subordinate hosts by making a deposit fragment with pending transfer information
+   * obtained from its superordinate domain.
+   *
+   * @param superordinateDomains Pairs of (superordinateDomainRepoId, (subordinateHostRepoId,
+   *     (pendingDeposit, revisionId))). This collection maps the subordinate host and the pending
+   *     deposit to include it to its superordinate domain.
+   * @param domainHistories Pairs of (domainRepoId, revisionId). This collection helps us find the
+   *     historical superordinate domain from its history entry and is obtained from calling {@link
+   *     #getMostRecentHistoryEntries} for domains.
+   */
+  private PCollection<KV<PendingDeposit, DepositFragment>> processSubordinateHosts(
+      PCollection<KV<String, KV<String, CoGbkResult>>> superordinateDomains,
+      PCollection<KV<String, Long>> domainHistories) {
+    Counter subordinateHostFragmentCounter = Metrics.counter("RDE", "SubordinateHostFragment");
+    Counter referencedSubordinateHostCounter = Metrics.counter("RDE", "ReferencedSubordinateHost");
+    return KeyedPCollectionTuple.of(HOST_TO_PENDING_DEPOSIT, superordinateDomains)
+        .and(REVISION_ID, domainHistories)
+        .apply(
+            "Join HostResource:PendingDeposits with DomainHistory on DomainResource",
+            CoGroupByKey.create())
+        .apply(
+            " Remove unreferenced DomainResource",
+            Filter.by(
+                kv -> {
+                  boolean toInclude =
+                      kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT).iterator().hasNext()
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedSubordinateHostCounter.inc();
+                  }
+                  return toInclude;
+                }))
+        .apply(
+            "Map subordinate HostResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      DomainBase superordinateDomain =
+                          (DomainBase)
+                              loadResourceByHistoryEntryId(
+                                  DomainHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      ImmutableSet.Builder<KV<PendingDeposit, DepositFragment>> results =
+                          new ImmutableSet.Builder<>();
+                      for (KV<String, CoGbkResult> hostToPendingDeposits :
+                          kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT)) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    hostToPendingDeposits.getKey(),
+                                    hostToPendingDeposits.getValue().getOnly(REVISION_ID));
+                        DepositFragment fragment =
+                            marshaller.marshalSubordinateHost(host, superordinateDomain);
+                        Streams.stream(hostToPendingDeposits.getValue().getAll(PENDING_DEPOSIT))
+                            .distinct()
+                            .forEach(
+                                pendingDeposit -> {
+                                  subordinateHostFragmentCounter.inc();
+                                  results.add(KV.of(pendingDeposit, fragment));
+                                });
+                      }
+                      return results.build();
+                    }));
   }
 
   /**
    * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
-   * the original TLD to pending deposit map.
+   * the original pending deposit set.
    */
   @SuppressWarnings("unchecked")
-  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+  static ImmutableSet<PendingDeposit> decodePendingDeposits(String encodedPendingDeposits) {
     try (ObjectInputStream ois =
         new ObjectInputStream(
             new ByteArrayInputStream(
-                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
-      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+                BaseEncoding.base64Url().omitPadding().decode(encodedPendingDeposits)))) {
+      return (ImmutableSet<PendingDeposit>) ois.readObject();
     } catch (IOException | ClassNotFoundException e) {
       throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
     }
   }
 
   /**
-   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
-   * worker by the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
+   * the pipeline launcher as a pipeline option.
    */
-  public static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
-      oos.writeObject(pendings);
+      oos.writeObject(pendingDeposits);
       oos.flush();
       return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
     }
@@ -284,14 +686,40 @@ public class RdePipeline implements Serializable {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
     RdePipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
-    // RegistryPipelineWorkerInitializer only initializes before pipeline executions, after the
-    // main() function constructed the graph. We need the registry environment set up so that we
-    // can create a CloudTasksUtils which uses the environment-dependent config file.
-    options.getRegistryEnvironment().setup();
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
+  /**
+   * A utility class that contains {@link TupleTag}s when {@link PCollectionTuple}s and {@link
+   * CoGbkResult}s are used.
+   */
+  protected abstract static class TupleTags {
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> DOMAIN_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_CONTACTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_HOSTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, KV<String, CoGbkResult>>> SUPERORDINATE_DOMAINS =
+        new TupleTag<KV<String, KV<String, CoGbkResult>>>() {};
+
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> EXTERNAL_HOST_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<PendingDeposit> PENDING_DEPOSIT =
+        new TupleTag<PendingDeposit>() {};
+
+    protected static final TupleTag<KV<String, CoGbkResult>> HOST_TO_PENDING_DEPOSIT =
+        new TupleTag<KV<String, CoGbkResult>>() {};
+
+    protected static final TupleTag<Long> REVISION_ID = new TupleTag<Long>() {};
+  }
+
   @Singleton
   @Component(
       modules = {

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -61,7 +61,8 @@ import org.json.JSONObject;
 /**
  * Definition of a Dataflow Flex template, which generates a given month's spec11 report.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=spec11}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -557,11 +557,23 @@ public final class RegistryConfig {
       return config.registryPolicy.requireSslCertificates;
     }
 
+    /**
+     * Returns the GCE machine type that a CPU-demanding pipeline should use.
+     *
+     * @see google.registry.beam.rde.RdePipeline
+     */
+    @Provides
+    @Config("highPerformanceMachineType")
+    public static String provideHighPerformanceMachineType(RegistryConfigSettings config) {
+      return config.beam.highPerformanceMachineType;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
      * @see google.registry.beam.invoicing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
+     * @see google.registry.beam.invoicing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -133,6 +133,7 @@ public class RegistryConfigSettings {
   /** Configuration for Apache Beam (Cloud Dataflow). */
   public static class Beam {
     public String defaultJobRegion;
+    public String highPerformanceMachineType;
     public String stagingBucketUrl;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -419,6 +419,14 @@ misc:
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
   defaultJobRegion: us-east1
+  # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
+  # to check the VM CPU quota for the job region. In a massively parallel
+  # pipeline this quota can be easily reached and needs to be raised, otherwise
+  # the job will run very slowly. Also note that there is a separate quota for
+  # external IPv4 address in a region, which means that machine type with higher
+  # core count per machine may be preferable in order to preserve IP addresses.
+  # See: https://cloud.google.com/compute/quotas#cpu_quota
+  highPerformanceMachineType: n2-standard-4
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -14,18 +14,15 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
-import java.time.Duration;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import javax.inject.Inject;
 
 /** Action for fanning out cron tasks for each commit log bucket. */
@@ -38,25 +35,27 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  private static final Random random = new Random();
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
-  @Inject TaskQueueUtils taskQueueUtils;
   @Inject @Parameter("endpoint") String endpoint;
   @Inject @Parameter("queue") String queue;
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
+
+
   @Override
   public void run() {
-    Queue taskQueue = getQueue(queue);
     for (int bucketId : CommitLogBucket.getBucketIds()) {
-      long delay =
-          jitterSeconds.map(i -> random.nextInt((int) Duration.ofSeconds(i).toMillis())).orElse(0);
-      TaskOptions taskOptions =
-          TaskOptions.Builder.withUrl(endpoint)
-              .param(BUCKET_PARAM, Integer.toString(bucketId))
-              .countdownMillis(delay);
-      taskQueueUtils.enqueue(taskQueue, taskOptions);
+      cloudTasksUtils.enqueue(
+          queue,
+          CloudTasksUtils.createPostTask(
+              endpoint,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
+              clock,
+              jitterSeconds));
     }
   }
 }

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -157,6 +157,12 @@
     <url-pattern>/_dr/cron/readDnsQueue</url-pattern>
   </servlet-mapping>
 
+  <!-- Replicates SQL transactions to Datastore during the Registry 3.0 migration. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/cron/replicateToDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Publishes DNS updates. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/flows/EppController.java

@@ -143,9 +143,6 @@ public final class EppController {
   /** Creates a response indicating an EPP failure. */
   @VisibleForTesting
   static EppOutput getErrorResponse(Result result, Trid trid) {
-    return EppOutput.create(new EppResponse.Builder()
-        .setResult(result)
-        .setTrid(trid)
-        .build());
+    return EppOutput.create(new EppResponse.Builder().setResult(result).setTrid(trid).build());
   }
 }

core/src/main/java/google/registry/flows/EppException.java

@@ -25,10 +25,12 @@ import google.registry.model.eppinput.EppInput.InnerCommand;
 import google.registry.model.eppinput.EppInput.ResourceCommandWrapper;
 import google.registry.model.eppoutput.Result;
 import google.registry.model.eppoutput.Result.Code;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Inherited;
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
+import javax.annotation.Nullable;
 
 /** Exception used to propagate all failures containing one or more EPP responses. */
 public abstract class EppException extends Exception {
@@ -37,7 +39,12 @@ public abstract class EppException extends Exception {
 
   /** Create an EppException with a custom message. */
   private EppException(String message) {
-    super(message);
+    this(message, null);
+  }
+
+  /** Create an EppException with a custom message and cause. */
+  private EppException(String message, @Nullable Throwable cause) {
+    super(message, cause);
     Code code = getClass().getAnnotation(EppResultCode.class).value();
     Preconditions.checkState(!code.isSuccess());
     this.result = Result.create(code, message);
@@ -255,4 +262,12 @@ public abstract class EppException extends Exception {
       super("Specified protocol version is not implemented");
     }
   }
+
+  /** Registry is currently undergoing maintenance and is in read-only mode. */
+  @EppResultCode(Code.COMMAND_FAILED)
+  public static class ReadOnlyModeEppException extends EppException {
+    ReadOnlyModeEppException(ReadOnlyModeException cause) {
+      super("Registry is currently undergoing maintenance and is in read-only mode", cause);
+    }
+  }
 }

core/src/main/java/google/registry/flows/FlowRunner.java

@@ -19,6 +19,7 @@ import static google.registry.xml.XmlTransformer.prettyPrint;
 
 import com.google.common.base.Strings;
 import com.google.common.flogger.FluentLogger;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.FlowModule.DryRun;
 import google.registry.flows.FlowModule.InputXml;
 import google.registry.flows.FlowModule.RegistrarId;
@@ -28,6 +29,7 @@ import google.registry.flows.session.LoginFlow;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppOutput;
 import google.registry.monitoring.whitebox.EppMetric;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import javax.inject.Inject;
 import javax.inject.Provider;
 
@@ -97,6 +99,8 @@ public class FlowRunner {
       return e.output;
     } catch (EppRuntimeException e) {
       throw e.getCause();
+    } catch (ReadOnlyModeException e) {
+      throw new ReadOnlyModeEppException(e);
     }
   }
 

core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java

@@ -50,6 +50,7 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that creates a new contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link ContactFlowUtils.BadInternationalizedPostalInfoException}

core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java

@@ -60,6 +60,7 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java

@@ -54,6 +54,7 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * explicitly approve the transfer request, which then becomes effective immediately.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}

core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java

@@ -54,6 +54,7 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the gaining client to
  * withdraw the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java

@@ -53,6 +53,7 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * reject the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java

@@ -63,6 +63,7 @@ import org.joda.time.Duration;
  * by the losing registrar or rejected, and the gaining registrar can also cancel the transfer
  * request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.AlreadyPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java

@@ -55,6 +55,7 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that updates a contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -133,6 +133,7 @@ import org.joda.time.Duration;
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException}
  * @error {@link
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException}
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -103,6 +103,7 @@ import org.joda.time.Duration;
 /**
  * An EPP flow that deletes a domain.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
@@ -187,7 +188,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
     } else {
       DateTime redemptionTime = now.plus(redemptionGracePeriodLength);
       asyncTaskEnqueuer.enqueueAsyncResave(
-          existingDomain, now, ImmutableSortedSet.of(redemptionTime, deletionTime));
+          existingDomain.createVKey(), now, ImmutableSortedSet.of(redemptionTime, deletionTime));
       builder
           .setDeletionTime(deletionTime)
           .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE))
@@ -371,6 +372,10 @@ public final class DomainDeleteFlow implements TransactionalFlow {
             String.format(
                 "Domain %s was deleted by registry administrator with final deletion effective: %s",
                 existingDomain.getDomainName(), deletionTime))
+        .setResponseData(
+            ImmutableList.of(
+                DomainPendingActionNotificationResponse.create(
+                    existingDomain.getDomainName(), true, trid, now)))
         .build();
   }
 

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -94,6 +94,7 @@ import org.joda.time.Duration;
  * longer than 10 years unless it comes in at the exact millisecond that the domain would have
  * expired.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -93,6 +93,7 @@ import org.joda.time.DateTime;
  * restored to a single year expiration starting at the restore time, regardless of what the
  * original expiration time was.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -110,7 +111,7 @@ import org.joda.time.DateTime;
  * @error {@link DomainRestoreRequestFlow.RestoreCommandIncludesChangesException}
  */
 @ReportingSpec(ActivityReportField.DOMAIN_RGP_RESTORE_REQUEST)
-public final class DomainRestoreRequestFlow implements TransactionalFlow  {
+public final class DomainRestoreRequestFlow implements TransactionalFlow {
 
   @Inject ResourceCommand resourceCommand;
   @Inject ExtensionManager extensionManager;

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -78,6 +78,7 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted and replaced with new ones with the correct approval time.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -65,6 +65,7 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java

@@ -67,6 +67,7 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -93,6 +93,7 @@ import org.joda.time.DateTime;
  * rejection or cancellation of the request, they will be deleted (and in the approval case,
  * replaced with new ones with the correct approval time).
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -102,7 +103,8 @@ import org.joda.time.DateTime;
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}
  * @error {@link google.registry.flows.exceptions.TransferPeriodMustBeOneYearException}
  * @error {@link InvalidTransferPeriodValueException}
- * @error {@link google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
+ * @error {@link
+ *     google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
  * @error {@link DomainFlowUtils.BadPeriodUnitException}
  * @error {@link DomainFlowUtils.CurrencyUnitMismatchException}
  * @error {@link DomainFlowUtils.CurrencyValueScaleException}
@@ -235,7 +237,7 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
             .build();
     DomainHistory domainHistory = buildDomainHistory(newDomain, registry, now, period);
 
-    asyncTaskEnqueuer.enqueueAsyncResave(newDomain, now, automaticTransferTime);
+    asyncTaskEnqueuer.enqueueAsyncResave(newDomain.createVKey(), now, automaticTransferTime);
     tm().putAll(
             new ImmutableSet.Builder<>()
                 .add(newDomain, domainHistory, requestPollMessage)

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -16,6 +16,7 @@ package google.registry.flows.domain;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
 import static com.google.common.collect.Sets.symmetricDifference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.flows.FlowUtils.persistEntityChanges;
@@ -43,6 +44,9 @@ import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_UPDATE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
 import com.google.common.net.InternetDomainName;
 import google.registry.dns.DnsQueue;
 import google.registry.flows.EppException;
@@ -76,6 +80,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
+import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.model.tld.Registry;
 import java.util.Optional;
@@ -92,6 +97,7 @@ import org.joda.time.DateTime;
  * superuser. As such, adding or removing these statuses incurs a billing event. There will be only
  * one charge per update, even if several such statuses are updated at once.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -175,6 +181,9 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     Optional<BillingEvent.OneTime> statusUpdateBillingEvent =
         createBillingEventForStatusUpdates(existingDomain, newDomain, domainHistory, now);
     statusUpdateBillingEvent.ifPresent(entitiesToSave::add);
+    Optional<PollMessage.OneTime> serverStatusUpdatePollMessage =
+        createPollMessageForServerStatusUpdates(existingDomain, newDomain, domainHistory, now);
+    serverStatusUpdatePollMessage.ifPresent(entitiesToSave::add);
     EntityChanges entityChanges =
         flowCustomLogic.beforeSave(
             BeforeSaveParameters.newBuilder()
@@ -306,4 +315,50 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     }
     return Optional.empty();
   }
+
+  /** Enqueues a poll message iff a superuser is adding/removing server statuses. */
+  private Optional<PollMessage.OneTime> createPollMessageForServerStatusUpdates(
+      DomainBase existingDomain, DomainBase newDomain, DomainHistory historyEntry, DateTime now) {
+    if (registrarId.equals(existingDomain.getPersistedCurrentSponsorRegistrarId())) {
+      // Don't send a poll message when a superuser registrar is updating its own domain.
+      return Optional.empty();
+    }
+    ImmutableSortedSet<String> addedServerStatuses =
+        Sets.difference(newDomain.getStatusValues(), existingDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+    ImmutableSortedSet<String> removedServerStatuses =
+        Sets.difference(existingDomain.getStatusValues(), newDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+
+    String msg = "";
+    if (addedServerStatuses.size() > 0 && removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s and removed the status(es)"
+                  + " %s.",
+              addedServerStatuses, removedServerStatuses);
+    } else if (addedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s.", addedServerStatuses);
+    } else if (removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has removed the status(es) %s.", removedServerStatuses);
+    } else {
+      return Optional.empty();
+    }
+
+    return Optional.ofNullable(
+        new PollMessage.OneTime.Builder()
+            .setParent(historyEntry)
+            .setEventTime(now)
+            .setRegistrarId(existingDomain.getCurrentSponsorRegistrarId())
+            .setMsg(msg)
+            .build());
+  }
 }

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -65,6 +65,7 @@ import org.joda.time.DateTime;
  * hosts cannot have any. This flow allows creating a host name, and if necessary enqueues tasks to
  * update DNS.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.IpAddressVersionMismatchException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -58,6 +58,7 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -73,11 +73,12 @@ import org.joda.time.DateTime;
  * hosts. Internal hosts must have at least one IP address associated with them, whereas external
  * hosts cannot have any.
  *
- * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When
- * a host is renamed from internal to external all IP addresses must be simultaneously removed, and
+ * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When a
+ * host is renamed from internal to external all IP addresses must be simultaneously removed, and
  * when it is renamed from external to internal at least one must be added. If the host is renamed
  * or IP addresses are added, tasks are enqueued to update DNS accordingly.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/model/AppEngineEnvironment.java

@@ -43,10 +43,22 @@ public class AppEngineEnvironment {
 
   private Environment environment;
 
+  /**
+   * Constructor for use by tests.
+   *
+   * <p>All test suites must use the same appId for environments, since when tearing down we do not
+   * clear cached environments in spawned threads. See {@link #unsetEnvironmentForAllThreads} for
+   * more information.
+   */
   public AppEngineEnvironment() {
-    this("PlaceholderAppId");
+    /**
+     * Use AppEngineExtension's appId here so that ofy and sql entities can be compared with {@code
+     * Objects#equals()}. The choice of this value does not impact functional correctness.
+     */
+    this("test");
   }
 
+  /** Constructor for use by applications, e.g., BEAM pipelines. */
   public AppEngineEnvironment(String appId) {
     environment = createAppEngineEnvironment(appId);
   }
@@ -65,7 +77,17 @@ public class AppEngineEnvironment {
     ApiProxy.clearEnvironmentForCurrentThread();
   }
 
+  /**
+   * Unsets the test environment in all threads with best effort.
+   *
+   * <p>This method unsets the environment factory and clears the cached environment in the current
+   * thread (the main test runner thread). We do not clear the cache in spawned threads, even though
+   * they may be reused. This is not a problem as long as the appId stays the same: those threads
+   * are used only in AppEngine or BEAM tests, and expect the presence of an environment.
+   */
   public void unsetEnvironmentForAllThreads() {
+    unsetEnvironmentForCurrentThread();
+
     try {
       Method method = ApiProxy.class.getDeclaredMethod("clearEnvironmentFactory");
       method.setAccessible(true);

core/src/main/java/google/registry/model/BackupGroupRoot.java

@@ -14,6 +14,7 @@
 
 package google.registry.model;
 
+import google.registry.util.PreconditionsUtils;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.AttributeOverride;
@@ -30,7 +31,7 @@ import javax.xml.bind.annotation.XmlTransient;
  * that we can enforce strictly increasing timestamps.
  */
 @MappedSuperclass
-public abstract class BackupGroupRoot extends ImmutableObject {
+public abstract class BackupGroupRoot extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * An automatically managed timestamp of when this object was last written to Datastore.
@@ -49,4 +50,14 @@ public abstract class BackupGroupRoot extends ImmutableObject {
   public UpdateAutoTimestamp getUpdateTimestamp() {
     return updateTimestamp;
   }
+
+  /**
+   * Copies {@link #updateTimestamp} from another entity.
+   *
+   * <p>This method is for the few cases when {@code updateTimestamp} is copied between different
+   * types of entities. Use {@link #clone} for same-type copying.
+   */
+  protected void copyUpdateTimestamp(BackupGroupRoot other) {
+    this.updateTimestamp = PreconditionsUtils.checkArgumentNotNull(other, "other").updateTimestamp;
+  }
 }

core/src/main/java/google/registry/model/CreateAutoTimestamp.java

@@ -14,8 +14,20 @@
 
 package google.registry.model;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.OnLoad;
 import google.registry.model.translators.CreateAutoTimestampTranslatorFactory;
+import google.registry.util.DateTimeUtils;
+import java.time.ZonedDateTime;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.PostLoad;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -23,9 +35,37 @@ import org.joda.time.DateTime;
  *
  * @see CreateAutoTimestampTranslatorFactory
  */
-public class CreateAutoTimestamp extends ImmutableObject {
+@Embeddable
+public class CreateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
-  DateTime timestamp;
+  @Transient DateTime timestamp;
+
+  @Column(nullable = false)
+  @Ignore
+  ZonedDateTime creationTime;
+
+  @PrePersist
+  @PreUpdate
+  void setTimestamp() {
+    if (creationTime == null) {
+      timestamp = jpaTm().getTransactionTime();
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @OnLoad
+  void onLoad() {
+    if (timestamp != null) {
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (creationTime != null) {
+      timestamp = DateTimeUtils.toJodaDateTime(creationTime);
+    }
+  }
 
   /** Returns the timestamp. */
   @Nullable
@@ -36,6 +76,7 @@ public class CreateAutoTimestamp extends ImmutableObject {
   public static CreateAutoTimestamp create(@Nullable DateTime timestamp) {
     CreateAutoTimestamp instance = new CreateAutoTimestamp();
     instance.timestamp = timestamp;
+    instance.creationTime = (timestamp == null) ? null : DateTimeUtils.toZonedDateTime(timestamp);
     return instance;
   }
 }

core/src/main/java/google/registry/model/EntityClasses.java

@@ -43,7 +43,6 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.server.Lock;
 import google.registry.model.server.ServerSecret;
 import google.registry.model.tld.Registry;
-import google.registry.model.tmch.TmchCrl;
 
 /** Sets of classes of the Objectify-registered entities in use throughout the model. */
 public final class EntityClasses {
@@ -85,8 +84,7 @@ public final class EntityClasses {
           Registrar.class,
           RegistrarContact.class,
           Registry.class,
-          ServerSecret.class,
-          TmchCrl.class);
+          ServerSecret.class);
 
   private EntityClasses() {}
 }

core/src/main/java/google/registry/model/EppResource.java

@@ -50,6 +50,8 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -112,7 +114,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
    * resource fields.
    */
-  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  @AttributeOverrides({@AttributeOverride(name = "creationTime", column = @Column())})
+  @Index
+  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -224,10 +228,8 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /** Used when replaying from SQL to DS to populate the Datastore indexes. */
   protected void saveIndexesToDatastore() {
-    ofyTm()
-        .putAll(
-            ForeignKeyIndex.create(this, getDeletionTime()),
-            EppResourceIndex.create(Key.create(this)));
+    ofyTm().putIgnoringReadOnly(ForeignKeyIndex.create(this, getDeletionTime()));
+    ofyTm().putIgnoringReadOnly(EppResourceIndex.create(Key.create(this)));
   }
 
   /** EppResources that are loaded via foreign keys should implement this marker interface. */

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -85,6 +85,9 @@ public abstract class ImmutableObject implements Cloneable {
   @Target(FIELD)
   public @interface Insignificant {}
 
+  // Note: if this class is made to implement Serializable, this field must become 'transient' since
+  // hashing is not stable across executions. Also note that @XmlTransient is forbidden on transient
+  // fields and need to be removed if transient is added.
   @Ignore @XmlTransient protected Integer hashCode;
 
   private boolean equalsImmutableObject(ImmutableObject other) {

core/src/main/java/google/registry/model/UnsafeSerializable.java

@@ -0,0 +1,34 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import java.io.Serializable;
+
+/**
+ * Marker interface for Nomulus entities whose serialization are implemented in a fragile way. These
+ * entities are made {@link Serializable} so that they can be passed between JVMs. The intended use
+ * case is BEAM pipeline-based cross-database data validation between Datastore and Cloud SQL during
+ * the migration. Note that only objects loaded from the SQL database need serialization support.
+ * Objects exported from Datastore can already be serialized as protocol buffers.
+ *
+ * <p>All entities implementing this interface take advantage of the fact that all Java collection
+ * classes we use, either directly or indirectly, including those in Java libraries, Guava,
+ * Objectify, and Hibernate are {@code Serializable}.
+ *
+ * <p>The {@code serialVersionUID} field has also been omitted in the implementing classes, since
+ * they are not used for persistence.
+ */
+// TODO(b/203609782): either remove this interface or fix implementors post migration.
+public interface UnsafeSerializable extends Serializable {}

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -38,7 +38,7 @@ import org.joda.time.DateTime;
  * @see UpdateAutoTimestampTranslatorFactory
  */
 @Embeddable
-public class UpdateAutoTimestamp extends ImmutableObject {
+public class UpdateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
   // When set to true, database converters/translators should do the auto update.  When set to
   // false, auto update should be suspended (this exists to allow us to preserve the original value

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -27,7 +27,6 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -39,6 +38,7 @@ import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.TimeOfYear;
 import google.registry.model.domain.DomainBase;
@@ -72,18 +72,33 @@ import org.joda.time.DateTime;
 /** A billable event in a domain's lifecycle. */
 @MappedSuperclass
 public abstract class BillingEvent extends ImmutableObject
-    implements Buildable, TransferServerApproveEntity {
+    implements Buildable, TransferServerApproveEntity, UnsafeSerializable {
 
   /** The reason for the bill, which maps 1:1 to skus in go/registry-billing-skus. */
   public enum Reason {
-    CREATE,
+    CREATE(true),
     @Deprecated // TODO(b/31676071): remove this legacy value once old data is cleaned up.
-    ERROR,
-    FEE_EARLY_ACCESS,
-    RENEW,
-    RESTORE,
-    SERVER_STATUS,
-    TRANSFER
+    ERROR(false),
+    FEE_EARLY_ACCESS(true),
+    RENEW(true),
+    RESTORE(true),
+    SERVER_STATUS(false),
+    TRANSFER(true);
+
+    private final boolean requiresPeriod;
+
+    Reason(boolean requiresPeriod) {
+      this.requiresPeriod = requiresPeriod;
+    }
+
+    /**
+     * Returns whether billing events with this reason have a period years associated with them.
+     *
+     * <p>Note that this is an "if an only if" condition.
+     */
+    public boolean hasPeriodYears() {
+      return requiresPeriod;
+    }
   }
 
   /** Set of flags that can be applied to billing events. */
@@ -267,7 +282,7 @@ public abstract class BillingEvent extends ImmutableObject
     public T build() {
       T instance = getInstance();
       checkNotNull(instance.reason, "Reason must be set");
-      checkNotNull(instance.clientId, "Client ID must be set");
+      checkNotNull(instance.clientId, "Registrar ID must be set");
       checkNotNull(instance.eventTime, "Event time must be set");
       checkNotNull(instance.targetId, "Target ID must be set");
       checkNotNull(instance.parent, "Parent must be set");
@@ -461,17 +476,14 @@ public abstract class BillingEvent extends ImmutableObject
         checkNotNull(instance.billingTime);
         checkNotNull(instance.cost);
         checkState(!instance.cost.isNegative(), "Costs should be non-negative.");
-        ImmutableSet<Reason> reasonsWithPeriods =
-            Sets.immutableEnumSet(
-                Reason.CREATE,
-                Reason.FEE_EARLY_ACCESS,
-                Reason.RENEW,
-                Reason.RESTORE,
-                Reason.TRANSFER);
-        checkState(
-            reasonsWithPeriods.contains(instance.reason) == (instance.periodYears != null),
-            "Period years must be set if and only if reason is "
-                + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        // TODO(mcilwain): Enforce this check on all billing events (not just more recent ones)
+        //                 post-migration after we add the missing period years values in SQL.
+        if (instance.eventTime.isAfter(DateTime.parse("2019-01-01T00:00:00Z"))) {
+          checkState(
+              instance.reason.hasPeriodYears() == (instance.periodYears != null),
+              "Period years must be set if and only if reason is "
+                  + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        }
         checkState(
             instance.getFlags().contains(Flag.SYNTHETIC)
                 == (instance.syntheticCreationTime != null),

core/src/main/java/google/registry/model/bulkquery/BulkQueryEntities.java

@@ -0,0 +1,109 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+
+/**
+ * Utilities for managing an alternative JPA entity model optimized for bulk loading multi-level
+ * entities such as {@link DomainBase} and {@link DomainHistory}.
+ *
+ * <p>In a bulk query for a multi-level JPA entity type, the JPA framework only generates a bulk
+ * query (SELECT * FROM table) for the base table. Then, for each row in the base table, additional
+ * queries are issued to load associated rows in child tables. This can be very slow when an entity
+ * type has multiple child tables.
+ *
+ * <p>We have defined an alternative entity model for {@code DomainBase} and {@code DomainHistory},
+ * where the base table as well as the child tables are mapped to single-level entity types. The
+ * idea is to load each of these types using a bulk query, and assemble them into the target type in
+ * memory in a pipeline. The main use case is Datastore-Cloud SQL validation during the Registry
+ * database migration, where we will need the full database snapshots frequently.
+ */
+public class BulkQueryEntities {
+  /**
+   * The JPA entity classes in persistence.xml to replace when creating the {@link
+   * JpaTransactionManager} for bulk query.
+   */
+  public static final ImmutableMap<String, String> JPA_ENTITIES_REPLACEMENTS =
+      ImmutableMap.of(
+          DomainBase.class.getCanonicalName(),
+          DomainBaseLite.class.getCanonicalName(),
+          DomainHistory.class.getCanonicalName(),
+          DomainHistoryLite.class.getCanonicalName());
+
+  /* The JPA entity classes that are not included in persistence.xml and need to be added to
+   * the {@link JpaTransactionManager} for bulk query.*/
+  public static final ImmutableList<String> JPA_ENTITIES_NEW =
+      ImmutableList.of(
+          DomainHost.class.getCanonicalName(), DomainHistoryHost.class.getCanonicalName());
+
+  public static DomainBase assembleDomainBase(
+      DomainBaseLite domainBaseLite,
+      ImmutableSet<GracePeriod> gracePeriods,
+      ImmutableSet<DelegationSignerData> delegationSignerData,
+      ImmutableSet<VKey<HostResource>> nsHosts) {
+    DomainBase.Builder builder = new DomainBase.Builder();
+    builder.copyFrom(domainBaseLite);
+    builder.setGracePeriods(gracePeriods);
+    builder.setDsData(delegationSignerData);
+    builder.setNameservers(nsHosts);
+    return builder.build();
+  }
+
+  public static DomainHistory assembleDomainHistory(
+      DomainHistoryLite domainHistoryLite,
+      ImmutableSet<DomainDsDataHistory> dsDataHistories,
+      ImmutableSet<VKey<HostResource>> domainHistoryHosts,
+      ImmutableSet<GracePeriodHistory> gracePeriodHistories,
+      ImmutableSet<DomainTransactionRecord> transactionRecords) {
+    DomainHistory.Builder builder = new DomainHistory.Builder();
+    builder.copyFrom(domainHistoryLite);
+    DomainContent rawDomainContent = domainHistoryLite.domainContent;
+    if (rawDomainContent != null) {
+      DomainContent newDomainContent =
+          domainHistoryLite
+              .domainContent
+              .asBuilder()
+              .setNameservers(domainHistoryHosts)
+              .setGracePeriods(
+                  gracePeriodHistories.stream()
+                      .map(GracePeriod::createFromHistory)
+                      .collect(toImmutableSet()))
+              .setDsData(
+                  dsDataHistories.stream()
+                      .map(DelegationSignerData::create)
+                      .collect(toImmutableSet()))
+              .build();
+      builder.setDomain(newDomainContent);
+    }
+    return builder.buildAndAssemble(
+        dsDataHistories, domainHistoryHosts, gracePeriodHistories, transactionRecords);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java

@@ -0,0 +1,49 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+
+/**
+ * A 'light' version of {@link DomainBase} with only base table ("Domain") attributes, which allows
+ * fast bulk loading. They are used in in-memory assembly of {@code DomainBase} instances along with
+ * bulk-loaded child entities ({@code GracePeriod} etc). The in-memory assembly achieves much higher
+ * performance than loading {@code DomainBase} directly.
+ *
+ * <p>Please refer to {@link BulkQueryEntities} for more information.
+ */
+@Entity(name = "Domain")
+@WithStringVKey
+@Access(AccessType.FIELD)
+public class DomainBaseLite extends DomainContent implements SqlOnlyEntity {
+
+  @Override
+  @javax.persistence.Id
+  @Access(AccessType.PROPERTY)
+  public String getRepoId() {
+    return super.getRepoId();
+  }
+
+  public static VKey<DomainBaseLite> createVKey(String repoId) {
+    return VKey.createSql(DomainBaseLite.class, repoId);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java

@@ -0,0 +1,50 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.host.HostResource;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+
+/**
+ * A name server host referenced by a {@link google.registry.model.domain.DomainHistory} record.
+ * Please refer to {@link BulkQueryEntities} for usage.
+ */
+@Entity
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryHost.class)
+public class DomainHistoryHost implements Serializable, SqlOnlyEntity {
+
+  @Id private Long domainHistoryHistoryRevisionId;
+  @Id private String domainHistoryDomainRepoId;
+  @Id private String hostRepoId;
+
+  private DomainHistoryHost() {}
+
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(domainHistoryDomainRepoId, domainHistoryHistoryRevisionId);
+  }
+
+  public VKey<HostResource> getHostVKey() {
+    return VKey.create(HostResource.class, hostRepoId);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java

@@ -0,0 +1,122 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.Period;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
+
+/**
+ * A 'light' version of {@link DomainHistory} with only base table ("DomainHistory") attributes,
+ * which allows fast bulk loading. They are used in in-memory assembly of {@code DomainHistory}
+ * instances along with bulk-loaded child entities ({@code GracePeriodHistory} etc). The in-memory
+ * assembly achieves much higher performance than loading {@code DomainHistory} directly.
+ *
+ * <p>Please refer to {@link BulkQueryEntities} for more information.
+ *
+ * <p>This class is adapted from {@link DomainHistory} by removing the {@code dsDataHistories},
+ * {@code gracePeriodHistories}, and {@code nsHosts} fields and associated methods.
+ */
+@Entity(name = "DomainHistory")
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryId.class)
+public class DomainHistoryLite extends HistoryEntry implements SqlOnlyEntity {
+
+  // Store DomainContent instead of DomainBase so we don't pick up its @Id
+  // Nullable for the sake of pre-Registry-3.0 history objects
+  @Nullable DomainContent domainContent;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    parent = Key.create(DomainBase.class, domainRepoId);
+  }
+
+  @Override
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @AttributeOverrides({
+    @AttributeOverride(name = "unit", column = @Column(name = "historyPeriodUnit")),
+    @AttributeOverride(name = "value", column = @Column(name = "historyPeriodValue"))
+  })
+  public Period getPeriod() {
+    return super.getPeriod();
+  }
+
+  /**
+   * For transfers, the id of the other registrar.
+   *
+   * <p>For requests and cancels, the other registrar is the losing party (because the registrar
+   * sending the EPP transfer command is the gaining party). For approves and rejects, the other
+   * registrar is the gaining party.
+   */
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @Column(name = "historyOtherRegistrarId")
+  @Override
+  public String getOtherRegistrarId() {
+    return super.getOtherRegistrarId();
+  }
+
+  @Id
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
+  /** The key to the {@link DomainBase} this is based off of. */
+  public VKey<DomainBase> getParentVKey() {
+    return VKey.create(DomainBase.class, getDomainRepoId());
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (domainContent == null) {
+      return;
+    }
+    // See inline comments in DomainHistory.postLoad for reasons for the following lines.
+    if (domainContent.getDomainName() == null) {
+      domainContent = null;
+    } else if (domainContent.getRepoId() == null) {
+      domainContent.setRepoId(parent.getName());
+    }
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHost.java

@@ -0,0 +1,46 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.host.HostResource;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+
+/** A name server host of a domain. Please refer to {@link BulkQueryEntities} for usage. */
+@Entity
+@Access(AccessType.FIELD)
+@IdClass(DomainHost.class)
+public class DomainHost implements Serializable, SqlOnlyEntity {
+
+  @Id private String domainRepoId;
+
+  @Id private String hostRepoId;
+
+  DomainHost() {}
+
+  public String getDomainRepoId() {
+    return domainRepoId;
+  }
+
+  public VKey<HostResource> getHostVKey() {
+    return VKey.create(HostResource.class, hostRepoId);
+  }
+}

core/src/main/java/google/registry/model/common/Cursor.java

@@ -27,13 +27,13 @@ import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.common.Cursor.CursorId;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.model.tld.Registry;
 import google.registry.persistence.VKey;
-import java.io.Serializable;
 import java.util.List;
 import java.util.Optional;
 import javax.persistence.Column;
@@ -53,7 +53,7 @@ import org.joda.time.DateTime;
 @javax.persistence.Entity
 @IdClass(CursorId.class)
 @InCrossTld
-public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
+public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity, UnsafeSerializable {
 
   /** The scope of a global cursor. A global cursor is a cursor that is not specific to one tld. */
   public static final String GLOBAL = "GLOBAL";
@@ -283,7 +283,7 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
     return cursorTime;
   }
 
-  static class CursorId extends ImmutableObject implements Serializable {
+  public static class CursorId extends ImmutableObject implements UnsafeSerializable {
 
     public CursorType type;
     public String scope;

core/src/main/java/google/registry/model/common/TimeOfYear.java

@@ -28,6 +28,7 @@ import com.google.common.collect.Range;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.List;
 import javax.persistence.Embeddable;
 import org.joda.time.DateTime;
@@ -45,7 +46,7 @@ import org.joda.time.DateTime;
  */
 @Embed
 @Embeddable
-public class TimeOfYear extends ImmutableObject {
+public class TimeOfYear extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * The time as "month day millis" with all fields left-padded with zeroes so that lexographic

core/src/main/java/google/registry/model/common/TimedTransitionProperty.java

@@ -27,7 +27,9 @@ import com.google.common.collect.Maps;
 import com.google.common.collect.Ordering;
 import com.googlecode.objectify.mapper.Mapper;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.util.TypeUtils;
+import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -53,11 +55,12 @@ import org.joda.time.DateTime;
  * to use for storing the list of transitions. The user is given this choice of subclass so that the
  * field of the value type stored in the transition can be given a customized name.
  */
-public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedTransition<V>>
-    extends ForwardingMap<DateTime, T> {
+public class TimedTransitionProperty<
+        V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
+    extends ForwardingMap<DateTime, T> implements UnsafeSerializable {
 
   /**
-   * A transition to a value of type {@code V} at a certain time.  This superclass only has a field
+   * A transition to a value of type {@code V} at a certain time. This superclass only has a field
    * for the {@code DateTime}, which means that subclasses should supply the field of type {@code V}
    * and implementations of the abstract getter and setter methods to access that field. This design
    * is so that subclasses tagged with @Embed can define a custom field name for their value, for
@@ -65,11 +68,12 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    *
    * <p>The public visibility of this class exists only so that it can be subclassed; clients should
    * never call any methods on this class or attempt to access its members, but should instead treat
-   * it as a customizable implementation detail of {@code TimedTransitionProperty}.  However, note
+   * it as a customizable implementation detail of {@code TimedTransitionProperty}. However, note
    * that subclasses must also have public visibility so that they can be instantiated via
    * reflection in a call to {@code fromValueMap}.
    */
-  public abstract static class TimedTransition<V> extends ImmutableObject {
+  public abstract static class TimedTransition<V extends Serializable> extends ImmutableObject
+      implements UnsafeSerializable {
     /** The time at which this value becomes the active value. */
     private DateTime transitionTime;
 
@@ -89,16 +93,16 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
   }
 
   /**
-   * Converts the provided value map into the equivalent transition map, using transition objects
-   * of the given TimedTransition subclass.  The value map must be sorted according to the natural
+   * Converts the provided value map into the equivalent transition map, using transition objects of
+   * the given TimedTransition subclass. The value map must be sorted according to the natural
    * ordering of its DateTime keys, and keys cannot be earlier than START_OF_TIME.
    */
   // NB: The Class<T> parameter could be eliminated by getting the class via reflection, but then
   // the callsite cannot infer T, so unless you explicitly call this as .<V, T>fromValueMap() it
   // will default to using just TimedTransition<V>, which fails at runtime.
-  private static <V, T extends TimedTransition<V>> NavigableMap<DateTime, T> makeTransitionMap(
-      ImmutableSortedMap<DateTime, V> valueMap,
-      final Class<T> timedTransitionSubclass) {
+  private static <V extends Serializable, T extends TimedTransition<V>>
+      NavigableMap<DateTime, T> makeTransitionMap(
+          ImmutableSortedMap<DateTime, V> valueMap, final Class<T> timedTransitionSubclass) {
     checkArgument(
         Ordering.natural().equals(valueMap.comparator()),
         "Timed transition value map must have transition time keys in chronological order");
@@ -121,9 +125,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    *
    * <p>This method should be the normal method for constructing a {@link TimedTransitionProperty}.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> fromValueMap(
-      ImmutableSortedMap<DateTime, V> valueMap,
-      final Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> fromValueMap(
+          ImmutableSortedMap<DateTime, V> valueMap, final Class<T> timedTransitionSubclass) {
     return new TimedTransitionProperty<>(ImmutableSortedMap.copyOf(
         makeTransitionMap(valueMap, timedTransitionSubclass)));
   }
@@ -175,10 +179,10 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * @param allowedTransitions optional map of all possible state-to-state transitions
    * @param allowedTransitionMapName optional transition map description string for error messages
    * @param initialValue optional initial value; if present, the first transition must have this
-   *        value
+   *     value
    * @param badInitialValueErrorMessage option error message string if the initial value is wrong
    */
-  public static <V, T extends TimedTransitionProperty.TimedTransition<V>>
+  public static <V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
       TimedTransitionProperty<V, T> make(
           ImmutableSortedMap<DateTime, V> newTransitions,
           Class<T> transitionClass,
@@ -200,7 +204,7 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * Validates that a transition map is not null or empty, starts at START_OF_TIME, and has
    * transitions which move from one value to another in allowed ways.
    */
-  public static <V, T extends TimedTransitionProperty.TimedTransition<V>>
+  public static <V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
       void validateTimedTransitionMap(
           @Nullable NavigableMap<DateTime, V> transitionMap,
           ImmutableMultimap<V, V> allowedTransitions,
@@ -240,8 +244,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * annotation. The map for those fields must be mutable so that Objectify can load values from
    * Datastore into the map, but clients should still never mutate the field's map directly.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> forMapify(
-      ImmutableSortedMap<DateTime, V> valueMap, Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> forMapify(
+          ImmutableSortedMap<DateTime, V> valueMap, Class<T> timedTransitionSubclass) {
     return new TimedTransitionProperty<>(
         new TreeMap<>(makeTransitionMap(valueMap, timedTransitionSubclass)));
   }
@@ -254,8 +259,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * annotation. The map for those fields must be mutable so that Objectify can load values from
    * Datastore into the map, but clients should still never mutate the field's map directly.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> forMapify(
-      V valueAtStartOfTime, Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> forMapify(
+          V valueAtStartOfTime, Class<T> timedTransitionSubclass) {
     return forMapify(
         ImmutableSortedMap.of(START_OF_TIME, valueAtStartOfTime), timedTransitionSubclass);
   }

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -20,6 +20,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.contact.ContactHistory.ContactHistoryId;
 import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.SqlEntity;
@@ -59,7 +60,7 @@ import javax.persistence.PostLoad;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(ContactHistoryId.class)
-public class ContactHistory extends HistoryEntry implements SqlEntity {
+public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable {
 
   // Store ContactBase instead of ContactResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects

core/src/main/java/google/registry/model/contact/Disclose.java

@@ -20,7 +20,9 @@ import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.eppcommon.PresenceMarker;
+import java.io.Serializable;
 import java.util.List;
 import javax.persistence.Embeddable;
 import javax.persistence.Embedded;
@@ -31,7 +33,7 @@ import javax.xml.bind.annotation.XmlType;
 @Embed
 @Embeddable
 @XmlType(propOrder = {"name", "org", "addr", "voice", "fax", "email"})
-public class Disclose extends ImmutableObject {
+public class Disclose extends ImmutableObject implements UnsafeSerializable {
 
   List<PostalInfoChoice> name;
 
@@ -78,7 +80,7 @@ public class Disclose extends ImmutableObject {
 
   /** The "intLocType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
   @Embed
-  public static class PostalInfoChoice extends ImmutableObject {
+  public static class PostalInfoChoice extends ImmutableObject implements Serializable {
     @XmlAttribute
     PostalInfo.Type type;
 

core/src/main/java/google/registry/model/contact/PostalInfo.java

@@ -20,6 +20,7 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.Buildable;
 import google.registry.model.Buildable.Overlayable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.persistence.Embeddable;
 import javax.persistence.EnumType;
@@ -38,7 +39,8 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @Embed
 @Embeddable
 @XmlType(propOrder = {"name", "org", "address", "type"})
-public class PostalInfo extends ImmutableObject implements Overlayable<PostalInfo> {
+public class PostalInfo extends ImmutableObject
+    implements Overlayable<PostalInfo>, UnsafeSerializable {
 
   /** The type of the address, either localized or international. */
   public enum Type {

core/src/main/java/google/registry/model/domain/DesignatedContact.java

@@ -21,6 +21,7 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.contact.ContactResource;
 import google.registry.persistence.VKey;
 import javax.persistence.Embeddable;
@@ -46,7 +47,7 @@ import javax.xml.bind.annotation.XmlEnumValue;
  */
 @Embed
 @Embeddable
-public class DesignatedContact extends ImmutableObject {
+public class DesignatedContact extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * XML type for contact types. This can be either: {@code "admin"}, {@code "billing"}, or

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -24,6 +24,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import java.util.Set;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -164,6 +165,11 @@ public class DomainBase extends DomainContent
     return cloneDomainProjectedAtTime(this, now);
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();
@@ -189,6 +195,7 @@ public class DomainBase extends DomainContent
     }
 
     public Builder copyFrom(DomainContent domainContent) {
+      this.getInstance().copyUpdateTimestamp(domainContent);
       return this.setAuthInfo(domainContent.getAuthInfo())
           .setAutorenewPollMessage(domainContent.getAutorenewPollMessage())
           .setAutorenewBillingEvent(domainContent.getAutorenewBillingEvent())

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -33,6 +33,7 @@ import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.HashSet;
 import java.util.Optional;
@@ -216,6 +217,10 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     return super.getId();
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), getId());
+  }
+
   /** Returns keys to the {@link HostResource} that are the nameservers for the domain. */
   public Set<VKey<HostResource>> getNsHosts() {
     return nsHosts;
@@ -299,6 +304,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   public void beforeSqlSaveOnReplay() {
     if (domainContent == null) {
       domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
+      domainContent.fullyQualifiedDomainName =
+          DomainNameUtils.canonicalizeDomainName(domainContent.fullyQualifiedDomainName);
       fillAuxiliaryFieldsFromDomain(this);
     }
   }
@@ -314,6 +321,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
           nullToEmptyImmutableCopy(domainHistory.domainContent.getGracePeriods()).stream()
               .map(gracePeriod -> GracePeriodHistory.createFrom(domainHistory.id, gracePeriod))
               .collect(toImmutableSet());
+    } else {
+      domainHistory.nsHosts = ImmutableSet.of();
     }
   }
 
@@ -393,8 +402,16 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       if (domainContent == null) {
         return this;
       }
+      // TODO(b/203609982): if actual type of domainContent is DomainBase, convert to DomainContent
+      // Note: a DomainHistory fetched by JPA has DomainContent in this field. Allowing DomainBase
+      // in the setter makes equality checks messy.
       getInstance().domainContent = domainContent;
-      return super.setParent(domainContent);
+      if (domainContent instanceof DomainBase) {
+        super.setParent(domainContent);
+      } else {
+        super.setParent(Key.create(DomainBase.class, domainContent.getRepoId()));
+      }
+      return this;
     }
 
     public Builder setDomainRepoId(String domainRepoId) {
@@ -412,5 +429,19 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       fillAuxiliaryFieldsFromDomain(instance);
       return instance;
     }
+
+    public DomainHistory buildAndAssemble(
+        ImmutableSet<DomainDsDataHistory> dsDataHistories,
+        ImmutableSet<VKey<HostResource>> domainHistoryHosts,
+        ImmutableSet<GracePeriodHistory> gracePeriodHistories,
+        ImmutableSet<DomainTransactionRecord> transactionRecords) {
+      DomainHistory instance = super.build();
+      instance.dsDataHistories = dsDataHistories;
+      instance.nsHosts = domainHistoryHosts;
+      instance.gracePeriodHistories = gracePeriodHistories;
+      instance.domainTransactionRecords = transactionRecords;
+      instance.hashCode = null;
+      return instance;
+    }
   }
 }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.model.replay.SqlOnlyEntity;
@@ -203,7 +204,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
   /** Entity class to represent a historic {@link GracePeriod}. */
   @Entity(name = "GracePeriodHistory")
   @Table(indexes = @Index(columnList = "domainRepoId"))
-  static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity {
+  public static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity {
     @Id Long gracePeriodHistoryRevisionId;
 
     /** ID for the associated {@link DomainHistory} entity. */
@@ -215,6 +216,10 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
       return super.getGracePeriodId();
     }
 
+    public DomainHistoryId getDomainHistoryId() {
+      return new DomainHistoryId(getDomainRepoId(), domainHistoryRevisionId);
+    }
+
     static GracePeriodHistory createFrom(long historyRevisionId, GracePeriod gracePeriod) {
       GracePeriodHistory instance = new GracePeriodHistory();
       instance.gracePeriodHistoryRevisionId = allocateId();

core/src/main/java/google/registry/model/domain/GracePeriodBase.java

@@ -17,6 +17,7 @@ package google.registry.model.domain;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.persistence.BillingVKey.BillingEventVKey;
@@ -35,7 +36,7 @@ import org.joda.time.DateTime;
 @Embed
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public class GracePeriodBase extends ImmutableObject {
+public class GracePeriodBase extends ImmutableObject implements UnsafeSerializable {
 
   /** Unique id required for hibernate representation. */
   @Transient long gracePeriodId;

core/src/main/java/google/registry/model/domain/Period.java

@@ -16,6 +16,7 @@ package google.registry.model.domain;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -25,7 +26,7 @@ import javax.xml.bind.annotation.XmlValue;
 /** The "periodType" from <a href="http://tools.ietf.org/html/rfc5731">RFC5731</a>. */
 @Embed
 @javax.persistence.Embeddable
-public class Period extends ImmutableObject {
+public class Period extends ImmutableObject implements UnsafeSerializable {
 
   @Enumerated(EnumType.STRING)
   @XmlAttribute

core/src/main/java/google/registry/model/domain/RegistryLock.java

@@ -28,6 +28,8 @@ import google.registry.util.DateTimeUtils;
 import java.time.ZonedDateTime;
 import java.util.Optional;
 import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
@@ -100,7 +102,11 @@ public final class RegistryLock extends ImmutableObject implements Buildable, Sq
   private String registrarPocId;
 
   /** When the lock is first requested. */
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "lockRequestTime", nullable = false))
+  })
   private CreateAutoTimestamp lockRequestTime = CreateAutoTimestamp.create(null);
 
   /** When the unlock is first requested. */

core/src/main/java/google/registry/model/domain/launch/LaunchNotice.java

@@ -27,6 +27,7 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.persistence.Embedded;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -39,7 +40,7 @@ import org.joda.time.DateTime;
 @Embed
 @XmlType(propOrder = {"noticeId", "expirationTime", "acceptedTime"})
 @javax.persistence.Embeddable
-public class LaunchNotice extends ImmutableObject {
+public class LaunchNotice extends ImmutableObject implements UnsafeSerializable {
 
   /** An empty instance to use in place of null. */
   private static final NoticeIdType EMPTY_NOTICE_ID = new NoticeIdType();
@@ -47,14 +48,13 @@ public class LaunchNotice extends ImmutableObject {
   /** An id with a validator-id attribute. */
   @Embed
   @javax.persistence.Embeddable
-  public static class NoticeIdType extends ImmutableObject {
+  public static class NoticeIdType extends ImmutableObject implements UnsafeSerializable {
 
     /**
-     * The Trademark Claims Notice ID from
-     * {@link "http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3"}.
+     * The Trademark Claims Notice ID from <a
+     * href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">the RFC</a>.
      */
-    @XmlValue
-    String tcnId;
+    @XmlValue String tcnId;
 
     /** The identifier of the TMDB provider to use, defaulting to the TMCH. */
     @IgnoreSave(IfNull.class)

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataBase.java

@@ -17,6 +17,7 @@ package google.registry.model.domain.secdns;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.MappedSuperclass;
@@ -31,7 +32,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @Embed
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public abstract class DomainDsDataBase extends ImmutableObject {
+public abstract class DomainDsDataBase extends ImmutableObject implements UnsafeSerializable {
 
   @Ignore @XmlTransient @Transient String domainRepoId;
 

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java

@@ -16,7 +16,9 @@ package google.registry.model.domain.secdns;
 
 import static google.registry.model.IdService.allocateId;
 
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.replay.SqlOnlyEntity;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -26,7 +28,8 @@ import javax.persistence.Id;
 
 /** Entity class to represent a historic {@link DelegationSignerData}. */
 @Entity
-public class DomainDsDataHistory extends DomainDsDataBase implements SqlOnlyEntity {
+public class DomainDsDataHistory extends DomainDsDataBase
+    implements SqlOnlyEntity, UnsafeSerializable {
 
   @Id Long dsDataHistoryRevisionId;
 
@@ -53,6 +56,10 @@ public class DomainDsDataHistory extends DomainDsDataBase implements SqlOnlyEnti
     return instance;
   }
 
+  public DomainHistory.DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), domainHistoryRevisionId);
+  }
+
   @Override
   @Access(AccessType.PROPERTY)
   public String getDomainRepoId() {

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -122,7 +122,6 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @Nullable @Index String domainName;
 
   /** When this token was created. */
-  @Column(nullable = false)
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /** Allowed registrar client IDs for this token, or null if all registrars are allowed. */

core/src/main/java/google/registry/model/eppcommon/Address.java

@@ -27,6 +27,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -56,7 +57,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public class Address extends ImmutableObject implements Jsonifiable {
+public class Address extends ImmutableObject implements Jsonifiable, UnsafeSerializable {
 
   /** The schema validation will enforce that this has 3 lines at most. */
   // TODO(b/177569726): Remove this field after migration. We need to figure out how to generate

core/src/main/java/google/registry/model/eppcommon/AuthInfo.java

@@ -16,6 +16,7 @@ package google.registry.model.eppcommon;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Embeddable;
 import javax.persistence.Embedded;
 import javax.persistence.MappedSuperclass;
@@ -35,7 +36,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public abstract class AuthInfo extends ImmutableObject {
+public abstract class AuthInfo extends ImmutableObject implements UnsafeSerializable {
 
   @Embedded protected PasswordAuth pw;
 
@@ -47,7 +48,7 @@ public abstract class AuthInfo extends ImmutableObject {
   @Embed
   @XmlType(namespace = "urn:ietf:params:xml:ns:eppcom-1.0")
   @Embeddable
-  public static class PasswordAuth extends ImmutableObject {
+  public static class PasswordAuth extends ImmutableObject implements UnsafeSerializable {
     @XmlValue
     @XmlJavaTypeAdapter(NormalizedStringAdapter.class)
     String value;

core/src/main/java/google/registry/model/eppcommon/PhoneNumber.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Embeddable;
 import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -49,7 +50,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public class PhoneNumber extends ImmutableObject {
+public class PhoneNumber extends ImmutableObject implements UnsafeSerializable {
 
   @XmlValue
   @XmlJavaTypeAdapter(CollapsedStringAdapter.class)

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -112,7 +112,6 @@ public enum StatusValue implements EppEnum {
    */
   PENDING_UPDATE(AllowedOn.NONE),
 
-
   /** A non-client-settable status that prevents deletes of EPP resources. */
   SERVER_DELETE_PROHIBITED(AllowedOn.ALL),
 
@@ -162,6 +161,10 @@ public enum StatusValue implements EppEnum {
     return xmlName.startsWith("client");
   }
 
+  public boolean isServerSettable() {
+    return xmlName.startsWith("server");
+  }
+
   public boolean isChargedStatus() {
     return xmlName.startsWith("server") && xmlName.endsWith("Prohibited");
   }

core/src/main/java/google/registry/model/eppcommon/Trid.java

@@ -18,6 +18,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import javax.xml.bind.annotation.XmlElement;
@@ -32,7 +33,7 @@ import javax.xml.bind.annotation.XmlType;
 @Embed
 @XmlType(propOrder = {"clientTransactionId", "serverTransactionId"})
 @javax.persistence.Embeddable
-public class Trid extends ImmutableObject {
+public class Trid extends ImmutableObject implements UnsafeSerializable {
 
   /** The server transaction id. */
   @XmlElement(name = "svTRID", namespace = "urn:ietf:params:xml:ns:epp-1.0")

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -20,11 +20,13 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.host.HostHistory.HostHistoryId;
 import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -60,7 +62,7 @@ import javax.persistence.PostLoad;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(HostHistoryId.class)
-public class HostHistory extends HistoryEntry implements SqlEntity {
+public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable {
 
   // Store HostBase instead of HostResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects
@@ -140,6 +142,8 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
   public void beforeSqlSaveOnReplay() {
     if (hostBase == null) {
       hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
+      hostBase.fullyQualifiedHostName =
+          DomainNameUtils.canonicalizeDomainName(hostBase.fullyQualifiedHostName);
     }
   }
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -22,6 +22,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 
@@ -51,6 +52,11 @@ public class HostResource extends HostBase
     return VKey.create(HostResource.class, getRepoId(), Key.create(this));
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedHostName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedHostName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -64,6 +64,7 @@ public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEn
     EppResourceIndex instance = instantiate(EppResourceIndex.class);
     instance.reference = resourceKey;
     instance.kind = resourceKey.getKind();
+    // TODO(b/207368050): figure out if this value has ever been used other than test cases
     instance.id = resourceKey.getString(); // creates a web-safe key string
     instance.bucket = bucket;
     return instance;

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -82,6 +82,7 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
       com.google.appengine.api.datastore.Entity rawEntity) {
     CommitLogMutation instance = new CommitLogMutation();
     instance.parent = checkNotNull(parent);
+    // TODO(b/207516684): figure out if this should be converted to a vkey string via stringify()
     // Creates a web-safe key string.
     instance.entityKey = KeyFactory.keyToString(rawEntity.getKey());
     instance.entityProtoBytes = convertToPb(rawEntity).toByteArray();
@@ -91,6 +92,8 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
   /** Returns the key of a mutation based on the {@code entityKey} of the entity it stores. */
   public static
       Key<CommitLogMutation> createKey(Key<CommitLogManifest> parent, Key<?> entityKey) {
+    // TODO(b/207516684): figure out if the return type needs to be VKey and
+    //  if the string used to create a key should remain the same
     return Key.create(parent, CommitLogMutation.class, entityKey.getString());
   }
 }

core/src/main/java/google/registry/model/poll/PendingActionNotificationResponse.java

@@ -17,6 +17,7 @@ package google.registry.model.poll;
 import com.google.common.annotations.VisibleForTesting;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
 import javax.persistence.Embeddable;
@@ -31,12 +32,13 @@ import org.joda.time.DateTime;
 /** The {@link ResponseData} returned when completing a pending action on a domain. */
 @XmlTransient
 @Embeddable
-public class PendingActionNotificationResponse extends ImmutableObject implements ResponseData {
+public class PendingActionNotificationResponse extends ImmutableObject
+    implements ResponseData, UnsafeSerializable {
 
   /** The inner name type that contains a name and the result boolean. */
   @Embed
   @Embeddable
-  static class NameOrId extends ImmutableObject {
+  static class NameOrId extends ImmutableObject implements UnsafeSerializable {
     @XmlValue
     String value;
 

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -33,6 +33,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
@@ -98,7 +99,7 @@ import org.joda.time.DateTime;
       @javax.persistence.Index(columnList = "eventTime")
     })
 public abstract class PollMessage extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity {
+    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity, UnsafeSerializable {
 
   /** Entity id. */
   @Id

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -72,6 +72,7 @@ import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
@@ -116,7 +117,7 @@ import org.joda.time.DateTime;
     })
 @InCrossTld
 public class Registrar extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity, Jsonifiable {
+    implements Buildable, DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable {
 
   /** Represents the type of a registrar entity. */
   public enum Type {
@@ -404,7 +405,7 @@ public class Registrar extends ImmutableObject
 
   /** A billing account entry for this registrar, consisting of a currency and an account Id. */
   @Embed
-  public static class BillingAccountEntry extends ImmutableObject {
+  public static class BillingAccountEntry extends ImmutableObject implements UnsafeSerializable {
 
     CurrencyUnit currency;
     String accountId;

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -46,6 +46,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registrar.RegistrarContact.RegistrarPocId;
@@ -82,7 +83,7 @@ import javax.persistence.Transient;
 @IdClass(RegistrarPocId.class)
 @InCrossTld
 public class RegistrarContact extends ImmutableObject
-    implements DatastoreAndSqlEntity, Jsonifiable {
+    implements DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable {
 
   @Parent @Transient Key<Registrar> parent;
 

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -148,7 +148,9 @@ public class ReplicateToDatastoreAction implements Runnable {
 
                 // Write the updated last transaction id to Datastore as part of this Datastore
                 // transaction.
-                auditedOfy().save().entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
+                auditedOfy()
+                    .saveIgnoringReadOnly()
+                    .entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
                 logger.atInfo().log(
                     "Finished applying single transaction Cloud SQL -> Cloud Datastore.");
               });
@@ -169,7 +171,7 @@ public class ReplicateToDatastoreAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire ReplicateToDatastoreAction lock, aborting.";
@@ -194,7 +196,7 @@ public class ReplicateToDatastoreAction implements Runnable {
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 

core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java

@@ -22,6 +22,8 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -45,11 +47,10 @@ import org.joda.time.DateTime;
 @Embed
 @Entity
 public class DomainTransactionRecord extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity {
+    implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable {
 
   @Id
   @Ignore
-  @ImmutableObject.DoNotCompare
   @GeneratedValue(strategy = GenerationType.IDENTITY)
   @ImmutableObject.Insignificant
   Long id;
@@ -58,6 +59,14 @@ public class DomainTransactionRecord extends ImmutableObject
   @Column(nullable = false)
   String tld;
 
+  // The following two fields are exposed in this entity to support bulk-loading in Cloud SQL by the
+  // Datastore-SQL validation. They are excluded from equality check since they are not set in
+  // Datastore.
+  // TODO(b/203609782): post migration, decide whether to keep these two fields.
+  @Ignore @ImmutableObject.Insignificant String domainRepoId;
+
+  @Ignore @ImmutableObject.Insignificant Long historyRevisionId;
+
   /**
    * The time this Transaction takes effect (counting grace periods and other nuances).
    *
@@ -174,6 +183,10 @@ public class DomainTransactionRecord extends ImmutableObject
     }
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(domainRepoId, historyRevisionId);
+  }
+
   public DateTime getReportingTime() {
     return reportingTime;
   }

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -31,6 +31,7 @@ import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactBase;
 import google.registry.model.contact.ContactHistory;
@@ -83,7 +84,8 @@ import org.joda.time.DateTime;
 @Entity
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public class HistoryEntry extends ImmutableObject implements Buildable, DatastoreEntity {
+public class HistoryEntry extends ImmutableObject
+    implements Buildable, DatastoreEntity, UnsafeSerializable {
 
   /** Represents the type of history entry. */
   public enum Type {

core/src/main/java/google/registry/model/server/Lock.java

@@ -32,6 +32,8 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.persistence.transaction.TransactionManager;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
 import java.io.Serializable;
@@ -212,6 +214,47 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
       Duration leaseLength,
       RequestStatusChecker requestStatusChecker,
       boolean checkThreadRunning) {
+    return acquireWithTransactionManager(
+        resourceName, tld, leaseLength, requestStatusChecker, checkThreadRunning, tm());
+  }
+
+  /**
+   * Try to acquire a lock in SQL. Returns absent if it can't be acquired.
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   */
+  public static Optional<Lock> acquireSql(
+      String resourceName,
+      @Nullable String tld,
+      Duration leaseLength,
+      RequestStatusChecker requestStatusChecker,
+      boolean checkThreadRunning) {
+    return acquireWithTransactionManager(
+        resourceName, tld, leaseLength, requestStatusChecker, checkThreadRunning, jpaTm());
+  }
+
+  /** Release the lock. */
+  public void release() {
+    releaseWithTransactionManager(tm());
+  }
+
+  /**
+   * Release the lock from SQL.
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   */
+  public void releaseSql() {
+    releaseWithTransactionManager(jpaTm());
+  }
+
+  /** Try to acquire a lock. Returns absent if it can't be acquired. */
+  private static Optional<Lock> acquireWithTransactionManager(
+      String resourceName,
+      @Nullable String tld,
+      Duration leaseLength,
+      RequestStatusChecker requestStatusChecker,
+      boolean checkThreadRunning,
+      TransactionManager transactionManager) {
     String scope = (tld != null) ? tld : GLOBAL;
     String lockId = makeLockId(resourceName, scope);
     // It's important to use transactNew rather than transact, because a Lock can be used to control
@@ -219,11 +262,12 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
     // must be definitively acquired before it is used, even when called inside another transaction.
     Supplier<AcquireResult> lockAcquirer =
         () -> {
-          DateTime now = tm().getTransactionTime();
+          DateTime now = transactionManager.getTransactionTime();
 
           // Checking if an unexpired lock still exists - if so, the lock can't be acquired.
           Lock lock =
-              tm().loadByKeyIfPresent(
+              transactionManager
+                  .loadByKeyIfPresent(
                       VKey.create(
                           Lock.class,
                           new LockId(resourceName, scope),
@@ -249,13 +293,15 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
               create(resourceName, scope, requestStatusChecker.getLogId(), now, leaseLength);
           // Locks are not parented under an EntityGroupRoot (so as to avoid write
           // contention) and don't need to be backed up.
-          tm().putIgnoringReadOnly(newLock);
+          transactionManager.putIgnoringReadOnly(newLock);
 
           return AcquireResult.create(now, lock, newLock, lockState);
         };
     // In ofy, backup is determined per-action, but in SQL it's determined per-transaction
     AcquireResult acquireResult =
-        tm().isOfy() ? tm().transactNew(lockAcquirer) : jpaTm().transactWithoutBackup(lockAcquirer);
+        transactionManager.isOfy()
+            ? transactionManager.transactNew(lockAcquirer)
+            : ((JpaTransactionManager) transactionManager).transactWithoutBackup(lockAcquirer);
 
     logAcquireResult(acquireResult);
     lockMetrics.recordAcquire(resourceName, scope, acquireResult.lockState());
@@ -263,7 +309,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
   }
 
   /** Release the lock. */
-  public void release() {
+  private void releaseWithTransactionManager(TransactionManager transactionManager) {
     // Just use the default clock because we aren't actually doing anything that will use the clock.
     Supplier<Void> lockReleaser =
         () -> {
@@ -274,15 +320,17 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
           VKey<Lock> key =
               VKey.create(
                   Lock.class, new LockId(resourceName, tld), Key.create(Lock.class, lockId));
-          Lock loadedLock = tm().loadByKeyIfPresent(key).orElse(null);
+          Lock loadedLock = transactionManager.loadByKeyIfPresent(key).orElse(null);
           if (Lock.this.equals(loadedLock)) {
             // Use deleteIgnoringReadOnly() so that we don't create a commit log entry for deleting
             // the lock.
             logger.atInfo().log("Deleting lock: %s", lockId);
-            tm().deleteIgnoringReadOnly(key);
+            transactionManager.deleteIgnoringReadOnly(key);
 
             lockMetrics.recordRelease(
-                resourceName, tld, new Duration(acquiredTime, tm().getTransactionTime()));
+                resourceName,
+                tld,
+                new Duration(acquiredTime, transactionManager.getTransactionTime()));
           } else {
             logger.atSevere().log(
                 "The lock we acquired was transferred to someone else before we"
@@ -294,11 +342,12 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
           }
           return null;
         };
+
     // In ofy, backup is determined per-action, but in SQL it's determined per-transaction
-    if (tm().isOfy()) {
-      tm().transact(lockReleaser);
+    if (transactionManager.isOfy()) {
+      transactionManager.transact(lockReleaser);
     } else {
-      jpaTm().transactWithoutBackup(lockReleaser);
+      ((JpaTransactionManager) transactionManager).transactWithoutBackup(lockReleaser);
     }
   }
 

core/src/main/java/google/registry/model/tld/Registry.java

@@ -52,6 +52,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
@@ -88,7 +89,8 @@ import org.joda.time.Duration;
 @Entity
 @javax.persistence.Entity(name = "Tld")
 @InCrossTld
-public class Registry extends ImmutableObject implements Buildable, DatastoreAndSqlEntity {
+public class Registry extends ImmutableObject
+    implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable {
 
   @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
@@ -308,7 +310,7 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
    * <p>There must be at least one entry in this set.
    *
    * <p>All entries of this list must be valid keys for the map of {@code DnsWriter}s injected by
-   * <code>@Inject Map<String, DnsWriter></code>
+   * {@code @Inject Map<String, DnsWriter>}
    */
   @Column(nullable = false)
   Set<String> dnsWriters;

core/src/main/java/google/registry/model/tmch/ClaimsList.java

@@ -27,6 +27,8 @@ import google.registry.model.replay.SqlOnlyEntity;
 import google.registry.model.tld.label.ReservedList.ReservedListEntry;
 import java.util.Map;
 import java.util.Optional;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
@@ -56,7 +58,11 @@ public class ClaimsList extends ImmutableObject implements SqlOnlyEntity {
   @GeneratedValue(strategy = GenerationType.IDENTITY)
   Long revisionId;
 
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "creationTimestamp", nullable = false))
+  })
   CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
 
   /**

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -16,25 +16,18 @@ package google.registry.model.tmch;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
-import com.googlecode.objectify.annotation.Entity;
-import google.registry.model.annotations.NotBackedUp;
-import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
-import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
 import javax.persistence.Column;
 import org.joda.time.DateTime;
 
-/** Datastore singleton for ICANN's TMCH CA certificate revocation list (CRL). */
-@Entity
+/** Singleton for ICANN's TMCH CA certificate revocation list (CRL). */
 @javax.persistence.Entity
 @Immutable
-@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEntity {
+public final class TmchCrl extends CrossTldSingleton implements SqlOnlyEntity {
 
   @Column(name = "certificateRevocations", nullable = false)
   String crl;
@@ -47,25 +40,23 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
 
   /** Returns the singleton instance of this entity, without memoization. */
   public static Optional<TmchCrl> get() {
-    return tm().transact(() -> tm().loadSingleton(TmchCrl.class));
+    return jpaTm().transact(() -> jpaTm().loadSingleton(TmchCrl.class));
   }
 
   /**
-   * Change the Datastore singleton to a new ASCII-armored X.509 CRL.
+   * Change the singleton to a new ASCII-armored X.509 CRL.
    *
    * <p>Please do not call this function unless your CRL is properly formatted, signed by the root,
    * and actually newer than the one currently in Datastore.
-   *
-   * <p>During the dual-write period, we write to both Datastore and SQL
    */
   public static void set(final String crl, final String url) {
-    tm().transact(
+    jpaTm()
+        .transact(
             () -> {
               TmchCrl tmchCrl = new TmchCrl();
-              tmchCrl.updated = tm().getTransactionTime();
+              tmchCrl.updated = jpaTm().getTransactionTime();
               tmchCrl.crl = checkNotNull(crl, "crl");
               tmchCrl.url = checkNotNull(url, "url");
-              ofyTm().transactNew(() -> ofyTm().putWithoutBackup(tmchCrl));
               jpaTm().transactNew(() -> jpaTm().putWithoutBackup(tmchCrl));
             });
   }
@@ -80,7 +71,7 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
     return crl;
   }
 
-  /** Time we last updated the Datastore with a newer ICANN CRL. */
+  /** Time we last updated the Database with a newer ICANN CRL. */
   public final DateTime getUpdated() {
     return updated;
   }