Test vkey behaviors when in a task queue (#1406)

* Test vkey behavior in task queue

build.gradle

@@ -200,28 +200,37 @@ rootProject.ext {
   pyver = { exe ->
     try {
       ext.execInBash(
-          exe + " -c 'import sys; print(sys.hexversion)'", "/") as Integer
+          exe + " -c 'import sys; print(sys.hexversion)' 2>/dev/null",
+          "/") as Integer
     } catch (org.gradle.process.internal.ExecException e) {
       return -1;
     }
   }
+
+  // Return the path to a usable python3 executable.
+  getPythonExecutable = {
+    // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
+    // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
+    // Debian so it seems like that should be available anywhere).
+    def MIN_PY_VER = 0x3070300
+    if (pyver('python') >= MIN_PY_VER) {
+      return 'python'
+    } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
+      return '/usr/bin/python3'
+    } else {
+      throw new GradleException("No usable Python version found (build " +
+                                "requires at least python 3.7.3)");
+    }
+  }
 }
 
 task runPresubmits(type: Exec) {
 
-  // Find a python version greater than 3.7.3 (this is somewhat arbitrary, we
-  // know we'd like at least 3.6, but 3.7.3 is the latest that ships with
-  // Debian so it seems like that should be available anywhere).
-  def MIN_PY_VER = 0x3070300
-  if (pyver('python') >= MIN_PY_VER) {
-    executable 'python'
-  } else if (pyver('/usr/bin/python3') >= MIN_PY_VER) {
-    executable '/usr/bin/python3'
-  } else {
-    throw new GradleException("No usable Python version found (build " +
-                              "requires at least python 3.7.3)");
-  }
   args('config/presubmits.py')
+
+  doFirst {
+    executable getPythonExecutable()
+  }
 }
 
 def javadocSource = []
@@ -435,9 +444,10 @@ rootProject.ext {
             ? "${rootDir}/.."
             : rootDir
     def formatDiffScript = "${scriptDir}/google-java-format-git-diff.sh"
+    def pythonExe = getPythonExecutable()
 
     return ext.execInBash(
-        "${formatDiffScript} ${action}", "${workingDir}")
+        "PYTHON=${pythonExe} ${formatDiffScript} ${action}", "${workingDir}")
   }
 }
 
@@ -445,18 +455,23 @@ rootProject.ext {
 // Note that this task checks modified Java files in the entire repository.
 task javaIncrementalFormatCheck {
   doLast {
-    def checkResult = invokeJavaDiffFormatScript("check")
-    if (checkResult == 'true') {
-      throw new IllegalStateException(
-          "Some Java files need to be reformatted. You may use the "
-              + "'javaIncrementalFormatDryRun' task to review\n "
-              + "the changes, or the 'javaIncrementalFormatApply' task "
-              + "to reformat.")
-    } else if (checkResult != 'false') {
-      throw new RuntimeException(
-          "Failed to invoke format check script:\n" + checkResult)
+    // We can only do this in a git tree.
+    if (new File("${rootDir}/.git").exists()) {
+      def checkResult = invokeJavaDiffFormatScript("check")
+      if (checkResult == 'true') {
+        throw new IllegalStateException(
+            "Some Java files need to be reformatted. You may use the "
+                + "'javaIncrementalFormatDryRun' task to review\n "
+                + "the changes, or the 'javaIncrementalFormatApply' task "
+                + "to reformat.")
+      } else if (checkResult != 'false') {
+        throw new RuntimeException(
+            "Failed to invoke format check script:\n" + checkResult)
+      }
+      println("Incremental Java format check ok.")
+    } else {
+      println("Omitting format check: not in a git directory.")
     }
-    println("Incremental Java format check ok.")
   }
 }
 
@@ -486,7 +501,8 @@ task javadoc(type: Javadoc) {
   options.addBooleanOption('Xdoclint:all,-missing', true)
   options.addBooleanOption("-allow-script-in-comments",true)
   options.tags = ["type:a:Generic Type",
-                  "error:a:Expected Error"]
+                  "error:a:Expected Error",
+                  "invariant:a:Guaranteed Property"]
 }
 
 tasks.build.dependsOn(tasks.javadoc)

config/presubmits.py

@@ -119,9 +119,10 @@ PRESUBMITS = {
       "AppEngineExtension.register(...) instead.",
 
     # PostgreSQLContainer instantiation must specify docker tag
+    # TODO(b/204572437): Fix the pattern to pass DatabaseSnapshotTest.java
     PresubmitCheck(
         r"[\s\S]*new\s+PostgreSQLContainer(<[\s\S]*>)?\(\s*\)[\s\S]*",
-        "java", {}):
+        "java", {"DatabaseSnapshotTest.java"}):
       "PostgreSQLContainer instantiation must specify docker tag.",
 
     # Various Soy linting checks
@@ -213,6 +214,7 @@ PRESUBMITS = {
         {"src/test", "ActivityReportingQueryBuilder.java",
          # This class contains helper method to make queries in Beam.
          "RegistryJpaIO.java",
+         "CreateSyntheticHistoryEntriesAction.java",
          # TODO(b/179158393): Remove everything below, which should be done
          # using Criteria
          "JpaTransactionManager.java",

core/build.gradle

@@ -705,7 +705,11 @@ createToolTask(
 
 
 createToolTask(
-        'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+    'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+
+createToolTask(
+    'createSyntheticHistoryEntries',
+    'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
 project.tasks.create('initSqlPipeline', JavaExec) {
   main = 'google.registry.beam.initsql.InitSqlPipeline'
@@ -783,27 +787,27 @@ createUberJar(
 // User should install gcloud and login to GCP before invoking this tasks.
 if (environment == 'alpha') {
   def pipelines = [
-      InitSql            :
+      initSql            :
           [
               mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
               metaData : 'google/registry/beam/init_sql_pipeline_metadata.json'
           ],
-      BulkDeleteDatastore:
+      bulkDeleteDatastore:
           [
               mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
               metaData : 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
           ],
-      Spec11             :
+      spec11             :
           [
               mainClass: 'google.registry.beam.spec11.Spec11Pipeline',
               metaData : 'google/registry/beam/spec11_pipeline_metadata.json'
           ],
-      Invoicing          :
+      invoicing          :
           [
               mainClass: 'google.registry.beam.invoicing.InvoicingPipeline',
               metaData : 'google/registry/beam/invoicing_pipeline_metadata.json'
           ],
-      Rde                :
+      rde                :
           [
               mainClass: 'google.registry.beam.rde.RdePipeline',
               metaData : 'google/registry/beam/rde_pipeline_metadata.json'
@@ -1111,6 +1115,10 @@ test {
   // TODO(weiminyu): Remove dependency on sqlIntegrationTest
 }.dependsOn(fragileTest, outcastTest, standardTest, registryToolIntegrationTest, sqlIntegrationTest)
 
+// When we override tests, we also break the cleanTest command.
+cleanTest.dependsOn(cleanFragileTest, cleanOutcastTest, cleanStandardTest,
+                    cleanRegistryToolIntegrationTest, cleanSqlIntegrationTest)
+
 project.build.dependsOn devtool
 project.build.dependsOn buildToolImage
 project.build.dependsOn ':stage'

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -55,10 +55,9 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+  static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
       InputStream inputStream) {
-    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
-        InputStream input = new BufferedInputStream(inputStream)) {
+    try (InputStream input = new BufferedInputStream(inputStream)) {
       Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
       checkState(commitLogs.hasNext());
       checkState(commitLogs.next() instanceof CommitLogCheckpoint);
@@ -105,7 +104,7 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+  static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
     return loadEntitiesByTransaction(inputStream).stream()
         .flatMap(ImmutableList::stream)
         .collect(toImmutableList());

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -112,7 +112,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire SQL commit log replay lock, aborting.";

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -105,29 +105,29 @@ public abstract class VersionedEntity implements Serializable {
    * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
    * information.
    */
-  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+  static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
     long commitTimeMillis = manifest.getCommitTime().getMillis();
     return manifest.getDeletions().stream()
         .map(com.googlecode.objectify.Key::getRaw)
-        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+        .map(key -> newBuilder().commitTimeMills(commitTimeMillis).key(key).build());
   }
 
   /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
-  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+  static VersionedEntity fromMutation(CommitLogMutation mutation) {
     return from(
         com.googlecode.objectify.Key.create(mutation).getParent().getId(),
         mutation.getEntityProtoBytes());
   }
 
   public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
-    return builder()
+    return newBuilder()
         .entityProtoBytes(entityProtoBytes)
         .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
         .commitTimeMills(commitTimeMillis)
         .build();
   }
 
-  static Builder builder() {
+  private static Builder newBuilder() {
     return new AutoValue_VersionedEntity.Builder();
   }
 
@@ -142,7 +142,7 @@ public abstract class VersionedEntity implements Serializable {
 
     public abstract VersionedEntity build();
 
-    public Builder entityProtoBytes(byte[] bytes) {
+    Builder entityProtoBytes(byte[] bytes) {
       return entityProtoBytes(new ImmutableBytes(bytes));
     }
   }

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -55,6 +55,7 @@ import org.joda.time.format.DateTimeFormatter;
     path = SendExpiringCertificateNotificationEmailAction.PATH,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public class SendExpiringCertificateNotificationEmailAction implements Runnable {
+
   public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
   /**
    * Used as an offset when storing the last notification email sent date.
@@ -96,8 +97,13 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   public void run() {
     response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
     try {
-      sendNotificationEmails();
+      int numEmailsSent = sendNotificationEmails();
+      String message =
+          String.format(
+              "Done. Sent %d expiring certificate notification emails in total.", numEmailsSent);
+      logger.atInfo().log(message);
       response.setStatus(SC_OK);
+      response.setPayload(message);
     } catch (Exception e) {
       logger.atWarning().withCause(e).log(
           "Exception thrown when sending expiring certificate notification emails.");
@@ -107,9 +113,11 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   }
 
   /**
-   * Returns a list of registrars that should receive expiring notification emails. There are two
-   * certificates that should be considered (the main certificate and failOver certificate). The
-   * registrars should receive notifications if one of the certificate checks returns true.
+   * Returns a list of registrars that should receive expiring notification emails.
+   *
+   * <p>There are two certificates that should be considered (the main certificate and failOver
+   * certificate). The registrars should receive notifications if one of the certificate checks
+   * returns true.
    */
   @VisibleForTesting
   ImmutableList<RegistrarInfo> getRegistrarsWithExpiringCertificates() {
@@ -151,15 +159,17 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
     }
     try {
       ImmutableSet<InternetAddress> recipients = getEmailAddresses(registrar, Type.TECH);
+      ImmutableSet<InternetAddress> ccs = getEmailAddresses(registrar, Type.ADMIN);
       Date expirationDate = certificateChecker.getCertificate(certificate.get()).getNotAfter();
       logger.atInfo().log(
-          "Registrar %s should receive an email that its %s SSL certificate will expire on %s.",
-          registrar.getRegistrarName(),
+          " %s SSL certificate of registrar '%s' will expire on %s.",
           certificateType.getDisplayName(),
+          registrar.getRegistrarName(),
           expirationDate.toString());
-      if (recipients.isEmpty()) {
+      if (recipients.isEmpty() && ccs.isEmpty()) {
         logger.atWarning().log(
-            "Registrar %s contains no email addresses to receive notification email.",
+            "Registrar %s contains no TECH nor ADMIN email addresses to receive notification"
+                + " email.",
             registrar.getRegistrarName());
         return false;
       }
@@ -174,7 +184,7 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
                       expirationDate,
                       registrar.getRegistrarId()))
               .setRecipients(recipients)
-              .setCcs(getEmailAddresses(registrar, Type.ADMIN))
+              .setCcs(ccs)
               .build());
       /*
        * A duration time offset is used here to ensure that date comparison between two
@@ -243,32 +253,32 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   /** Sends notification emails to registrars with expiring certificates. */
   @VisibleForTesting
   int sendNotificationEmails() {
-    int emailsSent = 0;
+    int numEmailsSent = 0;
     for (RegistrarInfo registrarInfo : getRegistrarsWithExpiringCertificates()) {
       Registrar registrar = registrarInfo.registrar();
-      if (registrarInfo.isCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringCertNotificationSentDate(),
-            CertificateType.PRIMARY,
-            registrar.getClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringCertNotificationSentDate(),
+              CertificateType.PRIMARY,
+              registrar.getClientCertificate())) {
+        numEmailsSent++;
       }
-      if (registrarInfo.isFailOverCertExpiring()) {
-        sendNotificationEmail(
-            registrar,
-            registrar.getLastExpiringFailoverCertNotificationSentDate(),
-            CertificateType.FAILOVER,
-            registrar.getFailoverClientCertificate());
-        emailsSent++;
+      if (registrarInfo.isFailOverCertExpiring()
+          && sendNotificationEmail(
+              registrar,
+              registrar.getLastExpiringFailoverCertNotificationSentDate(),
+              CertificateType.FAILOVER,
+              registrar.getFailoverClientCertificate())) {
+        numEmailsSent++;
       }
     }
-    logger.atInfo().log(
-        "Attempted to send %d expiring certificate notification emails.", emailsSent);
-    return emailsSent;
+    return numEmailsSent;
   }
 
-  /** Returns a list of email addresses of the registrar that should receive a notification email */
+  /**
+   * Returns a list of email addresses of the registrar that should receive a notification email.
+   */
   @VisibleForTesting
   ImmutableSet<InternetAddress> getEmailAddresses(Registrar registrar, Type contactType) {
     ImmutableSortedSet<RegistrarContact> contacts = registrar.getContactsOfType(contactType);
@@ -327,6 +337,7 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
 
   @AutoValue
   public abstract static class RegistrarInfo {
+
     static RegistrarInfo create(
         Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
       return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(

core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java

@@ -0,0 +1,138 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.net.MediaType;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.contact.ContactHistory;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * An action that wipes out Personal Identifiable Information (PII) fields of {@link ContactHistory}
+ * entities.
+ *
+ * <p>ContactHistory entities should be retained in the database for only certain amount of time.
+ * This periodic wipe out action only applies to SQL.
+ */
+@Action(
+    service = Service.BACKEND,
+    path = WipeOutContactHistoryPiiAction.PATH,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeOutContactHistoryPiiAction implements Runnable {
+
+  public static final String PATH = "/_dr/task/wipeOutContactHistoryPii";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private final Clock clock;
+  private final Response response;
+  private final int minMonthsBeforeWipeOut;
+  private final int wipeOutQueryBatchSize;
+
+  @Inject
+  public WipeOutContactHistoryPiiAction(
+      Clock clock,
+      @Config("minMonthsBeforeWipeOut") int minMonthsBeforeWipeOut,
+      @Config("wipeOutQueryBatchSize") int wipeOutQueryBatchSize,
+      Response response) {
+    this.clock = clock;
+    this.response = response;
+    this.minMonthsBeforeWipeOut = minMonthsBeforeWipeOut;
+    this.wipeOutQueryBatchSize = wipeOutQueryBatchSize;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
+    try {
+      int totalNumOfWipedEntities = 0;
+      DateTime wipeOutTime = clock.nowUtc().minusMonths(minMonthsBeforeWipeOut);
+      logger.atInfo().log(
+          "About to wipe out all PII of contact history entities prior to %s.", wipeOutTime);
+
+      int numOfWipedEntities = 0;
+      do {
+        numOfWipedEntities =
+            jpaTm()
+                .transact(
+                    () ->
+                        wipeOutContactHistoryData(
+                            getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime)));
+        totalNumOfWipedEntities += numOfWipedEntities;
+      } while (numOfWipedEntities > 0);
+      String msg =
+          String.format(
+              "Done. Wiped out PII of %d ContactHistory entities in total.",
+              totalNumOfWipedEntities);
+      logger.atInfo().log(msg);
+      response.setPayload(msg);
+      response.setStatus(SC_OK);
+
+    } catch (Exception e) {
+      logger.atSevere().withCause(e).log(
+          "Exception thrown during the process of wiping out contact history PII.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(
+          String.format(
+              "Exception thrown during the process of wiping out contact history PII with cause"
+                  + ": %s",
+              e));
+    }
+  }
+
+  /**
+   * Returns a stream of up to {@link #wipeOutQueryBatchSize} {@link ContactHistory} entities
+   * containing PII that are prior to @param wipeOutTime.
+   */
+  @VisibleForTesting
+  Stream<ContactHistory> getNextContactHistoryEntitiesWithPiiBatch(DateTime wipeOutTime) {
+    // email is one of the required fields in EPP, meaning it's initially not null.
+    // Therefore, checking if it's null is one way to avoid processing contact history entities
+    // that have been processed previously. Refer to RFC 5733 for more information.
+    return jpaTm()
+        .query(
+            "FROM ContactHistory WHERE modificationTime < :wipeOutTime " + "AND email IS NOT NULL",
+            ContactHistory.class)
+        .setParameter("wipeOutTime", wipeOutTime)
+        .setMaxResults(wipeOutQueryBatchSize)
+        .getResultStream();
+  }
+
+  /** Wipes out the PII of each of the {@link ContactHistory} entities in the stream. */
+  @VisibleForTesting
+  int wipeOutContactHistoryData(Stream<ContactHistory> contactHistoryEntities) {
+    AtomicInteger numOfEntities = new AtomicInteger(0);
+    contactHistoryEntities.forEach(
+        contactHistoryEntity -> {
+          jpaTm().update(contactHistoryEntity.asBuilder().wipeOutPii().build());
+          numOfEntities.incrementAndGet();
+        });
+    logger.atInfo().log(
+        "Wiped out all PII fields of %d ContactHistory entities.", numOfEntities.get());
+    return numOfEntities.get();
+  }
+}

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -92,7 +92,12 @@ public class WipeoutDatastoreAction implements Runnable {
               .setJobName(createJobName("bulk-delete-datastore-", clock))
               .setContainerSpecGcsPath(
                   String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+              .setParameters(
+                  ImmutableMap.of(
+                      "kindsToDelete",
+                      "*",
+                      "registryEnvironment",
+                      RegistryEnvironment.get().name()));
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/java/google/registry/beam/common/DatabaseSnapshot.java

@@ -0,0 +1,88 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.common;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.flogger.FluentLogger;
+import java.util.List;
+import javax.persistence.EntityManager;
+import javax.persistence.EntityTransaction;
+
+/**
+ * A database snapshot shareable by concurrent queries from multiple database clients. A snapshot is
+ * uniquely identified by its {@link #getSnapshotId snapshotId}, and must stay open until all
+ * concurrent queries to this snapshot have attached to it by calling {@link
+ * google.registry.persistence.transaction.JpaTransactionManager#setDatabaseSnapshot}. However, it
+ * can be closed before those queries complete.
+ *
+ * <p>This feature is <em>Postgresql-only</em>.
+ *
+ * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to avoid
+ * exhausting predicate locks at the SERIALIZABLE level.
+ */
+// TODO(b/193662898): vendor-independent support for richer transaction semantics.
+public class DatabaseSnapshot implements AutoCloseable {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private String snapshotId;
+  private EntityManager entityManager;
+  private EntityTransaction transaction;
+
+  private DatabaseSnapshot() {}
+
+  public String getSnapshotId() {
+    checkState(entityManager != null, "Snapshot not opened yet.");
+    checkState(entityManager.isOpen(), "Snapshot already closed.");
+    return snapshotId;
+  }
+
+  private DatabaseSnapshot open() {
+    entityManager = jpaTm().getStandaloneEntityManager();
+    transaction = entityManager.getTransaction();
+    transaction.setRollbackOnly();
+    transaction.begin();
+
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+
+    List<?> snapshotIds =
+        entityManager.createNativeQuery("SELECT pg_export_snapshot();").getResultList();
+    checkState(snapshotIds.size() == 1, "Unexpected number of snapshots: %s", snapshotIds.size());
+    snapshotId = (String) snapshotIds.get(0);
+    return this;
+  }
+
+  @Override
+  public void close() {
+    if (transaction != null && transaction.isActive()) {
+      try {
+        transaction.rollback();
+      } catch (Exception e) {
+        logger.atWarning().withCause(e).log("Failed to close a Database Snapshot");
+      }
+    }
+    if (entityManager != null && entityManager.isOpen()) {
+      entityManager.close();
+    }
+  }
+
+  public static DatabaseSnapshot createSnapshot() {
+    return new DatabaseSnapshot().open();
+  }
+}

core/src/main/java/google/registry/beam/common/JpaDemoPipeline.java

@@ -17,7 +17,6 @@ package google.registry.beam.common;
 import static com.google.common.base.Verify.verify;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.model.contact.ContactResource;
 import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -59,18 +58,16 @@ public class JpaDemoPipeline implements Serializable {
                   public void processElement() {
                     // AppEngineEnvironment is needed as long as JPA entity classes still depends
                     // on Objectify.
-                    try (AppEngineEnvironment allowOfyEntity = new AppEngineEnvironment()) {
-                      int result =
-                          (Integer)
-                              jpaTm()
-                                  .transact(
-                                      () ->
-                                          jpaTm()
-                                              .getEntityManager()
-                                              .createNativeQuery("select 1;")
-                                              .getSingleResult());
-                      verify(result == 1, "Expecting 1, got %s.", result);
-                    }
+                    int result =
+                        (Integer)
+                            jpaTm()
+                                .transact(
+                                    () ->
+                                        jpaTm()
+                                            .getEntityManager()
+                                            .createNativeQuery("select 1;")
+                                            .getSingleResult());
+                    verify(result == 1, "Expecting 1, got %s.", result);
                     counter.inc();
                   }
                 }));

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -20,11 +20,9 @@ import static org.apache.beam.sdk.values.TypeDescriptors.integers;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
-import google.registry.model.ofy.ObjectifyService;
 import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -140,6 +138,9 @@ public final class RegistryJpaIO {
 
     abstract Coder<T> coder();
 
+    @Nullable
+    abstract String snapshotId();
+
     abstract Builder<R, T> toBuilder();
 
     @Override
@@ -147,7 +148,9 @@ public final class RegistryJpaIO {
     public PCollection<T> expand(PBegin input) {
       return input
           .apply("Starting " + name(), Create.of((Void) null))
-          .apply("Run query for " + name(), ParDo.of(new QueryRunner<>(query(), resultMapper())))
+          .apply(
+              "Run query for " + name(),
+              ParDo.of(new QueryRunner<>(query(), resultMapper(), snapshotId())))
           .setCoder(coder())
           .apply("Reshuffle", Reshuffle.viaRandomKey());
     }
@@ -164,6 +167,18 @@ public final class RegistryJpaIO {
       return toBuilder().coder(coder).build();
     }
 
+    /**
+     * Specifies the database snapshot to use for this query.
+     *
+     * <p>This feature is <em>Postgresql-only</em>. User is responsible for keeping the snapshot
+     * available until all JVM workers have started using it by calling {@link
+     * JpaTransactionManager#setDatabaseSnapshot}.
+     */
+    // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+    public Read<R, T> withSnapshot(String snapshotId) {
+      return toBuilder().snapshotId(snapshotId).build();
+    }
+
     static <R, T> Builder<R, T> builder() {
       return new AutoValue_RegistryJpaIO_Read.Builder<R, T>()
           .name(DEFAULT_NAME)
@@ -181,6 +196,8 @@ public final class RegistryJpaIO {
 
       abstract Builder<R, T> coder(Coder coder);
 
+      abstract Builder<R, T> snapshotId(@Nullable String sharedSnapshotId);
+
       abstract Read<R, T> build();
 
       Builder<R, T> criteriaQuery(CriteriaQuerySupplier<R> criteriaQuery) {
@@ -203,22 +220,28 @@ public final class RegistryJpaIO {
     static class QueryRunner<R, T> extends DoFn<Void, T> {
       private final RegistryQuery<R> query;
       private final SerializableFunction<R, T> resultMapper;
+      // java.util.Optional is not serializable. Use of Guava Optional is discouraged.
+      @Nullable private final String snapshotId;
 
-      QueryRunner(RegistryQuery<R> query, SerializableFunction<R, T> resultMapper) {
+      QueryRunner(
+          RegistryQuery<R> query,
+          SerializableFunction<R, T> resultMapper,
+          @Nullable String snapshotId) {
         this.query = query;
         this.resultMapper = resultMapper;
+        this.snapshotId = snapshotId;
       }
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
-        // AppEngineEnvironment is need for handling VKeys, which involve Ofy keys. Unlike
-        // SqlBatchWriter, it is unnecessary to initialize ObjectifyService in this class.
-        try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-          // TODO(b/187210388): JpaTransactionManager should support non-transactional query.
-          jpaTm()
-              .transactNoRetry(
-                  () -> query.stream().map(resultMapper::apply).forEach(outputReceiver::output));
-        }
+        jpaTm()
+            .transactNoRetry(
+                () -> {
+                  if (snapshotId != null) {
+                    jpaTm().setDatabaseSnapshot(snapshotId);
+                  }
+                  query.stream().map(resultMapper::apply).forEach(outputReceiver::output);
+                });
       }
     }
   }
@@ -364,16 +387,6 @@ public final class RegistryJpaIO {
       this.withAutoTimestamp = withAutoTimestamp;
     }
 
-    @Setup
-    public void setup() {
-      // AppEngineEnvironment is needed as long as Objectify keys are still involved in the handling
-      // of SQL entities (e.g., in VKeys). ObjectifyService needs to be initialized when conversion
-      // between Ofy entity and Datastore entity is needed.
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ObjectifyService.initOfy();
-      }
-    }
-
     @ProcessElement
     public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
       if (withAutoTimestamp) {
@@ -386,19 +399,17 @@ public final class RegistryJpaIO {
     }
 
     private void actuallyProcessElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
-      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-        ImmutableList<Object> entities =
-            Streams.stream(kv.getValue())
-                .map(this.jpaConverter::apply)
-                // TODO(b/177340730): post migration delete the line below.
-                .filter(Objects::nonNull)
-                .collect(ImmutableList.toImmutableList());
-        try {
-          jpaTm().transact(() -> jpaTm().putAll(entities));
-          counter.inc(entities.size());
-        } catch (RuntimeException e) {
-          processSingly(entities);
-        }
+      ImmutableList<Object> entities =
+          Streams.stream(kv.getValue())
+              .map(this.jpaConverter::apply)
+              // TODO(b/177340730): post migration delete the line below.
+              .filter(Objects::nonNull)
+              .collect(ImmutableList.toImmutableList());
+      try {
+        jpaTm().transact(() -> jpaTm().putAll(entities));
+        counter.inc(entities.size());
+      } catch (RuntimeException e) {
+        processSingly(entities);
       }
     }
 

core/src/main/java/google/registry/beam/common/RegistryPipelineComponent.java

@@ -21,6 +21,7 @@ import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.persistence.PersistenceModule;
+import google.registry.persistence.PersistenceModule.BeamBulkQueryJpaTm;
 import google.registry.persistence.PersistenceModule.BeamJpaTm;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -45,9 +46,19 @@ public interface RegistryPipelineComponent {
   @Config("projectId")
   String getProjectId();
 
+  /** Returns the regular {@link JpaTransactionManager} for general use. */
   @BeamJpaTm
   Lazy<JpaTransactionManager> getJpaTransactionManager();
 
+  /**
+   * Returns a {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities
+   * ({@link google.registry.model.domain.DomainBase} and {@link
+   * google.registry.model.domain.DomainHistory}). Please refer to {@link
+   * google.registry.model.bulkquery.BulkQueryEntities} for more information.
+   */
+  @BeamBulkQueryJpaTm
+  Lazy<JpaTransactionManager> getBulkQueryJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
 

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -16,6 +16,7 @@ package google.registry.beam.common;
 
 import google.registry.beam.common.RegistryJpaIO.Write;
 import google.registry.config.RegistryEnvironment;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import java.util.Objects;
 import javax.annotation.Nullable;
@@ -34,7 +35,6 @@ import org.apache.beam.sdk.options.Description;
 public interface RegistryPipelineOptions extends GcpOptions {
 
   @Description("The Registry environment.")
-  @Nullable
   RegistryEnvironment getRegistryEnvironment();
 
   void setRegistryEnvironment(RegistryEnvironment environment);
@@ -45,6 +45,12 @@ public interface RegistryPipelineOptions extends GcpOptions {
 
   void setIsolationOverride(TransactionIsolationLevel isolationOverride);
 
+  @Description("The JPA Transaction Manager to use.")
+  @Default.Enum(value = "REGULAR")
+  JpaTransactionManagerType getJpaTransactionManagerType();
+
+  void setJpaTransactionManagerType(JpaTransactionManagerType jpaTransactionManagerType);
+
   @Description("The number of entities to write to the SQL database in one operation.")
   @Default.Integer(20)
   int getSqlWriteBatchSize();

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -20,6 +20,8 @@ import com.google.auto.service.AutoService;
 import com.google.common.flogger.FluentLogger;
 import dagger.Lazy;
 import google.registry.config.RegistryEnvironment;
+import google.registry.config.SystemPropertySetter;
+import google.registry.model.AppEngineEnvironment;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import org.apache.beam.sdk.harness.JvmInitializer;
@@ -35,18 +37,35 @@ import org.apache.beam.sdk.options.PipelineOptions;
 @AutoService(JvmInitializer.class)
 public class RegistryPipelineWorkerInitializer implements JvmInitializer {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  public static final String PROPERTY = "google.registry.beam";
 
   @Override
   public void beforeProcessing(PipelineOptions options) {
     RegistryPipelineOptions registryOptions = options.as(RegistryPipelineOptions.class);
     RegistryEnvironment environment = registryOptions.getRegistryEnvironment();
     if (environment == null || environment.equals(RegistryEnvironment.UNITTEST)) {
-      return;
+      throw new RuntimeException(
+          "A registry environment must be specified in the pipeline options.");
     }
     logger.atInfo().log("Setting up RegistryEnvironment %s.", environment);
     environment.setup();
-    Lazy<JpaTransactionManager> transactionManagerLazy =
-        toRegistryPipelineComponent(registryOptions).getJpaTransactionManager();
+    RegistryPipelineComponent registryPipelineComponent =
+        toRegistryPipelineComponent(registryOptions);
+    Lazy<JpaTransactionManager> transactionManagerLazy;
+    switch (registryOptions.getJpaTransactionManagerType()) {
+      case BULK_QUERY:
+        transactionManagerLazy = registryPipelineComponent.getBulkQueryJpaTransactionManager();
+        break;
+      case REGULAR:
+      default:
+        transactionManagerLazy = registryPipelineComponent.getJpaTransactionManager();
+    }
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
+    // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
+    // loads all ofy entities.
+    new AppEngineEnvironment("Beam").setEnvironmentForAllThreads();
+    // Set the system property so that we can call IdService.allocateId() without access to
+    // datastore.
+    SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");
   }
 }

core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java

@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.datastore.v1.Entity;
+import google.registry.config.RegistryEnvironment;
 import java.util.Iterator;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
@@ -308,6 +309,11 @@ public class BulkDeleteDatastorePipeline {
 
   public interface BulkDeletePipelineOptions extends GcpOptions {
 
+    @Description("The Registry environment.")
+    RegistryEnvironment getRegistryEnvironment();
+
+    void setRegistryEnvironment(RegistryEnvironment environment);
+
     @Description(
         "The Datastore KINDs to be deleted. The format may be:\n"
             + "\t- The list of kinds to be deleted as a comma-separated string, or\n"

core/src/main/java/google/registry/beam/initsql/InitSqlPipeline.java

@@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
-import google.registry.backup.AppEngineEnvironment;
 import google.registry.backup.VersionedEntity;
 import google.registry.beam.common.RegistryJpaIO;
 import google.registry.beam.initsql.Transforms.RemoveDomainBaseForeignKeys;
@@ -230,9 +229,7 @@ public class InitSqlPipeline implements Serializable {
   }
 
   private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
-    try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-      return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
-    }
+    return entityClasses.stream().map(Key::getKind).collect(ImmutableList.toImmutableList());
   }
 
   public static void main(String[] args) {

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -22,6 +22,7 @@ import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.util.Comparator.comparing;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -261,8 +262,8 @@ public final class Transforms {
 
   // Production data repair configs go below. See b/185954992.
 
-  // Prober domains in bad state, without associated contacts, hosts, billings, and history.
-  // They can be safely ignored.
+  // Prober domains in bad state, without associated contacts, hosts, billings, and non-synthesized
+  // history. They can be safely ignored.
   private static final ImmutableSet<String> IGNORED_DOMAINS =
       ImmutableSet.of("6AF6D2-IQCANT", "2-IQANYT");
 
@@ -277,7 +278,7 @@ public final class Transforms {
 
   // Prober contacts referencing phantom registrars. They and their associated history entries can
   // be safely ignored.
-  private static final ImmutableSet IGNORED_CONTACTS =
+  private static final ImmutableSet<String> IGNORED_CONTACTS =
       ImmutableSet.of(
           "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE");
 
@@ -298,7 +299,7 @@ public final class Transforms {
       return !IGNORED_HOSTS.contains(roid);
     }
     if (entity.getKind().equals("HistoryEntry")) {
-      // Remove production bad data: History of the contacts to be ignored:
+      // Remove production bad data: Histories of ignored EPP resources:
       com.google.appengine.api.datastore.Key parentKey = entity.getKey().getParent();
       if (parentKey.getKind().equals("ContactResource")) {
         String contactRoid = parentKey.getName();
@@ -308,6 +309,10 @@ public final class Transforms {
         String hostRoid = parentKey.getName();
         return !IGNORED_HOSTS.contains(hostRoid);
       }
+      if (parentKey.getKind().equals("DomainBase")) {
+        String domainRoid = parentKey.getName();
+        return !IGNORED_DOMAINS.contains(domainRoid);
+      }
     }
     // End of production-specific checks.
 
@@ -320,7 +325,8 @@ public final class Transforms {
     return true;
   }
 
-  private static Entity repairBadData(Entity entity) {
+  @VisibleForTesting
+  static Entity repairBadData(Entity entity) {
     if (entity.getKind().equals("Cancellation")
         && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) {
       // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the
@@ -328,6 +334,15 @@ public final class Transforms {
       // instead of append. See b/185954992.
       entity.setUnindexedProperty("reason", Reason.RENEW.name());
       entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name()));
+    } else if (entity.getKind().equals("DomainBase")) {
+      // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this.
+      entity.setIndexedProperty(
+          "fullyQualifiedDomainName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedDomainName")));
+    } else if (entity.getKind().equals("HostResource")) {
+      entity.setIndexedProperty(
+          "fullyQualifiedHostName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedHostName")));
     }
     return entity;
   }
@@ -365,7 +380,8 @@ public final class Transforms {
    * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
    * given {@link Iterable}.
    */
-  static PTransform<PBegin, PCollection<String>> toStringPCollection(Iterable<String> strings) {
+  private static PTransform<PBegin, PCollection<String>> toStringPCollection(
+      Iterable<String> strings) {
     return Create.of(strings).withCoder(StringUtf8Coder.of());
   }
 
@@ -373,7 +389,7 @@ public final class Transforms {
    * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
    * caller-provided {@code transformer}.
    */
-  static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+  private static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
       DoFn<ReadableFile, VersionedEntity> transformer) {
     return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
       @Override
@@ -389,7 +405,7 @@ public final class Transforms {
     private final DateTime fromTime;
     private final DateTime toTime;
 
-    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+    FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
       checkNotNull(fromTime, "fromTime");
       checkNotNull(toTime, "toTime");
       checkArgument(

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -19,10 +19,13 @@ import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static google.registry.rde.RdeModule.BRDA_QUEUE;
+import static google.registry.rde.RdeModule.RDE_UPLOAD_QUEUE;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.auto.value.AutoValue;
 import com.google.cloud.storage.BlobId;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.PgpHelper;
@@ -31,14 +34,20 @@ import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
 import google.registry.model.tld.Registry;
+import google.registry.rde.BrdaCopyAction;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.Ghostryde;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.RdeCounter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.rde.RdeModule;
 import google.registry.rde.RdeResourceType;
+import google.registry.rde.RdeUploadAction;
 import google.registry.rde.RdeUtil;
+import google.registry.request.Action.Service;
+import google.registry.request.RequestParameters;
 import google.registry.tldconfig.idn.IdnTableEnum;
+import google.registry.util.CloudTasksUtils;
 import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
 import google.registry.xml.ValidationMode;
@@ -68,6 +77,8 @@ public class RdeIO {
 
     abstract GcsUtils gcsUtils();
 
+    abstract CloudTasksUtils cloudTasksUtils();
+
     abstract String rdeBucket();
 
     // It's OK to return a primitive array because we are only using it to construct the
@@ -83,7 +94,9 @@ public class RdeIO {
 
     @AutoValue.Builder
     abstract static class Builder {
-      abstract Builder setGcsUtils(GcsUtils gcsUtils);
+      abstract Builder setGcsUtils(GcsUtils value);
+
+      abstract Builder setCloudTasksUtils(CloudTasksUtils value);
 
       abstract Builder setRdeBucket(String value);
 
@@ -100,7 +113,8 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()));
+          .apply("Update cursors", ParDo.of(new CursorUpdater()))
+          .apply("Enqueue upload action", ParDo.of(new UploadEnqueuer(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -236,11 +250,12 @@ public class RdeIO {
     }
   }
 
-  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, PendingDeposit> {
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
     @ProcessElement
-    public void processElement(@Element KV<PendingDeposit, Integer> input) {
+    public void processElement(
+        @Element KV<PendingDeposit, Integer> input, OutputReceiver<PendingDeposit> outputReceiver) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -268,6 +283,45 @@ public class RdeIO {
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
               });
+      outputReceiver.output(input.getKey());
+    }
+  }
+
+  private static class UploadEnqueuer extends DoFn<PendingDeposit, Void> {
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private UploadEnqueuer(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
+
+    @ProcessElement
+    public void processElement(@Element PendingDeposit input, PipelineOptions options) {
+      if (input.mode() == RdeMode.FULL) {
+        cloudTasksUtils.enqueue(
+            RDE_UPLOAD_QUEUE,
+            CloudTasksUtils.createPostTask(
+                RdeUploadAction.PATH,
+                Service.BACKEND.getServiceId(),
+                ImmutableMultimap.of(
+                    RequestParameters.PARAM_TLD,
+                    input.tld(),
+                    RdeModule.PARAM_PREFIX,
+                    options.getJobName() + '/')));
+      } else {
+        cloudTasksUtils.enqueue(
+            BRDA_QUEUE,
+            CloudTasksUtils.createPostTask(
+                BrdaCopyAction.PATH,
+                Service.BACKEND.getServiceId(),
+                ImmutableMultimap.of(
+                    RequestParameters.PARAM_TLD,
+                    input.tld(),
+                    RdeModule.PARAM_WATERMARK,
+                    input.watermark().toString(),
+                    RdeModule.PARAM_PREFIX,
+                    options.getJobName() + '/')));
+      }
     }
   }
 }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -27,6 +27,7 @@ import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
@@ -44,6 +45,8 @@ import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
 import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
+import google.registry.util.CloudTasksUtils;
+import google.registry.util.UtilsModule;
 import google.registry.xml.ValidationMode;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -66,7 +69,6 @@ import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
-import org.apache.beam.sdk.transforms.Reshuffle;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
@@ -93,6 +95,7 @@ public class RdePipeline implements Serializable {
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
+  private final CloudTasksUtils cloudTasksUtils;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
@@ -111,13 +114,14 @@ public class RdePipeline implements Serializable {
   }
 
   @Inject
-  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils) {
+  RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
     this.pendings = decodePendings(options.getPendings());
-    this.rdeBucket = options.getGcsBucket();
+    this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
   }
 
   PipelineResult run() {
@@ -140,10 +144,11 @@ public class RdePipeline implements Serializable {
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
     input.apply(
-        "Write to GCS and update cursors",
+        "Write to GCS, update cursors, and enqueue upload tasks",
         RdeIO.Write.builder()
             .setRdeBucket(rdeBucket)
             .setGcsUtils(gcsUtils)
+            .setCloudTasksUtils(cloudTasksUtils)
             .setValidationMode(mode)
             .setStagingKeyBytes(stagingKeyBytes)
             .build());
@@ -177,18 +182,13 @@ public class RdePipeline implements Serializable {
                     }));
   }
 
-  @SuppressWarnings("deprecation") // Reshuffle is still recommended by Dataflow.
   <T extends EppResource>
       PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
           Pipeline pipeline, Class<T> clazz) {
     return createInputs(pipeline, clazz)
         .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply(
-            "Reshuffle KV<PendingDeposit, DepositFragment> of "
-                + clazz.getSimpleName()
-                + " to prevent fusion",
-            Reshuffle.of());
+        .setCoder(
+            KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)));
   }
 
   <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
@@ -202,7 +202,7 @@ public class RdePipeline implements Serializable {
             String.class,
             // TODO: consider adding coders for entities and pass them directly instead of using
             // VKeys.
-            x -> VKey.create(clazz, x)));
+            x -> VKey.createSql(clazz, x)));
   }
 
   <T extends EppResource>
@@ -270,7 +270,7 @@ public class RdePipeline implements Serializable {
    * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
    * worker by the pipeline launcher as a pipeline option.
    */
-  static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
@@ -282,13 +282,24 @@ public class RdePipeline implements Serializable {
 
   public static void main(String[] args) throws IOException, ClassNotFoundException {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
-    RdePipelineOptions options = PipelineOptionsFactory.fromArgs(args).as(RdePipelineOptions.class);
+    RdePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
+    // RegistryPipelineWorkerInitializer only initializes before pipeline executions, after the
+    // main() function constructed the graph. We need the registry environment set up so that we
+    // can create a CloudTasksUtils which uses the environment-dependent config file.
+    options.getRegistryEnvironment().setup();
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
   @Singleton
-  @Component(modules = {CredentialModule.class, ConfigModule.class})
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
   interface RdePipelineComponent {
     RdePipeline rdePipeline();
 

core/src/main/java/google/registry/beam/rde/RdePipelineOptions.java

@@ -31,9 +31,9 @@ public interface RdePipelineOptions extends RegistryPipelineOptions {
   void setValidationMode(String value);
 
   @Description("The GCS bucket where the encrypted RDE deposits will be uploaded to.")
-  String getGcsBucket();
+  String getRdeStagingBucket();
 
-  void setGcsBucket(String value);
+  void setRdeStagingBucket(String value);
 
   @Description("The Base64-encoded PGP public key to encrypt the deposits.")
   String getStagingKey();

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -16,6 +16,7 @@ package google.registry.beam.spec11;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
@@ -30,6 +31,7 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.reporting.Spec11ThreatMatch;
 import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
 import google.registry.util.Retrier;
 import google.registry.util.SqlTemplate;
 import google.registry.util.UtilsModule;
@@ -41,6 +43,7 @@ import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.io.TextIO;
 import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -113,15 +116,43 @@ public class Spec11Pipeline implements Serializable {
   }
 
   static PCollection<DomainNameInfo> readFromCloudSql(Pipeline pipeline) {
-    Read<Object[], DomainNameInfo> read =
+    Read<Object[], KV<String, String>> read =
         RegistryJpaIO.read(
-            "select d, r.emailAddress from Domain d join Registrar r on"
-                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
-                + " and d.deletionTime > now()",
+            "select d.repoId, r.emailAddress from Domain d join Registrar r on"
+                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL' and"
+                + " d.deletionTime > now()",
             false,
             Spec11Pipeline::parseRow);
 
-    return pipeline.apply("Read active domains from Cloud SQL", read);
+    return pipeline
+        .apply("Read active domains from Cloud SQL", read)
+        .apply(
+            "Build DomainNameInfo",
+            ParDo.of(
+                new DoFn<KV<String, String>, DomainNameInfo>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, String> input, OutputReceiver<DomainNameInfo> output) {
+                    DomainBase domainBase =
+                        jpaTm()
+                            .transact(
+                                () ->
+                                    jpaTm()
+                                        .loadByKey(
+                                            VKey.createSql(DomainBase.class, input.getKey())));
+                    String emailAddress = input.getValue();
+                    if (emailAddress == null) {
+                      emailAddress = "";
+                    }
+                    DomainNameInfo domainNameInfo =
+                        DomainNameInfo.create(
+                            domainBase.getDomainName(),
+                            domainBase.getRepoId(),
+                            domainBase.getCurrentSponsorRegistrarId(),
+                            emailAddress);
+                    output.output(domainNameInfo);
+                  }
+                }));
   }
 
   static PCollection<DomainNameInfo> readFromBigQuery(
@@ -142,17 +173,8 @@ public class Spec11Pipeline implements Serializable {
             .withTemplateCompatibility());
   }
 
-  private static DomainNameInfo parseRow(Object[] row) {
-    DomainBase domainBase = (DomainBase) row[0];
-    String emailAddress = (String) row[1];
-    if (emailAddress == null) {
-      emailAddress = "";
-    }
-    return DomainNameInfo.create(
-        domainBase.getDomainName(),
-        domainBase.getRepoId(),
-        domainBase.getCurrentSponsorRegistrarId(),
-        emailAddress);
+  private static KV<String, String> parseRow(Object[] row) {
+    return KV.of((String) row[0], (String) row[1]);
   }
 
   static void saveToSql(

core/src/main/java/google/registry/config/CloudTasksUtilsModule.java

@@ -22,10 +22,13 @@ import dagger.Provides;
 import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.util.CloudTasksUtils;
+import google.registry.util.CloudTasksUtils.GcpCloudTasksClient;
+import google.registry.util.CloudTasksUtils.SerializableCloudTasksClient;
 import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
 import java.io.IOException;
-import javax.inject.Provider;
+import java.io.Serializable;
+import java.util.function.Supplier;
 import javax.inject.Singleton;
 
 /**
@@ -42,24 +45,35 @@ public abstract class CloudTasksUtilsModule {
   public static CloudTasksUtils provideCloudTasksUtils(
       @Config("projectId") String projectId,
       @Config("locationId") String locationId,
-      // Use a provider so that we can use try-with-resources with the client, which implements
-      // Autocloseable.
-      Provider<CloudTasksClient> clientProvider,
+      SerializableCloudTasksClient client,
       Retrier retrier) {
-    return new CloudTasksUtils(retrier, projectId, locationId, clientProvider);
+    return new CloudTasksUtils(retrier, projectId, locationId, client);
+  }
+
+  // Provides a supplier instead of using a Dagger @Provider because the latter is not serializable.
+  @Provides
+  public static Supplier<CloudTasksClient> provideCloudTasksClientSupplier(
+      @DefaultCredential GoogleCredentialsBundle credentials) {
+    return (Supplier<CloudTasksClient> & Serializable)
+        () -> {
+          CloudTasksClient client;
+          try {
+            client =
+                CloudTasksClient.create(
+                    CloudTasksSettings.newBuilder()
+                        .setCredentialsProvider(
+                            FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
+                        .build());
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+          return client;
+        };
   }
 
   @Provides
-  public static CloudTasksClient provideCloudTasksClient(
-      @DefaultCredential GoogleCredentialsBundle credentials) {
-    try {
-      return CloudTasksClient.create(
-          CloudTasksSettings.newBuilder()
-              .setCredentialsProvider(
-                  FixedCredentialsProvider.create(credentials.getGoogleCredentials()))
-              .build());
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
+  public static SerializableCloudTasksClient provideSerializableCloudTasksClient(
+      final Supplier<CloudTasksClient> clientSupplier) {
+    return new GcpCloudTasksClient(clientSupplier);
   }
 }

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -1306,6 +1306,18 @@ public final class RegistryConfig {
     public static ImmutableSet<String> provideAllowedEcdsaCurves(RegistryConfigSettings config) {
       return ImmutableSet.copyOf(config.sslCertificateValidation.allowedEcdsaCurves);
     }
+
+    @Provides
+    @Config("minMonthsBeforeWipeOut")
+    public static int provideMinMonthsBeforeWipeOut(RegistryConfigSettings config) {
+      return config.contactHistory.minMonthsBeforeWipeOut;
+    }
+
+    @Provides
+    @Config("wipeOutQueryBatchSize")
+    public static int provideWipeOutQueryBatchSize(RegistryConfigSettings config) {
+      return config.contactHistory.wipeOutQueryBatchSize;
+    }
   }
 
   /** Returns the App Engine project ID, which is based off the environment name. */

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -41,6 +41,7 @@ public class RegistryConfigSettings {
   public Keyring keyring;
   public RegistryTool registryTool;
   public SslCertificateValidation sslCertificateValidation;
+  public ContactHistory contactHistory;
 
   /** Configuration options that apply to the entire App Engine project. */
   public static class AppEngine {
@@ -234,4 +235,10 @@ public class RegistryConfigSettings {
     public String expirationWarningEmailBodyText;
     public String expirationWarningEmailSubjectText;
   }
+
+  /** Configuration for contact history. */
+  public static class ContactHistory {
+    public int minMonthsBeforeWipeOut;
+    public int wipeOutQueryBatchSize;
+  }
 }

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -442,6 +442,13 @@ registryTool:
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
 
+# Configuration options for handling contact history.
+contactHistory:
+  # The number of months that a ContactHistory entity should be stored in the database.
+  minMonthsBeforeWipeOut: 18
+  # The batch size for querying ContactHistory table in the database.
+  wipeOutQueryBatchSize: 500
+
 # Configuration options for checking SSL certificates.
 sslCertificateValidation:
   # A map specifying the maximum amount of days the certificate can be valid.

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -391,6 +391,13 @@
     <url-pattern>/_dr/task/relockDomain</url-pattern>
   </servlet-mapping>
 
+  <!-- Background action to wipe out PII fields of ContactHistory entities that
+have been in the database for a certain period of time. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutContactHistoryPii</url-pattern>
+  </servlet-mapping>
+
   <!-- Action to wipeout Cloud SQL data -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -348,4 +348,14 @@
     <schedule>every 3 minutes</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -246,4 +246,14 @@
     <schedule>every 3 minutes</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -371,6 +371,10 @@ public final class DomainDeleteFlow implements TransactionalFlow {
             String.format(
                 "Domain %s was deleted by registry administrator with final deletion effective: %s",
                 existingDomain.getDomainName(), deletionTime))
+        .setResponseData(
+            ImmutableList.of(
+                DomainPendingActionNotificationResponse.create(
+                    existingDomain.getDomainName(), true, trid, now)))
         .build();
   }
 

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -229,6 +229,15 @@ public final class DomainRenewFlow implements TransactionalFlow {
 
   private DomainHistory buildDomainHistory(
       DomainBase newDomain, DateTime now, Period period, Duration renewGracePeriod) {
+    Optional<MetadataExtension> metadataExtensionOpt =
+        eppInput.getSingleExtension(MetadataExtension.class);
+    if (metadataExtensionOpt.isPresent()) {
+      MetadataExtension metadataExtension = metadataExtensionOpt.get();
+      if (metadataExtension.getReason() != null) {
+        historyBuilder.setReason(metadataExtension.getReason());
+      }
+      historyBuilder.setRequestedByRegistrar(metadataExtension.getRequestedByRegistrar());
+    }
     return historyBuilder
         .setType(DOMAIN_RENEW)
         .setPeriod(period)

core/src/main/java/google/registry/model/AppEngineEnvironment.java

@@ -12,12 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package google.registry.backup;
+package google.registry.model;
 
 import com.google.apphosting.api.ApiProxy;
 import com.google.apphosting.api.ApiProxy.Environment;
 import com.google.common.collect.ImmutableMap;
-import java.io.Closeable;
+import google.registry.model.ofy.ObjectifyService;
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 
@@ -38,26 +39,61 @@ import java.lang.reflect.Proxy;
  * <p>Note that conversion from Objectify objects to Datastore {@code Entities} still requires the
  * Datastore service.
  */
-public class AppEngineEnvironment implements Closeable {
+public class AppEngineEnvironment {
 
-  private boolean isPlaceHolderNeeded;
+  private Environment environment;
 
+  /**
+   * Constructor for use by tests.
+   *
+   * <p>All test suites must use the same appId for environments, since when tearing down we do not
+   * clear cached environments in spawned threads. See {@link #unsetEnvironmentForAllThreads} for
+   * more information.
+   */
   public AppEngineEnvironment() {
-    this("PlaceholderAppId");
+    /**
+     * Use AppEngineExtension's appId here so that ofy and sql entities can be compared with {@code
+     * Objects#equals()}. The choice of this value does not impact functional correctness.
+     */
+    this("test");
   }
 
+  /** Constructor for use by applications, e.g., BEAM pipelines. */
   public AppEngineEnvironment(String appId) {
-    isPlaceHolderNeeded = ApiProxy.getCurrentEnvironment() == null;
-    // isPlaceHolderNeeded may be true when we are invoked in a test with AppEngineExtension.
-    if (isPlaceHolderNeeded) {
-      ApiProxy.setEnvironmentForCurrentThread(createAppEngineEnvironment(appId));
-    }
+    environment = createAppEngineEnvironment(appId);
   }
 
-  @Override
-  public void close() {
-    if (isPlaceHolderNeeded) {
-      ApiProxy.setEnvironmentForCurrentThread(null);
+  public void setEnvironmentForCurrentThread() {
+    ApiProxy.setEnvironmentForCurrentThread(environment);
+    ObjectifyService.initOfy();
+  }
+
+  public void setEnvironmentForAllThreads() {
+    setEnvironmentForCurrentThread();
+    ApiProxy.setEnvironmentFactory(() -> environment);
+  }
+
+  public void unsetEnvironmentForCurrentThread() {
+    ApiProxy.clearEnvironmentForCurrentThread();
+  }
+
+  /**
+   * Unsets the test environment in all threads with best effort.
+   *
+   * <p>This method unsets the environment factory and clears the cached environment in the current
+   * thread (the main test runner thread). We do not clear the cache in spawned threads, even though
+   * they may be reused. This is not a problem as long as the appId stays the same: those threads
+   * are used only in AppEngine or BEAM tests, and expect the presence of an environment.
+   */
+  public void unsetEnvironmentForAllThreads() {
+    unsetEnvironmentForCurrentThread();
+
+    try {
+      Method method = ApiProxy.class.getDeclaredMethod("clearEnvironmentFactory");
+      method.setAccessible(true);
+      method.invoke(null);
+    } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+      throw new RuntimeException(e);
     }
   }
 

core/src/main/java/google/registry/model/BackupGroupRoot.java

@@ -14,6 +14,7 @@
 
 package google.registry.model;
 
+import google.registry.util.PreconditionsUtils;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.AttributeOverride;
@@ -30,7 +31,7 @@ import javax.xml.bind.annotation.XmlTransient;
  * that we can enforce strictly increasing timestamps.
  */
 @MappedSuperclass
-public abstract class BackupGroupRoot extends ImmutableObject {
+public abstract class BackupGroupRoot extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * An automatically managed timestamp of when this object was last written to Datastore.
@@ -49,4 +50,14 @@ public abstract class BackupGroupRoot extends ImmutableObject {
   public UpdateAutoTimestamp getUpdateTimestamp() {
     return updateTimestamp;
   }
+
+  /**
+   * Copies {@link #updateTimestamp} from another entity.
+   *
+   * <p>This method is for the few cases when {@code updateTimestamp} is copied between different
+   * types of entities. Use {@link #clone} for same-type copying.
+   */
+  protected void copyUpdateTimestamp(BackupGroupRoot other) {
+    this.updateTimestamp = PreconditionsUtils.checkArgumentNotNull(other, "other").updateTimestamp;
+  }
 }

core/src/main/java/google/registry/model/CreateAutoTimestamp.java

@@ -23,7 +23,7 @@ import org.joda.time.DateTime;
  *
  * @see CreateAutoTimestampTranslatorFactory
  */
-public class CreateAutoTimestamp extends ImmutableObject {
+public class CreateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
   DateTime timestamp;
 

core/src/main/java/google/registry/model/IdService.java

@@ -18,13 +18,14 @@ import static com.google.common.base.Preconditions.checkState;
 
 import com.google.appengine.api.datastore.DatastoreServiceFactory;
 import com.google.common.annotations.VisibleForTesting;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
 import google.registry.config.RegistryEnvironment;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Allocates a globally unique {@link Long} number to use as an Ofy {@code @Id}.
  *
- * <p>In non-test environments the Id is generated by Datastore, whereas in tests it's from an
+ * <p>In non-test, non-beam environments the Id is generated by Datastore, otherwise it's from an
  * atomic long number that's incremented every time this method is called.
  */
 public final class IdService {
@@ -35,13 +36,25 @@ public final class IdService {
    */
   private static final String APP_WIDE_ALLOCATION_KIND = "common";
 
-  /** Counts of used ids for use in unit tests. Outside tests this is never used. */
-  private static final AtomicLong nextTestId = new AtomicLong(1); // ids cannot be zero
+  /**
+   * Counts of used ids for use in unit tests or Beam.
+   *
+   * <p>Note that one should only use self-allocate Ids in Beam for entities whose Ids are not
+   * important and are not persisted back to the database, i. e. nowhere the uniqueness of the ID is
+   * required.
+   */
+  private static final AtomicLong nextSelfAllocatedId = new AtomicLong(1); // ids cannot be zero
+
+  private static final boolean isSelfAllocated() {
+    return RegistryEnvironment.UNITTEST.equals(RegistryEnvironment.get())
+        || "true".equals(System.getProperty(RegistryPipelineWorkerInitializer.PROPERTY, "false"));
+  }
 
   /** Allocates an id. */
+  // TODO(b/201547855): Find a way to allocate a unique ID without datastore.
   public static long allocateId() {
-    return RegistryEnvironment.UNITTEST.equals(RegistryEnvironment.get())
-        ? nextTestId.getAndIncrement()
+    return isSelfAllocated()
+        ? nextSelfAllocatedId.getAndIncrement()
         : DatastoreServiceFactory.getDatastoreService()
             .allocateIds(APP_WIDE_ALLOCATION_KIND, 1)
             .iterator()
@@ -49,13 +62,11 @@ public final class IdService {
             .getId();
   }
 
-  /** Resets the global test id counter (i.e. sets the next id to 1). */
+  /** Resets the global self-allocated id counter (i.e. sets the next id to 1). */
   @VisibleForTesting
-  public static void resetNextTestId() {
+  public static void resetSelfAllocatedId() {
     checkState(
-        RegistryEnvironment.UNITTEST.equals(RegistryEnvironment.get()),
-        "Can't call resetTestIdCounts() from RegistryEnvironment.%s",
-        RegistryEnvironment.get());
-    nextTestId.set(1); // ids cannot be zero
+        isSelfAllocated(), "Can only call resetSelfAllocatedId() in unit tests or Beam pipelines");
+    nextSelfAllocatedId.set(1); // ids cannot be zero
   }
 }

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -85,6 +85,9 @@ public abstract class ImmutableObject implements Cloneable {
   @Target(FIELD)
   public @interface Insignificant {}
 
+  // Note: if this class is made to implement Serializable, this field must become 'transient' since
+  // hashing is not stable across executions. Also note that @XmlTransient is forbidden on transient
+  // fields and need to be removed if transient is added.
   @Ignore @XmlTransient protected Integer hashCode;
 
   private boolean equalsImmutableObject(ImmutableObject other) {

core/src/main/java/google/registry/model/UnsafeSerializable.java

@@ -0,0 +1,34 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import java.io.Serializable;
+
+/**
+ * Marker interface for Nomulus entities whose serialization are implemented in a fragile way. These
+ * entities are made {@link Serializable} so that they can be passed between JVMs. The intended use
+ * case is BEAM pipeline-based cross-database data validation between Datastore and Cloud SQL during
+ * the migration. Note that only objects loaded from the SQL database need serialization support.
+ * Objects exported from Datastore can already be serialized as protocol buffers.
+ *
+ * <p>All entities implementing this interface take advantage of the fact that all Java collection
+ * classes we use, either directly or indirectly, including those in Java libraries, Guava,
+ * Objectify, and Hibernate are {@code Serializable}.
+ *
+ * <p>The {@code serialVersionUID} field has also been omitted in the implementing classes, since
+ * they are not used for persistence.
+ */
+// TODO(b/203609782): either remove this interface or fix implementors post migration.
+public interface UnsafeSerializable extends Serializable {}

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -15,12 +15,8 @@
 package google.registry.model;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static org.joda.time.DateTimeZone.UTC;
 
-import com.google.common.flogger.FluentLogger;
-import com.google.common.flogger.StackSize;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.OnLoad;
 import google.registry.model.translators.UpdateAutoTimestampTranslatorFactory;
@@ -42,14 +38,12 @@ import org.joda.time.DateTime;
  * @see UpdateAutoTimestampTranslatorFactory
  */
 @Embeddable
-public class UpdateAutoTimestamp extends ImmutableObject {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+public class UpdateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
   // When set to true, database converters/translators should do the auto update.  When set to
   // false, auto update should be suspended (this exists to allow us to preserve the original value
   // during a replay).
-  private static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
+  private static final ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
 
   @Transient DateTime timestamp;
 
@@ -63,16 +57,7 @@ public class UpdateAutoTimestamp extends ImmutableObject {
   @PrePersist
   @PreUpdate
   void setTimestamp() {
-    // On the off chance that this is called outside of a transaction, log it instead of failing
-    // with an exception from attempting to call jpaTm().getTransactionTime(), and then fall back
-    // to DateTime.now(UTC).
-    if (!jpaTm().inTransaction()) {
-      logger.atSevere().withStackTrace(StackSize.MEDIUM).log(
-          "Failed to update automatic timestamp because this wasn't called in a JPA transaction%s.",
-          ofyTm().inTransaction() ? " (but there is an open Ofy transaction)" : "");
-      timestamp = DateTime.now(UTC);
-      lastUpdateTime = DateTimeUtils.toZonedDateTime(timestamp);
-    } else if (autoUpdateEnabled() || lastUpdateTime == null) {
+    if (autoUpdateEnabled() || lastUpdateTime == null) {
       timestamp = jpaTm().getTransactionTime();
       lastUpdateTime = DateTimeUtils.toZonedDateTime(timestamp);
     }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -39,6 +39,7 @@ import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.TimeOfYear;
 import google.registry.model.domain.DomainBase;
@@ -72,7 +73,7 @@ import org.joda.time.DateTime;
 /** A billable event in a domain's lifecycle. */
 @MappedSuperclass
 public abstract class BillingEvent extends ImmutableObject
-    implements Buildable, TransferServerApproveEntity {
+    implements Buildable, TransferServerApproveEntity, UnsafeSerializable {
 
   /** The reason for the bill, which maps 1:1 to skus in go/registry-billing-skus. */
   public enum Reason {

core/src/main/java/google/registry/model/bulkquery/BulkQueryEntities.java

@@ -0,0 +1,109 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+
+/**
+ * Utilities for managing an alternative JPA entity model optimized for bulk loading multi-level
+ * entities such as {@link DomainBase} and {@link DomainHistory}.
+ *
+ * <p>In a bulk query for a multi-level JPA entity type, the JPA framework only generates a bulk
+ * query (SELECT * FROM table) for the base table. Then, for each row in the base table, additional
+ * queries are issued to load associated rows in child tables. This can be very slow when an entity
+ * type has multiple child tables.
+ *
+ * <p>We have defined an alternative entity model for {@code DomainBase} and {@code DomainHistory},
+ * where the base table as well as the child tables are mapped to single-level entity types. The
+ * idea is to load each of these types using a bulk query, and assemble them into the target type in
+ * memory in a pipeline. The main use case is Datastore-Cloud SQL validation during the Registry
+ * database migration, where we will need the full database snapshots frequently.
+ */
+public class BulkQueryEntities {
+  /**
+   * The JPA entity classes in persistence.xml to replace when creating the {@link
+   * JpaTransactionManager} for bulk query.
+   */
+  public static final ImmutableMap<String, String> JPA_ENTITIES_REPLACEMENTS =
+      ImmutableMap.of(
+          DomainBase.class.getCanonicalName(),
+          DomainBaseLite.class.getCanonicalName(),
+          DomainHistory.class.getCanonicalName(),
+          DomainHistoryLite.class.getCanonicalName());
+
+  /* The JPA entity classes that are not included in persistence.xml and need to be added to
+   * the {@link JpaTransactionManager} for bulk query.*/
+  public static final ImmutableList<String> JPA_ENTITIES_NEW =
+      ImmutableList.of(
+          DomainHost.class.getCanonicalName(), DomainHistoryHost.class.getCanonicalName());
+
+  public static DomainBase assembleDomainBase(
+      DomainBaseLite domainBaseLite,
+      ImmutableSet<GracePeriod> gracePeriods,
+      ImmutableSet<DelegationSignerData> delegationSignerData,
+      ImmutableSet<VKey<HostResource>> nsHosts) {
+    DomainBase.Builder builder = new DomainBase.Builder();
+    builder.copyFrom(domainBaseLite);
+    builder.setGracePeriods(gracePeriods);
+    builder.setDsData(delegationSignerData);
+    builder.setNameservers(nsHosts);
+    return builder.build();
+  }
+
+  public static DomainHistory assembleDomainHistory(
+      DomainHistoryLite domainHistoryLite,
+      ImmutableSet<DomainDsDataHistory> dsDataHistories,
+      ImmutableSet<VKey<HostResource>> domainHistoryHosts,
+      ImmutableSet<GracePeriodHistory> gracePeriodHistories,
+      ImmutableSet<DomainTransactionRecord> transactionRecords) {
+    DomainHistory.Builder builder = new DomainHistory.Builder();
+    builder.copyFrom(domainHistoryLite);
+    DomainContent rawDomainContent = domainHistoryLite.domainContent;
+    if (rawDomainContent != null) {
+      DomainContent newDomainContent =
+          domainHistoryLite
+              .domainContent
+              .asBuilder()
+              .setNameservers(domainHistoryHosts)
+              .setGracePeriods(
+                  gracePeriodHistories.stream()
+                      .map(GracePeriod::createFromHistory)
+                      .collect(toImmutableSet()))
+              .setDsData(
+                  dsDataHistories.stream()
+                      .map(DelegationSignerData::create)
+                      .collect(toImmutableSet()))
+              .build();
+      builder.setDomain(newDomainContent);
+    }
+    return builder.buildAndAssemble(
+        dsDataHistories, domainHistoryHosts, gracePeriodHistories, transactionRecords);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainBaseLite.java

@@ -0,0 +1,49 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import google.registry.persistence.WithStringVKey;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+
+/**
+ * A 'light' version of {@link DomainBase} with only base table ("Domain") attributes, which allows
+ * fast bulk loading. They are used in in-memory assembly of {@code DomainBase} instances along with
+ * bulk-loaded child entities ({@code GracePeriod} etc). The in-memory assembly achieves much higher
+ * performance than loading {@code DomainBase} directly.
+ *
+ * <p>Please refer to {@link BulkQueryEntities} for more information.
+ */
+@Entity(name = "Domain")
+@WithStringVKey
+@Access(AccessType.FIELD)
+public class DomainBaseLite extends DomainContent implements SqlOnlyEntity {
+
+  @Override
+  @javax.persistence.Id
+  @Access(AccessType.PROPERTY)
+  public String getRepoId() {
+    return super.getRepoId();
+  }
+
+  public static VKey<DomainBaseLite> createVKey(String repoId) {
+    return VKey.createSql(DomainBaseLite.class, repoId);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHistoryHost.java

@@ -0,0 +1,50 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.host.HostResource;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+
+/**
+ * A name server host referenced by a {@link google.registry.model.domain.DomainHistory} record.
+ * Please refer to {@link BulkQueryEntities} for usage.
+ */
+@Entity
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryHost.class)
+public class DomainHistoryHost implements Serializable, SqlOnlyEntity {
+
+  @Id private Long domainHistoryHistoryRevisionId;
+  @Id private String domainHistoryDomainRepoId;
+  @Id private String hostRepoId;
+
+  private DomainHistoryHost() {}
+
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(domainHistoryDomainRepoId, domainHistoryHistoryRevisionId);
+  }
+
+  public VKey<HostResource> getHostVKey() {
+    return VKey.create(HostResource.class, hostRepoId);
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java

@@ -0,0 +1,122 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import com.googlecode.objectify.Key;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.Period;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
+
+/**
+ * A 'light' version of {@link DomainHistory} with only base table ("DomainHistory") attributes,
+ * which allows fast bulk loading. They are used in in-memory assembly of {@code DomainHistory}
+ * instances along with bulk-loaded child entities ({@code GracePeriodHistory} etc). The in-memory
+ * assembly achieves much higher performance than loading {@code DomainHistory} directly.
+ *
+ * <p>Please refer to {@link BulkQueryEntities} for more information.
+ *
+ * <p>This class is adapted from {@link DomainHistory} by removing the {@code dsDataHistories},
+ * {@code gracePeriodHistories}, and {@code nsHosts} fields and associated methods.
+ */
+@Entity(name = "DomainHistory")
+@Access(AccessType.FIELD)
+@IdClass(DomainHistoryId.class)
+public class DomainHistoryLite extends HistoryEntry implements SqlOnlyEntity {
+
+  // Store DomainContent instead of DomainBase so we don't pick up its @Id
+  // Nullable for the sake of pre-Registry-3.0 history objects
+  @Nullable DomainContent domainContent;
+
+  @Id
+  @Access(AccessType.PROPERTY)
+  public String getDomainRepoId() {
+    // We need to handle null case here because Hibernate sometimes accesses this method before
+    // parent gets initialized
+    return parent == null ? null : parent.getName();
+  }
+
+  /** This method is private because it is only used by Hibernate. */
+  @SuppressWarnings("unused")
+  private void setDomainRepoId(String domainRepoId) {
+    parent = Key.create(DomainBase.class, domainRepoId);
+  }
+
+  @Override
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @AttributeOverrides({
+    @AttributeOverride(name = "unit", column = @Column(name = "historyPeriodUnit")),
+    @AttributeOverride(name = "value", column = @Column(name = "historyPeriodValue"))
+  })
+  public Period getPeriod() {
+    return super.getPeriod();
+  }
+
+  /**
+   * For transfers, the id of the other registrar.
+   *
+   * <p>For requests and cancels, the other registrar is the losing party (because the registrar
+   * sending the EPP transfer command is the gaining party). For approves and rejects, the other
+   * registrar is the gaining party.
+   */
+  @Nullable
+  @Access(AccessType.PROPERTY)
+  @Column(name = "historyOtherRegistrarId")
+  @Override
+  public String getOtherRegistrarId() {
+    return super.getOtherRegistrarId();
+  }
+
+  @Id
+  @Column(name = "historyRevisionId")
+  @Access(AccessType.PROPERTY)
+  @Override
+  public long getId() {
+    return super.getId();
+  }
+
+  /** The key to the {@link DomainBase} this is based off of. */
+  public VKey<DomainBase> getParentVKey() {
+    return VKey.create(DomainBase.class, getDomainRepoId());
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (domainContent == null) {
+      return;
+    }
+    // See inline comments in DomainHistory.postLoad for reasons for the following lines.
+    if (domainContent.getDomainName() == null) {
+      domainContent = null;
+    } else if (domainContent.getRepoId() == null) {
+      domainContent.setRepoId(parent.getName());
+    }
+  }
+}

core/src/main/java/google/registry/model/bulkquery/DomainHost.java

@@ -0,0 +1,46 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.bulkquery;
+
+import google.registry.model.host.HostResource;
+import google.registry.model.replay.SqlOnlyEntity;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+
+/** A name server host of a domain. Please refer to {@link BulkQueryEntities} for usage. */
+@Entity
+@Access(AccessType.FIELD)
+@IdClass(DomainHost.class)
+public class DomainHost implements Serializable, SqlOnlyEntity {
+
+  @Id private String domainRepoId;
+
+  @Id private String hostRepoId;
+
+  DomainHost() {}
+
+  public String getDomainRepoId() {
+    return domainRepoId;
+  }
+
+  public VKey<HostResource> getHostVKey() {
+    return VKey.create(HostResource.class, hostRepoId);
+  }
+}

core/src/main/java/google/registry/model/common/Cursor.java

@@ -27,13 +27,13 @@ import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.common.Cursor.CursorId;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.model.tld.Registry;
 import google.registry.persistence.VKey;
-import java.io.Serializable;
 import java.util.List;
 import java.util.Optional;
 import javax.persistence.Column;
@@ -53,7 +53,7 @@ import org.joda.time.DateTime;
 @javax.persistence.Entity
 @IdClass(CursorId.class)
 @InCrossTld
-public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
+public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity, UnsafeSerializable {
 
   /** The scope of a global cursor. A global cursor is a cursor that is not specific to one tld. */
   public static final String GLOBAL = "GLOBAL";
@@ -283,7 +283,7 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
     return cursorTime;
   }
 
-  static class CursorId extends ImmutableObject implements Serializable {
+  public static class CursorId extends ImmutableObject implements UnsafeSerializable {
 
     public CursorType type;
     public String scope;

core/src/main/java/google/registry/model/common/TimeOfYear.java

@@ -28,6 +28,7 @@ import com.google.common.collect.Range;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.List;
 import javax.persistence.Embeddable;
 import org.joda.time.DateTime;
@@ -45,7 +46,7 @@ import org.joda.time.DateTime;
  */
 @Embed
 @Embeddable
-public class TimeOfYear extends ImmutableObject {
+public class TimeOfYear extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * The time as "month day millis" with all fields left-padded with zeroes so that lexographic

core/src/main/java/google/registry/model/common/TimedTransitionProperty.java

@@ -27,7 +27,9 @@ import com.google.common.collect.Maps;
 import com.google.common.collect.Ordering;
 import com.googlecode.objectify.mapper.Mapper;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.util.TypeUtils;
+import java.io.Serializable;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -53,11 +55,12 @@ import org.joda.time.DateTime;
  * to use for storing the list of transitions. The user is given this choice of subclass so that the
  * field of the value type stored in the transition can be given a customized name.
  */
-public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedTransition<V>>
-    extends ForwardingMap<DateTime, T> {
+public class TimedTransitionProperty<
+        V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
+    extends ForwardingMap<DateTime, T> implements UnsafeSerializable {
 
   /**
-   * A transition to a value of type {@code V} at a certain time.  This superclass only has a field
+   * A transition to a value of type {@code V} at a certain time. This superclass only has a field
    * for the {@code DateTime}, which means that subclasses should supply the field of type {@code V}
    * and implementations of the abstract getter and setter methods to access that field. This design
    * is so that subclasses tagged with @Embed can define a custom field name for their value, for
@@ -65,11 +68,12 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    *
    * <p>The public visibility of this class exists only so that it can be subclassed; clients should
    * never call any methods on this class or attempt to access its members, but should instead treat
-   * it as a customizable implementation detail of {@code TimedTransitionProperty}.  However, note
+   * it as a customizable implementation detail of {@code TimedTransitionProperty}. However, note
    * that subclasses must also have public visibility so that they can be instantiated via
    * reflection in a call to {@code fromValueMap}.
    */
-  public abstract static class TimedTransition<V> extends ImmutableObject {
+  public abstract static class TimedTransition<V extends Serializable> extends ImmutableObject
+      implements UnsafeSerializable {
     /** The time at which this value becomes the active value. */
     private DateTime transitionTime;
 
@@ -89,16 +93,16 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
   }
 
   /**
-   * Converts the provided value map into the equivalent transition map, using transition objects
-   * of the given TimedTransition subclass.  The value map must be sorted according to the natural
+   * Converts the provided value map into the equivalent transition map, using transition objects of
+   * the given TimedTransition subclass. The value map must be sorted according to the natural
    * ordering of its DateTime keys, and keys cannot be earlier than START_OF_TIME.
    */
   // NB: The Class<T> parameter could be eliminated by getting the class via reflection, but then
   // the callsite cannot infer T, so unless you explicitly call this as .<V, T>fromValueMap() it
   // will default to using just TimedTransition<V>, which fails at runtime.
-  private static <V, T extends TimedTransition<V>> NavigableMap<DateTime, T> makeTransitionMap(
-      ImmutableSortedMap<DateTime, V> valueMap,
-      final Class<T> timedTransitionSubclass) {
+  private static <V extends Serializable, T extends TimedTransition<V>>
+      NavigableMap<DateTime, T> makeTransitionMap(
+          ImmutableSortedMap<DateTime, V> valueMap, final Class<T> timedTransitionSubclass) {
     checkArgument(
         Ordering.natural().equals(valueMap.comparator()),
         "Timed transition value map must have transition time keys in chronological order");
@@ -121,9 +125,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    *
    * <p>This method should be the normal method for constructing a {@link TimedTransitionProperty}.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> fromValueMap(
-      ImmutableSortedMap<DateTime, V> valueMap,
-      final Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> fromValueMap(
+          ImmutableSortedMap<DateTime, V> valueMap, final Class<T> timedTransitionSubclass) {
     return new TimedTransitionProperty<>(ImmutableSortedMap.copyOf(
         makeTransitionMap(valueMap, timedTransitionSubclass)));
   }
@@ -175,10 +179,10 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * @param allowedTransitions optional map of all possible state-to-state transitions
    * @param allowedTransitionMapName optional transition map description string for error messages
    * @param initialValue optional initial value; if present, the first transition must have this
-   *        value
+   *     value
    * @param badInitialValueErrorMessage option error message string if the initial value is wrong
    */
-  public static <V, T extends TimedTransitionProperty.TimedTransition<V>>
+  public static <V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
       TimedTransitionProperty<V, T> make(
           ImmutableSortedMap<DateTime, V> newTransitions,
           Class<T> transitionClass,
@@ -200,7 +204,7 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * Validates that a transition map is not null or empty, starts at START_OF_TIME, and has
    * transitions which move from one value to another in allowed ways.
    */
-  public static <V, T extends TimedTransitionProperty.TimedTransition<V>>
+  public static <V extends Serializable, T extends TimedTransitionProperty.TimedTransition<V>>
       void validateTimedTransitionMap(
           @Nullable NavigableMap<DateTime, V> transitionMap,
           ImmutableMultimap<V, V> allowedTransitions,
@@ -240,8 +244,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * annotation. The map for those fields must be mutable so that Objectify can load values from
    * Datastore into the map, but clients should still never mutate the field's map directly.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> forMapify(
-      ImmutableSortedMap<DateTime, V> valueMap, Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> forMapify(
+          ImmutableSortedMap<DateTime, V> valueMap, Class<T> timedTransitionSubclass) {
     return new TimedTransitionProperty<>(
         new TreeMap<>(makeTransitionMap(valueMap, timedTransitionSubclass)));
   }
@@ -254,8 +259,9 @@ public class TimedTransitionProperty<V, T extends TimedTransitionProperty.TimedT
    * annotation. The map for those fields must be mutable so that Objectify can load values from
    * Datastore into the map, but clients should still never mutate the field's map directly.
    */
-  public static <V, T extends TimedTransition<V>> TimedTransitionProperty<V, T> forMapify(
-      V valueAtStartOfTime, Class<T> timedTransitionSubclass) {
+  public static <V extends Serializable, T extends TimedTransition<V>>
+      TimedTransitionProperty<V, T> forMapify(
+          V valueAtStartOfTime, Class<T> timedTransitionSubclass) {
     return forMapify(
         ImmutableSortedMap.of(START_OF_TIME, valueAtStartOfTime), timedTransitionSubclass);
   }

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -20,6 +20,7 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.contact.ContactHistory.ContactHistoryId;
 import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.SqlEntity;
@@ -59,7 +60,7 @@ import javax.persistence.PostLoad;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(ContactHistoryId.class)
-public class ContactHistory extends HistoryEntry implements SqlEntity {
+public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable {
 
   // Store ContactBase instead of ContactResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects
@@ -227,5 +228,11 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
       getInstance().parent = Key.create(ContactResource.class, contactRepoId);
       return this;
     }
+
+    public Builder wipeOutPii() {
+      getInstance().contactBase =
+          getInstance().getContactBase().get().asBuilder().wipeOut().build();
+      return this;
+    }
   }
 }

core/src/main/java/google/registry/model/contact/Disclose.java

@@ -20,7 +20,9 @@ import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.eppcommon.PresenceMarker;
+import java.io.Serializable;
 import java.util.List;
 import javax.persistence.Embeddable;
 import javax.persistence.Embedded;
@@ -31,7 +33,7 @@ import javax.xml.bind.annotation.XmlType;
 @Embed
 @Embeddable
 @XmlType(propOrder = {"name", "org", "addr", "voice", "fax", "email"})
-public class Disclose extends ImmutableObject {
+public class Disclose extends ImmutableObject implements UnsafeSerializable {
 
   List<PostalInfoChoice> name;
 
@@ -78,7 +80,7 @@ public class Disclose extends ImmutableObject {
 
   /** The "intLocType" from <a href="http://tools.ietf.org/html/rfc5733">RFC5733</a>. */
   @Embed
-  public static class PostalInfoChoice extends ImmutableObject {
+  public static class PostalInfoChoice extends ImmutableObject implements Serializable {
     @XmlAttribute
     PostalInfo.Type type;
 

core/src/main/java/google/registry/model/contact/PostalInfo.java

@@ -20,6 +20,7 @@ import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.Buildable;
 import google.registry.model.Buildable.Overlayable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.persistence.Embeddable;
 import javax.persistence.EnumType;
@@ -38,7 +39,8 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @Embed
 @Embeddable
 @XmlType(propOrder = {"name", "org", "address", "type"})
-public class PostalInfo extends ImmutableObject implements Overlayable<PostalInfo> {
+public class PostalInfo extends ImmutableObject
+    implements Overlayable<PostalInfo>, UnsafeSerializable {
 
   /** The type of the address, either localized or international. */
   public enum Type {

core/src/main/java/google/registry/model/domain/DesignatedContact.java

@@ -21,6 +21,7 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Index;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.contact.ContactResource;
 import google.registry.persistence.VKey;
 import javax.persistence.Embeddable;
@@ -46,7 +47,7 @@ import javax.xml.bind.annotation.XmlEnumValue;
  */
 @Embed
 @Embeddable
-public class DesignatedContact extends ImmutableObject {
+public class DesignatedContact extends ImmutableObject implements UnsafeSerializable {
 
   /**
    * XML type for contact types. This can be either: {@code "admin"}, {@code "billing"}, or

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -24,6 +24,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import java.util.Set;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -164,6 +165,11 @@ public class DomainBase extends DomainContent
     return cloneDomainProjectedAtTime(this, now);
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();
@@ -189,6 +195,7 @@ public class DomainBase extends DomainContent
     }
 
     public Builder copyFrom(DomainContent domainContent) {
+      this.getInstance().copyUpdateTimestamp(domainContent);
       return this.setAuthInfo(domainContent.getAuthInfo())
           .setAutorenewPollMessage(domainContent.getAutorenewPollMessage())
           .setAutorenewBillingEvent(domainContent.getAutorenewBillingEvent())

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -865,7 +865,8 @@ public class DomainContent extends EppResource
     public B setDomainName(String domainName) {
       checkArgument(
           domainName.equals(canonicalizeDomainName(domainName)),
-          "Domain name must be in puny-coded, lower-case form");
+          "Domain name %s not in puny-coded, lower-case form",
+          domainName);
       getInstance().fullyQualifiedDomainName = domainName;
       return thisCastToDerived();
     }

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -33,6 +33,7 @@ import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.HashSet;
 import java.util.Optional;
@@ -216,6 +217,10 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     return super.getId();
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), getId());
+  }
+
   /** Returns keys to the {@link HostResource} that are the nameservers for the domain. */
   public Set<VKey<HostResource>> getNsHosts() {
     return nsHosts;
@@ -299,6 +304,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   public void beforeSqlSaveOnReplay() {
     if (domainContent == null) {
       domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
+      domainContent.fullyQualifiedDomainName =
+          DomainNameUtils.canonicalizeDomainName(domainContent.fullyQualifiedDomainName);
       fillAuxiliaryFieldsFromDomain(this);
     }
   }
@@ -314,6 +321,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
           nullToEmptyImmutableCopy(domainHistory.domainContent.getGracePeriods()).stream()
               .map(gracePeriod -> GracePeriodHistory.createFrom(domainHistory.id, gracePeriod))
               .collect(toImmutableSet());
+    } else {
+      domainHistory.nsHosts = ImmutableSet.of();
     }
   }
 
@@ -393,8 +402,16 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       if (domainContent == null) {
         return this;
       }
+      // TODO(b/203609982): if actual type of domainContent is DomainBase, convert to DomainContent
+      // Note: a DomainHistory fetched by JPA has DomainContent in this field. Allowing DomainBase
+      // in the setter makes equality checks messy.
       getInstance().domainContent = domainContent;
-      return super.setParent(domainContent);
+      if (domainContent instanceof DomainBase) {
+        super.setParent(domainContent);
+      } else {
+        super.setParent(Key.create(DomainBase.class, domainContent.getRepoId()));
+      }
+      return this;
     }
 
     public Builder setDomainRepoId(String domainRepoId) {
@@ -412,5 +429,19 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       fillAuxiliaryFieldsFromDomain(instance);
       return instance;
     }
+
+    public DomainHistory buildAndAssemble(
+        ImmutableSet<DomainDsDataHistory> dsDataHistories,
+        ImmutableSet<VKey<HostResource>> domainHistoryHosts,
+        ImmutableSet<GracePeriodHistory> gracePeriodHistories,
+        ImmutableSet<DomainTransactionRecord> transactionRecords) {
+      DomainHistory instance = super.build();
+      instance.dsDataHistories = dsDataHistories;
+      instance.nsHosts = domainHistoryHosts;
+      instance.gracePeriodHistories = gracePeriodHistories;
+      instance.domainTransactionRecords = transactionRecords;
+      instance.hashCode = null;
+      return instance;
+    }
   }
 }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -22,8 +22,10 @@ import com.google.common.annotations.VisibleForTesting;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.replay.DatastoreAndSqlEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import google.registry.persistence.BillingVKey.BillingEventVKey;
 import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
@@ -202,7 +204,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
   /** Entity class to represent a historic {@link GracePeriod}. */
   @Entity(name = "GracePeriodHistory")
   @Table(indexes = @Index(columnList = "domainRepoId"))
-  static class GracePeriodHistory extends GracePeriodBase {
+  public static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity {
     @Id Long gracePeriodHistoryRevisionId;
 
     /** ID for the associated {@link DomainHistory} entity. */
@@ -214,6 +216,10 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
       return super.getGracePeriodId();
     }
 
+    public DomainHistoryId getDomainHistoryId() {
+      return new DomainHistoryId(getDomainRepoId(), domainHistoryRevisionId);
+    }
+
     static GracePeriodHistory createFrom(long historyRevisionId, GracePeriod gracePeriod) {
       GracePeriodHistory instance = new GracePeriodHistory();
       instance.gracePeriodHistoryRevisionId = allocateId();

core/src/main/java/google/registry/model/domain/GracePeriodBase.java

@@ -17,6 +17,7 @@ package google.registry.model.domain;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.persistence.BillingVKey.BillingEventVKey;
@@ -35,7 +36,7 @@ import org.joda.time.DateTime;
 @Embed
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public class GracePeriodBase extends ImmutableObject {
+public class GracePeriodBase extends ImmutableObject implements UnsafeSerializable {
 
   /** Unique id required for hibernate representation. */
   @Transient long gracePeriodId;

core/src/main/java/google/registry/model/domain/Period.java

@@ -16,6 +16,7 @@ package google.registry.model.domain;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -25,7 +26,7 @@ import javax.xml.bind.annotation.XmlValue;
 /** The "periodType" from <a href="http://tools.ietf.org/html/rfc5731">RFC5731</a>. */
 @Embed
 @javax.persistence.Embeddable
-public class Period extends ImmutableObject {
+public class Period extends ImmutableObject implements UnsafeSerializable {
 
   @Enumerated(EnumType.STRING)
   @XmlAttribute

core/src/main/java/google/registry/model/domain/launch/LaunchNotice.java

@@ -27,6 +27,7 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.persistence.Embedded;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -39,7 +40,7 @@ import org.joda.time.DateTime;
 @Embed
 @XmlType(propOrder = {"noticeId", "expirationTime", "acceptedTime"})
 @javax.persistence.Embeddable
-public class LaunchNotice extends ImmutableObject {
+public class LaunchNotice extends ImmutableObject implements UnsafeSerializable {
 
   /** An empty instance to use in place of null. */
   private static final NoticeIdType EMPTY_NOTICE_ID = new NoticeIdType();
@@ -47,14 +48,13 @@ public class LaunchNotice extends ImmutableObject {
   /** An id with a validator-id attribute. */
   @Embed
   @javax.persistence.Embeddable
-  public static class NoticeIdType extends ImmutableObject {
+  public static class NoticeIdType extends ImmutableObject implements UnsafeSerializable {
 
     /**
-     * The Trademark Claims Notice ID from
-     * {@link "http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3"}.
+     * The Trademark Claims Notice ID from <a
+     * href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">the RFC</a>.
      */
-    @XmlValue
-    String tcnId;
+    @XmlValue String tcnId;
 
     /** The identifier of the TMDB provider to use, defaulting to the TMCH. */
     @IgnoreSave(IfNull.class)

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataBase.java

@@ -17,6 +17,7 @@ package google.registry.model.domain.secdns;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.MappedSuperclass;
@@ -31,7 +32,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @Embed
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public abstract class DomainDsDataBase extends ImmutableObject {
+public abstract class DomainDsDataBase extends ImmutableObject implements UnsafeSerializable {
 
   @Ignore @XmlTransient @Transient String domainRepoId;
 

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java

@@ -16,7 +16,9 @@ package google.registry.model.domain.secdns;
 
 import static google.registry.model.IdService.allocateId;
 
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.replay.SqlOnlyEntity;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
@@ -26,7 +28,8 @@ import javax.persistence.Id;
 
 /** Entity class to represent a historic {@link DelegationSignerData}. */
 @Entity
-public class DomainDsDataHistory extends DomainDsDataBase implements SqlOnlyEntity {
+public class DomainDsDataHistory extends DomainDsDataBase
+    implements SqlOnlyEntity, UnsafeSerializable {
 
   @Id Long dsDataHistoryRevisionId;
 
@@ -53,6 +56,10 @@ public class DomainDsDataHistory extends DomainDsDataBase implements SqlOnlyEnti
     return instance;
   }
 
+  public DomainHistory.DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), domainHistoryRevisionId);
+  }
+
   @Override
   @Access(AccessType.PROPERTY)
   public String getDomainRepoId() {

core/src/main/java/google/registry/model/eppcommon/Address.java

@@ -27,6 +27,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -56,7 +57,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public class Address extends ImmutableObject implements Jsonifiable {
+public class Address extends ImmutableObject implements Jsonifiable, UnsafeSerializable {
 
   /** The schema validation will enforce that this has 3 lines at most. */
   // TODO(b/177569726): Remove this field after migration. We need to figure out how to generate

core/src/main/java/google/registry/model/eppcommon/AuthInfo.java

@@ -16,6 +16,7 @@ package google.registry.model.eppcommon;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Embeddable;
 import javax.persistence.Embedded;
 import javax.persistence.MappedSuperclass;
@@ -35,7 +36,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public abstract class AuthInfo extends ImmutableObject {
+public abstract class AuthInfo extends ImmutableObject implements UnsafeSerializable {
 
   @Embedded protected PasswordAuth pw;
 
@@ -47,7 +48,7 @@ public abstract class AuthInfo extends ImmutableObject {
   @Embed
   @XmlType(namespace = "urn:ietf:params:xml:ns:eppcom-1.0")
   @Embeddable
-  public static class PasswordAuth extends ImmutableObject {
+  public static class PasswordAuth extends ImmutableObject implements UnsafeSerializable {
     @XmlValue
     @XmlJavaTypeAdapter(NormalizedStringAdapter.class)
     String value;

core/src/main/java/google/registry/model/eppcommon/PhoneNumber.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Embeddable;
 import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlAttribute;
@@ -49,7 +50,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 @XmlTransient
 @Embeddable
 @MappedSuperclass
-public class PhoneNumber extends ImmutableObject {
+public class PhoneNumber extends ImmutableObject implements UnsafeSerializable {
 
   @XmlValue
   @XmlJavaTypeAdapter(CollapsedStringAdapter.class)

core/src/main/java/google/registry/model/eppcommon/Trid.java

@@ -18,6 +18,7 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
 import javax.xml.bind.annotation.XmlElement;
@@ -32,7 +33,7 @@ import javax.xml.bind.annotation.XmlType;
 @Embed
 @XmlType(propOrder = {"clientTransactionId", "serverTransactionId"})
 @javax.persistence.Embeddable
-public class Trid extends ImmutableObject {
+public class Trid extends ImmutableObject implements UnsafeSerializable {
 
   /** The server transaction id. */
   @XmlElement(name = "svTRID", namespace = "urn:ietf:params:xml:ns:epp-1.0")

core/src/main/java/google/registry/model/host/HostBase.java

@@ -196,7 +196,8 @@ public class HostBase extends EppResource {
     public B setHostName(String hostName) {
       checkArgument(
           hostName.equals(canonicalizeDomainName(hostName)),
-          "Host name must be in puny-coded, lower-case form");
+          "Host name %s not in puny-coded, lower-case form",
+          hostName);
       getInstance().fullyQualifiedHostName = hostName;
       return thisCastToDerived();
     }

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -20,11 +20,13 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.host.HostHistory.HostHistoryId;
 import google.registry.model.replay.DatastoreEntity;
 import google.registry.model.replay.SqlEntity;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.util.DomainNameUtils;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -60,7 +62,7 @@ import javax.persistence.PostLoad;
 @EntitySubclass
 @Access(AccessType.FIELD)
 @IdClass(HostHistoryId.class)
-public class HostHistory extends HistoryEntry implements SqlEntity {
+public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerializable {
 
   // Store HostBase instead of HostResource so we don't pick up its @Id
   // Nullable for the sake of pre-Registry-3.0 history objects
@@ -140,6 +142,8 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
   public void beforeSqlSaveOnReplay() {
     if (hostBase == null) {
       hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
+      hostBase.fullyQualifiedHostName =
+          DomainNameUtils.canonicalizeDomainName(hostBase.fullyQualifiedHostName);
     }
   }
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -22,6 +22,7 @@ import google.registry.model.annotations.ReportedOn;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
+import google.registry.util.DomainNameUtils;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 
@@ -51,6 +52,11 @@ public class HostResource extends HostBase
     return VKey.create(HostResource.class, getRepoId(), Key.create(this));
   }
 
+  @Override
+  public void beforeSqlSaveOnReplay() {
+    fullyQualifiedHostName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedHostName);
+  }
+
   @Override
   public void beforeDatastoreSaveOnReplay() {
     saveIndexesToDatastore();

core/src/main/java/google/registry/model/poll/PendingActionNotificationResponse.java

@@ -17,6 +17,7 @@ package google.registry.model.poll;
 import com.google.common.annotations.VisibleForTesting;
 import com.googlecode.objectify.annotation.Embed;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppResponse.ResponseData;
 import javax.persistence.Embeddable;
@@ -31,12 +32,13 @@ import org.joda.time.DateTime;
 /** The {@link ResponseData} returned when completing a pending action on a domain. */
 @XmlTransient
 @Embeddable
-public class PendingActionNotificationResponse extends ImmutableObject implements ResponseData {
+public class PendingActionNotificationResponse extends ImmutableObject
+    implements ResponseData, UnsafeSerializable {
 
   /** The inner name type that contains a name and the result boolean. */
   @Embed
   @Embeddable
-  static class NameOrId extends ImmutableObject {
+  static class NameOrId extends ImmutableObject implements UnsafeSerializable {
     @XmlValue
     String value;
 

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -33,6 +33,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ExternalMessagingName;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactResource;
@@ -98,7 +99,7 @@ import org.joda.time.DateTime;
       @javax.persistence.Index(columnList = "eventTime")
     })
 public abstract class PollMessage extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity {
+    implements Buildable, DatastoreAndSqlEntity, TransferServerApproveEntity, UnsafeSerializable {
 
   /** Entity id. */
   @Id

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -72,6 +72,7 @@ import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
@@ -116,7 +117,7 @@ import org.joda.time.DateTime;
     })
 @InCrossTld
 public class Registrar extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity, Jsonifiable {
+    implements Buildable, DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable {
 
   /** Represents the type of a registrar entity. */
   public enum Type {
@@ -404,7 +405,7 @@ public class Registrar extends ImmutableObject
 
   /** A billing account entry for this registrar, consisting of a currency and an account Id. */
   @Embed
-  public static class BillingAccountEntry extends ImmutableObject {
+  public static class BillingAccountEntry extends ImmutableObject implements UnsafeSerializable {
 
     CurrencyUnit currency;
     String accountId;

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -46,6 +46,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registrar.RegistrarContact.RegistrarPocId;
@@ -82,7 +83,7 @@ import javax.persistence.Transient;
 @IdClass(RegistrarPocId.class)
 @InCrossTld
 public class RegistrarContact extends ImmutableObject
-    implements DatastoreAndSqlEntity, Jsonifiable {
+    implements DatastoreAndSqlEntity, Jsonifiable, UnsafeSerializable {
 
   @Parent @Transient Key<Registrar> parent;
 

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -169,7 +169,7 @@ public class ReplicateToDatastoreAction implements Runnable {
       return;
     }
     Optional<Lock> lock =
-        Lock.acquire(
+        Lock.acquireSql(
             this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
     if (!lock.isPresent()) {
       String message = "Can't acquire ReplicateToDatastoreAction lock, aborting.";

core/src/main/java/google/registry/model/reporting/DomainTransactionRecord.java

@@ -22,6 +22,8 @@ import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -45,11 +47,10 @@ import org.joda.time.DateTime;
 @Embed
 @Entity
 public class DomainTransactionRecord extends ImmutableObject
-    implements Buildable, DatastoreAndSqlEntity {
+    implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable {
 
   @Id
   @Ignore
-  @ImmutableObject.DoNotCompare
   @GeneratedValue(strategy = GenerationType.IDENTITY)
   @ImmutableObject.Insignificant
   Long id;
@@ -58,6 +59,14 @@ public class DomainTransactionRecord extends ImmutableObject
   @Column(nullable = false)
   String tld;
 
+  // The following two fields are exposed in this entity to support bulk-loading in Cloud SQL by the
+  // Datastore-SQL validation. They are excluded from equality check since they are not set in
+  // Datastore.
+  // TODO(b/203609782): post migration, decide whether to keep these two fields.
+  @Ignore @ImmutableObject.Insignificant String domainRepoId;
+
+  @Ignore @ImmutableObject.Insignificant Long historyRevisionId;
+
   /**
    * The time this Transaction takes effect (counting grace periods and other nuances).
    *
@@ -174,6 +183,10 @@ public class DomainTransactionRecord extends ImmutableObject
     }
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(domainRepoId, historyRevisionId);
+  }
+
   public DateTime getReportingTime() {
     return reportingTime;
   }

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -31,6 +31,7 @@ import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.Buildable;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactBase;
 import google.registry.model.contact.ContactHistory;
@@ -83,7 +84,8 @@ import org.joda.time.DateTime;
 @Entity
 @MappedSuperclass
 @Access(AccessType.FIELD)
-public class HistoryEntry extends ImmutableObject implements Buildable, DatastoreEntity {
+public class HistoryEntry extends ImmutableObject
+    implements Buildable, DatastoreEntity, UnsafeSerializable {
 
   /** Represents the type of history entry. */
   public enum Type {

core/src/main/java/google/registry/model/server/Lock.java

@@ -32,6 +32,8 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.persistence.transaction.TransactionManager;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
 import java.io.Serializable;
@@ -212,6 +214,47 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
       Duration leaseLength,
       RequestStatusChecker requestStatusChecker,
       boolean checkThreadRunning) {
+    return acquireWithTransactionManager(
+        resourceName, tld, leaseLength, requestStatusChecker, checkThreadRunning, tm());
+  }
+
+  /**
+   * Try to acquire a lock in SQL. Returns absent if it can't be acquired.
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   */
+  public static Optional<Lock> acquireSql(
+      String resourceName,
+      @Nullable String tld,
+      Duration leaseLength,
+      RequestStatusChecker requestStatusChecker,
+      boolean checkThreadRunning) {
+    return acquireWithTransactionManager(
+        resourceName, tld, leaseLength, requestStatusChecker, checkThreadRunning, jpaTm());
+  }
+
+  /** Release the lock. */
+  public void release() {
+    releaseWithTransactionManager(tm());
+  }
+
+  /**
+   * Release the lock from SQL.
+   *
+   * <p>This method exists so that Beam pipelines can acquire / load / release locks.
+   */
+  public void releaseSql() {
+    releaseWithTransactionManager(jpaTm());
+  }
+
+  /** Try to acquire a lock. Returns absent if it can't be acquired. */
+  private static Optional<Lock> acquireWithTransactionManager(
+      String resourceName,
+      @Nullable String tld,
+      Duration leaseLength,
+      RequestStatusChecker requestStatusChecker,
+      boolean checkThreadRunning,
+      TransactionManager transactionManager) {
     String scope = (tld != null) ? tld : GLOBAL;
     String lockId = makeLockId(resourceName, scope);
     // It's important to use transactNew rather than transact, because a Lock can be used to control
@@ -219,11 +262,12 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
     // must be definitively acquired before it is used, even when called inside another transaction.
     Supplier<AcquireResult> lockAcquirer =
         () -> {
-          DateTime now = tm().getTransactionTime();
+          DateTime now = transactionManager.getTransactionTime();
 
           // Checking if an unexpired lock still exists - if so, the lock can't be acquired.
           Lock lock =
-              tm().loadByKeyIfPresent(
+              transactionManager
+                  .loadByKeyIfPresent(
                       VKey.create(
                           Lock.class,
                           new LockId(resourceName, scope),
@@ -249,13 +293,15 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
               create(resourceName, scope, requestStatusChecker.getLogId(), now, leaseLength);
           // Locks are not parented under an EntityGroupRoot (so as to avoid write
           // contention) and don't need to be backed up.
-          tm().putIgnoringReadOnly(newLock);
+          transactionManager.putIgnoringReadOnly(newLock);
 
           return AcquireResult.create(now, lock, newLock, lockState);
         };
     // In ofy, backup is determined per-action, but in SQL it's determined per-transaction
     AcquireResult acquireResult =
-        tm().isOfy() ? tm().transactNew(lockAcquirer) : jpaTm().transactWithoutBackup(lockAcquirer);
+        transactionManager.isOfy()
+            ? transactionManager.transactNew(lockAcquirer)
+            : ((JpaTransactionManager) transactionManager).transactWithoutBackup(lockAcquirer);
 
     logAcquireResult(acquireResult);
     lockMetrics.recordAcquire(resourceName, scope, acquireResult.lockState());
@@ -263,7 +309,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
   }
 
   /** Release the lock. */
-  public void release() {
+  private void releaseWithTransactionManager(TransactionManager transactionManager) {
     // Just use the default clock because we aren't actually doing anything that will use the clock.
     Supplier<Void> lockReleaser =
         () -> {
@@ -274,15 +320,17 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
           VKey<Lock> key =
               VKey.create(
                   Lock.class, new LockId(resourceName, tld), Key.create(Lock.class, lockId));
-          Lock loadedLock = tm().loadByKeyIfPresent(key).orElse(null);
+          Lock loadedLock = transactionManager.loadByKeyIfPresent(key).orElse(null);
           if (Lock.this.equals(loadedLock)) {
             // Use deleteIgnoringReadOnly() so that we don't create a commit log entry for deleting
             // the lock.
             logger.atInfo().log("Deleting lock: %s", lockId);
-            tm().deleteIgnoringReadOnly(key);
+            transactionManager.deleteIgnoringReadOnly(key);
 
             lockMetrics.recordRelease(
-                resourceName, tld, new Duration(acquiredTime, tm().getTransactionTime()));
+                resourceName,
+                tld,
+                new Duration(acquiredTime, transactionManager.getTransactionTime()));
           } else {
             logger.atSevere().log(
                 "The lock we acquired was transferred to someone else before we"
@@ -294,11 +342,12 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
           }
           return null;
         };
+
     // In ofy, backup is determined per-action, but in SQL it's determined per-transaction
-    if (tm().isOfy()) {
-      tm().transact(lockReleaser);
+    if (transactionManager.isOfy()) {
+      transactionManager.transact(lockReleaser);
     } else {
-      jpaTm().transactWithoutBackup(lockReleaser);
+      ((JpaTransactionManager) transactionManager).transactWithoutBackup(lockReleaser);
     }
   }
 

core/src/main/java/google/registry/model/tld/Registry.java

@@ -52,6 +52,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
@@ -88,7 +89,8 @@ import org.joda.time.Duration;
 @Entity
 @javax.persistence.Entity(name = "Tld")
 @InCrossTld
-public class Registry extends ImmutableObject implements Buildable, DatastoreAndSqlEntity {
+public class Registry extends ImmutableObject
+    implements Buildable, DatastoreAndSqlEntity, UnsafeSerializable {
 
   @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
@@ -308,7 +310,7 @@ public class Registry extends ImmutableObject implements Buildable, DatastoreAnd
    * <p>There must be at least one entry in this set.
    *
    * <p>All entries of this list must be valid keys for the map of {@code DnsWriter}s injected by
-   * <code>@Inject Map<String, DnsWriter></code>
+   * {@code @Inject Map<String, DnsWriter>}
    */
   @Column(nullable = false)
   Set<String> dnsWriters;

core/src/main/java/google/registry/model/tmch/ClaimsEntry.java

@@ -15,7 +15,7 @@
 package google.registry.model.tmch;
 
 import google.registry.model.ImmutableObject;
-import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import java.io.Serializable;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -30,7 +30,7 @@ import javax.persistence.Id;
  * work.
  */
 @Entity(name = "ClaimsEntry")
-class ClaimsEntry extends ImmutableObject implements NonReplicatedEntity, Serializable {
+class ClaimsEntry extends ImmutableObject implements SqlOnlyEntity, Serializable {
   @Id private Long revisionId;
   @Id private String domainLabel;
 

core/src/main/java/google/registry/model/transfer/BaseTransferObject.java

@@ -16,6 +16,7 @@ package google.registry.model.transfer;
 
 import google.registry.model.Buildable.GenericBuilder;
 import google.registry.model.ImmutableObject;
+import google.registry.model.UnsafeSerializable;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
@@ -27,7 +28,7 @@ import org.joda.time.DateTime;
 /** Fields common to {@link TransferData} and {@link TransferResponse}. */
 @XmlTransient
 @MappedSuperclass
-public abstract class BaseTransferObject extends ImmutableObject {
+public abstract class BaseTransferObject extends ImmutableObject implements UnsafeSerializable {
   /**
    * The status of the current or last transfer. Can be null if never transferred. Note that we
    * leave IgnoreSave off this field so that we can ensure that TransferData loaded from Objectify

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -33,6 +33,7 @@ import google.registry.batch.ResaveAllEppResourcesAction;
 import google.registry.batch.ResaveEntityAction;
 import google.registry.batch.SendExpiringCertificateNotificationEmailAction;
 import google.registry.batch.WipeOutCloudSqlAction;
+import google.registry.batch.WipeOutContactHistoryPiiAction;
 import google.registry.batch.WipeoutDatastoreAction;
 import google.registry.cron.CommitLogFanoutAction;
 import google.registry.cron.CronModule;
@@ -219,6 +220,8 @@ interface BackendRequestComponent {
 
   WipeoutDatastoreAction wipeoutDatastoreAction();
 
+  WipeOutContactHistoryPiiAction wipeOutContactHistoryPiiAction();
+
   @Subcomponent.Builder
   abstract class Builder implements RequestComponentBuilder<BackendRequestComponent> {
 

core/src/main/java/google/registry/persistence/BulkQueryJpaFactory.java

@@ -0,0 +1,65 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.util.Clock;
+import java.util.List;
+import javax.persistence.EntityManagerFactory;
+import org.hibernate.jpa.boot.internal.ParsedPersistenceXmlDescriptor;
+import org.hibernate.jpa.boot.spi.Bootstrap;
+
+/**
+ * Defines factory method for instantiating the bulk-query optimized {@link JpaTransactionManager}.
+ */
+public final class BulkQueryJpaFactory {
+
+  private BulkQueryJpaFactory() {}
+
+  static EntityManagerFactory createBulkQueryEntityManagerFactory(
+      ImmutableMap<String, String> cloudSqlConfigs) {
+    ParsedPersistenceXmlDescriptor descriptor =
+        PersistenceXmlUtility.getParsedPersistenceXmlDescriptor();
+
+    List<String> updatedManagedClasses =
+        Streams.concat(
+                descriptor.getManagedClassNames().stream(),
+                BulkQueryEntities.JPA_ENTITIES_NEW.stream())
+            .map(
+                name -> {
+                  if (BulkQueryEntities.JPA_ENTITIES_REPLACEMENTS.containsKey(name)) {
+                    return BulkQueryEntities.JPA_ENTITIES_REPLACEMENTS.get(name);
+                  }
+                  return name;
+                })
+            .collect(ImmutableList.toImmutableList());
+
+    descriptor.getManagedClassNames().clear();
+    descriptor.getManagedClassNames().addAll(updatedManagedClasses);
+
+    return Bootstrap.getEntityManagerFactoryBuilder(descriptor, cloudSqlConfigs).build();
+  }
+
+  public static JpaTransactionManager createBulkQueryJpaTransactionManager(
+      ImmutableMap<String, String> cloudSqlConfigs, Clock clock) {
+    return new JpaTransactionManagerImpl(
+        createBulkQueryEntityManagerFactory(cloudSqlConfigs), clock);
+  }
+}

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -21,6 +21,7 @@ import static google.registry.config.RegistryConfig.getHibernateHikariIdleTimeou
 import static google.registry.config.RegistryConfig.getHibernateHikariMaximumPoolSize;
 import static google.registry.config.RegistryConfig.getHibernateHikariMinimumIdle;
 import static google.registry.config.RegistryConfig.getHibernateLogSqlQueries;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.api.client.auth.oauth2.Credential;
 import com.google.common.annotations.VisibleForTesting;
@@ -34,6 +35,7 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.persistence.transaction.TransactionManager;
 import google.registry.privileges.secretmanager.SqlCredential;
 import google.registry.privileges.secretmanager.SqlCredentialStore;
 import google.registry.privileges.secretmanager.SqlUser;
@@ -150,13 +152,36 @@ public abstract class PersistenceModule {
   @Singleton
   @BeamPipelineCloudSqlConfigs
   static ImmutableMap<String, String> provideBeamPipelineCloudSqlConfigs(
-      @Config("beamCloudSqlJdbcUrl") String jdbcUrl,
-      @Config("beamCloudSqlInstanceConnectionName") String instanceConnectionName,
-      @DefaultHibernateConfigs ImmutableMap<String, String> defaultConfigs,
+      SqlCredentialStore credentialStore,
+      @Config("instanceConnectionNameOverride")
+          Optional<Provider<String>> instanceConnectionNameOverride,
       @Config("beamIsolationOverride")
-          Optional<Provider<TransactionIsolationLevel>> isolationOverride) {
-    return createPartialSqlConfigs(
-        jdbcUrl, instanceConnectionName, defaultConfigs, isolationOverride);
+          Optional<Provider<TransactionIsolationLevel>> isolationOverride,
+      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs) {
+    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
+    // TODO(b/175700623): make sql username configurable from config file.
+    SqlCredential credential = credentialStore.getCredential(new RobotUser(RobotId.NOMULUS));
+    overrides.put(Environment.USER, credential.login());
+    overrides.put(Environment.PASS, credential.password());
+    // Override the default minimum which is tuned for the Registry server. A worker VM should
+    // release all connections if it no longer interacts with the database.
+    overrides.put(HIKARI_MINIMUM_IDLE, "0");
+    /**
+     * Disable Hikari's maxPoolSize limit check by setting it to an absurdly large number. The
+     * effective (and desirable) limit is the number of pipeline threads on the pipeline worker,
+     * which can be configured using pipeline options. See {@link RegistryPipelineOptions} for more
+     * information.
+     */
+    overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(Integer.MAX_VALUE));
+    instanceConnectionNameOverride
+        .map(Provider::get)
+        .ifPresent(
+            instanceConnectionName ->
+                overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName));
+    isolationOverride
+        .map(Provider::get)
+        .ifPresent(isolation -> overrides.put(Environment.ISOLATION, isolation.name()));
+    return ImmutableMap.copyOf(overrides);
   }
 
   @VisibleForTesting
@@ -206,6 +231,12 @@ public abstract class PersistenceModule {
     }
   }
 
+  @Provides
+  @Singleton
+  static TransactionManager provideTransactionManager() {
+    return tm();
+  }
+
   @Provides
   @Singleton
   @AppEngineJpaTm
@@ -222,37 +253,17 @@ public abstract class PersistenceModule {
   @Singleton
   @BeamJpaTm
   static JpaTransactionManager provideBeamJpaTm(
-      SqlCredentialStore credentialStore,
-      @Config("instanceConnectionNameOverride")
-          Optional<Provider<String>> instanceConnectionNameOverride,
-      @Config("beamIsolationOverride")
-          Optional<Provider<TransactionIsolationLevel>> isolationOverride,
-      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
-      Clock clock) {
-    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
-    // TODO(b/175700623): make sql username configurable from config file.
-    SqlCredential credential = credentialStore.getCredential(new RobotUser(RobotId.NOMULUS));
-    overrides.put(Environment.USER, credential.login());
-    overrides.put(Environment.PASS, credential.password());
-    // Override the default minimum which is tuned for the Registry server. A worker VM should
-    // release all connections if it no longer interacts with the database.
-    overrides.put(HIKARI_MINIMUM_IDLE, "0");
-    /**
-     * Disable Hikari's maxPoolSize limit check by setting it to an absurdly large number. The
-     * effective (and desirable) limit is the number of pipeline threads on the pipeline worker,
-     * which can be configured using pipeline options. See {@link RegistryPipelineOptions} for more
-     * information.
-     */
-    overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(Integer.MAX_VALUE));
-    instanceConnectionNameOverride
-        .map(Provider::get)
-        .ifPresent(
-            instanceConnectionName ->
-                overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, instanceConnectionName));
-    isolationOverride
-        .map(Provider::get)
-        .ifPresent(isolation -> overrides.put(Environment.ISOLATION, isolation.name()));
-    return new JpaTransactionManagerImpl(create(overrides), clock);
+      @BeamPipelineCloudSqlConfigs ImmutableMap<String, String> beamCloudSqlConfigs, Clock clock) {
+    return new JpaTransactionManagerImpl(create(beamCloudSqlConfigs), clock);
+  }
+
+  @Provides
+  @Singleton
+  @BeamBulkQueryJpaTm
+  static JpaTransactionManager provideBeamBulkQueryJpaTm(
+      @BeamPipelineCloudSqlConfigs ImmutableMap<String, String> beamCloudSqlConfigs, Clock clock) {
+    return new JpaTransactionManagerImpl(
+        BulkQueryJpaFactory.createBulkQueryEntityManagerFactory(beamCloudSqlConfigs), clock);
   }
 
   @Provides
@@ -338,6 +349,17 @@ public abstract class PersistenceModule {
     }
   }
 
+  /** Types of {@link JpaTransactionManager JpaTransactionManagers}. */
+  public enum JpaTransactionManagerType {
+    /** The regular {@link JpaTransactionManager} for general use. */
+    REGULAR,
+    /**
+     * The {@link JpaTransactionManager} optimized for bulk loading multi-level JPA entities. Please
+     * see {@link google.registry.model.bulkquery.BulkQueryEntities} for more information.
+     */
+    BULK_QUERY
+  }
+
   /** Dagger qualifier for JDBC {@link Connection} with schema management privilege. */
   @Qualifier
   @Documented
@@ -349,11 +371,18 @@ public abstract class PersistenceModule {
   @interface AppEngineJpaTm {}
 
   /** Dagger qualifier for {@link JpaTransactionManager} used inside BEAM pipelines. */
-  // Note: @SocketFactoryJpaTm will be phased out in favor of this qualifier.
   @Qualifier
   @Documented
   public @interface BeamJpaTm {}
 
+  /**
+   * Dagger qualifier for {@link JpaTransactionManager} that uses an alternative entity model for
+   * faster bulk queries.
+   */
+  @Qualifier
+  @Documented
+  public @interface BeamBulkQueryJpaTm {}
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for Nomulus tool. */
   @Qualifier
   @Documented

core/src/main/java/google/registry/persistence/VKey.java

@@ -18,10 +18,13 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.ImmutableObject;
 import google.registry.model.translators.VKeyTranslatorFactory;
+import google.registry.util.SerializeUtils;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.Nullable;
@@ -36,6 +39,15 @@ public class VKey<T> extends ImmutableObject implements Serializable {
 
   private static final long serialVersionUID = -5291472863840231240L;
 
+  // Info that's stored in in vkey string generated via stringify().
+  private static final String SQL_LOOKUP_KEY = "sql";
+  private static final String OFY_LOOKUP_KEY = "ofy";
+  private static final String CLASS_TYPE = "kind";
+
+  // Web safe delimiters that won't be used in base 64.
+  private static final String KV_SEPARATOR = ":";
+  private static final String DELIMITER = "@";
+
   // The SQL key for the referenced entity.
   Serializable sqlKey;
 
@@ -114,6 +126,47 @@ public class VKey<T> extends ImmutableObject implements Serializable {
     return new VKey<T>(kind, Key.create(kind, name), name);
   }
 
+  /**
+   * Constructs a {@link VKey} from the string representation of a vkey.
+   *
+   * <p>There are two types of string representations: 1) existing ofy key string handled by
+   * fromWebsafeKey() and 2) string encoded via stringify() where @ separates the substrings and
+   * each of the substrings contains a look up key, ":", and its corresponding value. The key info
+   * is encoded via Base64. The string begins with "kind:" and it must contains at least ofy key or
+   * sql key.
+   *
+   * <p>Example of a Vkey string by fromWebsafeKey(): "agR0ZXN0chYLEgpEb21haW5CYXNlIgZST0lELTEM"
+   *
+   * <p>Example of a vkey string by stringify(): "google.registry.testing.TestObject@sql:rO0ABX" +
+   * "QAA2Zvbw@ofy:agR0ZXN0cjELEg9FbnRpdHlHcm91cFJvb3QiCWNyb3NzLXRsZAwLEgpUZXN0T2JqZWN0IgNmb28M",
+   * where sql key and ofy key are values are encoded in Base64.
+   */
+  public static <T> VKey<T> create(String keyString) throws Exception {
+    if (!keyString.startsWith(CLASS_TYPE + KV_SEPARATOR)) {
+      // to handle the existing ofy key string
+      return fromWebsafeKey(keyString);
+    } else {
+      ImmutableMap<String, String> kvs =
+          ImmutableMap.copyOf(
+              Splitter.on(DELIMITER).withKeyValueSeparator(KV_SEPARATOR).split(keyString));
+      Class classType = Class.forName(kvs.get(CLASS_TYPE));
+
+      if (kvs.containsKey(SQL_LOOKUP_KEY) && kvs.containsKey(OFY_LOOKUP_KEY)) {
+        return VKey.create(
+            classType,
+            SerializeUtils.parse(Serializable.class, kvs.get(SQL_LOOKUP_KEY)),
+            Key.create(kvs.get(OFY_LOOKUP_KEY)));
+      } else if (kvs.containsKey(SQL_LOOKUP_KEY)) {
+        return VKey.createSql(
+            classType, SerializeUtils.parse(Serializable.class, kvs.get(SQL_LOOKUP_KEY)));
+      } else if (kvs.containsKey(OFY_LOOKUP_KEY)) {
+        return VKey.createOfy(classType, Key.create(kvs.get(OFY_LOOKUP_KEY)));
+      } else {
+        throw new IllegalArgumentException(String.format("Cannot parse key string: %s", keyString));
+      }
+    }
+  }
+
   /**
    * Returns a clone with an ofy key restored from {@code ancestors}.
    *
@@ -233,4 +286,29 @@ public class VKey<T> extends ImmutableObject implements Serializable {
   public static <T> VKey<T> fromWebsafeKey(String ofyKeyRepr) {
     return from(Key.create(ofyKeyRepr));
   }
+
+  /**
+   * Constructs the string representation of a {@link VKey}.
+   *
+   * <p>The string representation of a vkey contains its type, and sql key or ofy key, or both. Each
+   * of the keys is first serialized into a byte array then encoded via Base64 into a web safe
+   * string.
+   *
+   * <p>The string representation of a vkey contains key values pairs separated by delimiter "@".
+   * Another delimiter ":" is put in between each key and value. The following is the complete
+   * format of the string: "kind:class_name@sql:encoded_sqlKey@ofy:encoded_ofyKey", where kind is
+   * required. The string representation may contain an encoded ofy key, or an encoded sql key, or
+   * both.
+   */
+  public String stringify() {
+    // class type is required to create a vkey
+    String key = CLASS_TYPE + KV_SEPARATOR + getKind().getName();
+    if (maybeGetSqlKey().isPresent()) {
+      key += DELIMITER + SQL_LOOKUP_KEY + KV_SEPARATOR + SerializeUtils.stringify(getSqlKey());
+    }
+    if (maybeGetOfyKey().isPresent()) {
+      key += DELIMITER + OFY_LOOKUP_KEY + KV_SEPARATOR + getOfyKey().getString();
+    }
+    return key;
+  }
 }

core/src/main/java/google/registry/persistence/converter/TimedTransitionPropertyConverterBase.java

@@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.persistence.converter.StringMapDescriptor.StringMap;
+import java.io.Serializable;
 import java.util.Map;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
@@ -30,7 +31,8 @@ import org.joda.time.DateTime;
  * Base JPA converter for {@link TimedTransitionProperty} objects that are stored in a column with
  * data type of hstore in the database.
  */
-public abstract class TimedTransitionPropertyConverterBase<K, V extends TimedTransition<K>>
+public abstract class TimedTransitionPropertyConverterBase<
+        K extends Serializable, V extends TimedTransition<K>>
     implements AttributeConverter<TimedTransitionProperty<K, V>, StringMap> {
 
   abstract Map.Entry<String, String> convertToDatabaseMapEntry(Map.Entry<DateTime, V> entry);

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManager.java

@@ -24,7 +24,33 @@ import javax.persistence.criteria.CriteriaQuery;
 /** Sub-interface of {@link TransactionManager} which defines JPA related methods. */
 public interface JpaTransactionManager extends TransactionManager {
 
-  /** Returns the {@link EntityManager} for the current request. */
+  /**
+   * Returns a long-lived {@link EntityManager} not bound to a particular transaction.
+   *
+   * <p>Caller is responsible for closing the returned instance.
+   */
+  EntityManager getStandaloneEntityManager();
+
+  /**
+   * Specifies a database snapshot exported by another transaction to use in the current
+   * transaction.
+   *
+   * <p>This is a Postgresql-specific feature. This method must be called before any other SQL
+   * commands in a transaction.
+   *
+   * <p>To support large queries, transaction isolation level is fixed at the REPEATABLE_READ to
+   * avoid exhausting predicate locks at the SERIALIZABLE level.
+   *
+   * @see google.registry.beam.common.DatabaseSnapshot
+   */
+  // TODO(b/193662898): vendor-independent support for richer transaction semantics.
+  JpaTransactionManager setDatabaseSnapshot(String snapshotId);
+
+  /**
+   * Returns the {@link EntityManager} for the current request.
+   *
+   * <p>The returned instance is closed when the current transaction completes.
+   */
   EntityManager getEntityManager();
 
   /**

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -98,14 +98,16 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   // EntityManagerFactory is thread safe.
   private final EntityManagerFactory emf;
   private final Clock clock;
+
   // TODO(b/177588434): Investigate alternatives for managing transaction information. ThreadLocal
   // adds an unnecessary restriction that each request has to be processed by one thread
   // synchronously.
-  private final ThreadLocal<TransactionInfo> transactionInfo =
+  private static final ThreadLocal<TransactionInfo> transactionInfo =
       ThreadLocal.withInitial(TransactionInfo::new);
+
   // If this value is present, use it to determine whether or not to replay SQL transactions to
   // Datastore, rather than using the schedule stored in Datastore.
-  private static ThreadLocal<Optional<Boolean>> replaySqlToDatastoreOverrideForTest =
+  private static final ThreadLocal<Optional<Boolean>> replaySqlToDatastoreOverrideForTest =
       ThreadLocal.withInitial(Optional::empty);
 
   public JpaTransactionManagerImpl(EntityManagerFactory emf, Clock clock) {
@@ -118,6 +120,11 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     emf.close();
   }
 
+  @Override
+  public EntityManager getStandaloneEntityManager() {
+    return emf.createEntityManager();
+  }
+
   @Override
   public EntityManager getEntityManager() {
     EntityManager entityManager = transactionInfo.get().entityManager;
@@ -129,6 +136,22 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     return entityManager;
   }
 
+  @Override
+  public JpaTransactionManager setDatabaseSnapshot(String snapshotId) {
+    // Postgresql-specific: 'set transaction' command must be called inside a transaction
+    assertInTransaction();
+
+    EntityManager entityManager = getEntityManager();
+    // Isolation is hardcoded to REPEATABLE READ, as specified by parent's Javadoc.
+    entityManager
+        .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
+        .executeUpdate();
+    entityManager
+        .createNativeQuery(String.format("SET TRANSACTION SNAPSHOT '%s'", snapshotId))
+        .executeUpdate();
+    return this;
+  }
+
   @Override
   public <T> TypedQuery<T> query(String sqlString, Class<T> resultClass) {
     return new DetachingTypedQuery<>(getEntityManager().createQuery(sqlString, resultClass));

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -132,13 +132,16 @@ public class TransactionManagerFactory {
   /**
    * Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}.
    *
+   * <p>DO NOT CALL THIS DIRECTLY IF POSSIBLE. Strongly prefer the use of <code>TmOverrideExtension
+   * </code> in test code instead.
+   *
    * <p>Used when overriding the per-test transaction manager for dual-database tests. Should be
    * matched with a corresponding invocation of {@link #removeTmOverrideForTest()} either at the end
    * of the test or in an <code>@AfterEach</code> handler.
    */
   @VisibleForTesting
-  public static void setTmForTest(TransactionManager newTm) {
-    tmForTest = Optional.of(newTm);
+  public static void setTmOverrideForTest(TransactionManager newTmOverride) {
+    tmForTest = Optional.of(newTmOverride);
   }
 
   /** Resets the overridden transaction manager post-test. */
@@ -152,10 +155,10 @@ public class TransactionManagerFactory {
     }
   }
 
-  /** Thrown when a write is attempted when the DB is in read-only mode. */
+  /** Registry is currently undergoing maintenance and is in read-only mode. */
   public static class ReadOnlyModeException extends IllegalStateException {
-    public ReadOnlyModeException() {
-      super("Registry is currently in read-only mode");
+    ReadOnlyModeException() {
+      super("Registry is currently undergoing maintenance and is in read-only mode");
     }
   }
 }

core/src/main/java/google/registry/rde/BrdaCopyAction.java

@@ -31,6 +31,7 @@ import google.registry.request.auth.Auth;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.bouncycastle.openpgp.PGPKeyPair;
 import org.bouncycastle.openpgp.PGPPrivateKey;
@@ -60,7 +61,7 @@ import org.joda.time.DateTime;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public final class BrdaCopyAction implements Runnable {
 
-  static final String PATH = "/_dr/task/brdaCopy";
+  public static final String PATH = "/_dr/task/brdaCopy";
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -69,6 +70,7 @@ public final class BrdaCopyAction implements Runnable {
   @Inject @Config("rdeBucket") String stagingBucket;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
   @Inject @Parameter(RdeModule.PARAM_WATERMARK) DateTime watermark;
+  @Inject @Parameter(RdeModule.PARAM_PREFIX) Optional<String> prefix;
   @Inject @Key("brdaReceiverKey") PGPPublicKey receiverKey;
   @Inject @Key("brdaSigningKey") PGPKeyPair signingKey;
   @Inject @Key("rdeStagingDecryptionKey") PGPPrivateKey stagingDecryptionKey;
@@ -84,11 +86,12 @@ public final class BrdaCopyAction implements Runnable {
   }
 
   private void copyAsRyde() throws IOException {
-    String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
-    BlobId xmlFilename = BlobId.of(stagingBucket, prefix + ".xml.ghostryde");
-    BlobId xmlLengthFilename = BlobId.of(stagingBucket, prefix + ".xml.length");
-    BlobId rydeFile = BlobId.of(brdaBucket, prefix + ".ryde");
-    BlobId sigFile = BlobId.of(brdaBucket, prefix + ".sig");
+    String nameWithoutPrefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
+    String name = prefix.orElse("") + nameWithoutPrefix;
+    BlobId xmlFilename = BlobId.of(stagingBucket, name + ".xml.ghostryde");
+    BlobId xmlLengthFilename = BlobId.of(stagingBucket, name + ".xml.length");
+    BlobId rydeFile = BlobId.of(brdaBucket, nameWithoutPrefix + ".ryde");
+    BlobId sigFile = BlobId.of(brdaBucket, nameWithoutPrefix + ".sig");
 
     long xmlLength = readXmlLength(xmlLengthFilename);
 
@@ -97,11 +100,12 @@ public final class BrdaCopyAction implements Runnable {
         InputStream ghostrydeDecoder = Ghostryde.decoder(gcsInput, stagingDecryptionKey);
         OutputStream rydeOut = gcsUtils.openOutputStream(rydeFile);
         OutputStream sigOut = gcsUtils.openOutputStream(sigFile);
-        RydeEncoder rydeEncoder = new RydeEncoder.Builder()
-            .setRydeOutput(rydeOut, receiverKey)
-            .setSignatureOutput(sigOut, signingKey)
-            .setFileMetadata(prefix, xmlLength, watermark)
-            .build()) {
+        RydeEncoder rydeEncoder =
+            new RydeEncoder.Builder()
+                .setRydeOutput(rydeOut, receiverKey)
+                .setSignatureOutput(sigOut, signingKey)
+                .setFileMetadata(nameWithoutPrefix, xmlLength, watermark)
+                .build()) {
       ByteStreams.copy(ghostrydeDecoder, rydeEncoder);
     }
   }

core/src/main/java/google/registry/rde/RdeModule.java

@@ -49,6 +49,10 @@ public abstract class RdeModule {
   public static final String PARAM_MODE = "mode";
   public static final String PARAM_REVISION = "revision";
   public static final String PARAM_LENIENT = "lenient";
+  public static final String PARAM_PREFIX = "prefix";
+  public static final String RDE_UPLOAD_QUEUE = "rde-upload";
+  public static final String RDE_REPORT_QUEUE = "rde-report";
+  public static final String BRDA_QUEUE = "brda";
 
   @Provides
   @Parameter(PARAM_WATERMARK)
@@ -92,10 +96,11 @@ public abstract class RdeModule {
     return extractBooleanParameter(req, PARAM_LENIENT);
   }
 
+  // TODO (jianglai): Make it a required parameter once we migrate to Cloud SQL.
   @Provides
-  @Named("brda")
-  static Queue provideQueueBrda() {
-    return getQueue("brda");
+  @Parameter(PARAM_PREFIX)
+  static Optional<String> providePrefix(HttpServletRequest req) {
+    return extractOptionalParameter(req, PARAM_PREFIX);
   }
 
   @Provides

core/src/main/java/google/registry/rde/RdeReportAction.java

@@ -68,6 +68,7 @@ public final class RdeReportAction implements Runnable, EscrowTask {
   @Inject Response response;
   @Inject RdeReporter reporter;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
+  @Inject @Parameter(RdeModule.PARAM_PREFIX) Optional<String> prefix;
   @Inject @Config("rdeBucket") String bucket;
   @Inject @Config("rdeInterval") Duration interval;
   @Inject @Config("rdeReportLockTimeout") Duration timeout;
@@ -96,8 +97,9 @@ public final class RdeReportAction implements Runnable, EscrowTask {
         RdeRevision.getCurrentRevision(tld, watermark, FULL)
             .orElseThrow(
                 () -> new IllegalStateException("RdeRevision was not set on generated deposit"));
-    String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
-    BlobId reportFilename = BlobId.of(bucket, prefix + "-report.xml.ghostryde");
+    String name =
+        prefix.orElse("") + RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
+    BlobId reportFilename = BlobId.of(bucket, name + "-report.xml.ghostryde");
     verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
     reporter.send(readReportFromGcs(reportFilename));
     response.setContentType(PLAIN_TEXT_UTF_8);

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -14,20 +14,33 @@
 
 package google.registry.rde;
 
+import static google.registry.beam.BeamUtils.createJobName;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.xml.ValidationMode.LENIENT;
 import static google.registry.xml.ValidationMode.STRICT;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
 
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSetMultimap;
 import com.google.common.collect.Multimaps;
 import com.google.common.flogger.FluentLogger;
+import com.google.common.io.BaseEncoding;
+import google.registry.beam.rde.RdePipeline;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.gcs.GcsUtils;
+import google.registry.keyring.api.KeyModule.Key;
 import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.mapreduce.inputs.NullInput;
@@ -47,6 +60,7 @@ import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.xml.ValidationMode;
+import java.io.IOException;
 import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
@@ -201,6 +215,7 @@ public final class RdeStagingAction implements Runnable {
 
   public static final String PATH = "/_dr/task/rdeStaging";
 
+  private static final String PIPELINE_NAME = "rde_pipeline";
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   @Inject Clock clock;
@@ -209,7 +224,11 @@ public final class RdeStagingAction implements Runnable {
   @Inject Response response;
   @Inject GcsUtils gcsUtils;
   @Inject MapreduceRunner mrRunner;
+  @Inject @Config("projectId") String projectId;
+  @Inject @Config("defaultJobRegion") String jobRegion;
   @Inject @Config("transactionCooldown") Duration transactionCooldown;
+  @Inject @Config("beamStagingBucketUrl") String stagingBucketUrl;
+  @Inject @Config("rdeBucket") String rdeBucket;
   @Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual;
   @Inject @Parameter(RdeModule.PARAM_DIRECTORY) Optional<String> directory;
   @Inject @Parameter(RdeModule.PARAM_MODE) ImmutableSet<String> modeStrings;
@@ -217,7 +236,8 @@ public final class RdeStagingAction implements Runnable {
   @Inject @Parameter(RdeModule.PARAM_WATERMARKS) ImmutableSet<DateTime> watermarks;
   @Inject @Parameter(RdeModule.PARAM_REVISION) Optional<Integer> revision;
   @Inject @Parameter(RdeModule.PARAM_LENIENT) boolean lenient;
-
+  @Inject @Key("rdeStagingEncryptionKey") byte[] stagingKeyBytes;
+  @Inject Dataflow dataflow;
   @Inject RdeStagingAction() {}
 
   @Override
@@ -228,27 +248,66 @@ public final class RdeStagingAction implements Runnable {
       String message = "Nothing needs to be deposited.";
       logger.atInfo().log(message);
       response.setStatus(SC_NO_CONTENT);
-      response.setPayload(message);
+      // No need to set payload as HTTP 204 response status code does not allow a payload.
       return;
     }
     for (PendingDeposit pending : pendings.values()) {
       logger.atInfo().log("Pending deposit: %s", pending);
     }
     ValidationMode validationMode = lenient ? LENIENT : STRICT;
-    RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings);
-    RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils);
-
-    mrRunner
-        .setJobName("Stage escrow deposits for all TLDs")
-        .setModuleName("backend")
-        .setDefaultReduceShards(pendings.size())
-        .runMapreduce(
-            mapper,
-            reducer,
-            ImmutableList.of(
-                // Add an extra shard that maps over a null resource. See the mapper code for why.
-                new NullInput<>(), EppResourceInputs.createEntityInput(EppResource.class)))
-        .sendLinkToMapreduceConsole(response);
+    if (tm().isOfy()) {
+      RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings);
+      RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils);
+      mrRunner
+          .setJobName("Stage escrow deposits for all TLDs")
+          .setModuleName("backend")
+          .setDefaultReduceShards(pendings.size())
+          .runMapreduce(
+              mapper,
+              reducer,
+              ImmutableList.of(
+                  // Add an extra shard that maps over a null resource. See the mapper code for why.
+                  new NullInput<>(), EppResourceInputs.createEntityInput(EppResource.class)))
+          .sendLinkToMapreduceConsole(response);
+    } else {
+      try {
+        LaunchFlexTemplateParameter parameter =
+            new LaunchFlexTemplateParameter()
+                .setJobName(createJobName("rde", clock))
+                .setContainerSpecGcsPath(
+                    String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+                .setParameters(
+                    ImmutableMap.of(
+                        "pendings",
+                        RdePipeline.encodePendings(pendings),
+                        "validationMode",
+                        validationMode.name(),
+                        "rdeStagingBucket",
+                        rdeBucket,
+                        "stagingKey",
+                        BaseEncoding.base64Url().omitPadding().encode(stagingKeyBytes),
+                        "registryEnvironment",
+                        RegistryEnvironment.get().name()));
+        LaunchFlexTemplateResponse launchResponse =
+            dataflow
+                .projects()
+                .locations()
+                .flexTemplates()
+                .launch(
+                    projectId,
+                    jobRegion,
+                    new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
+                .execute();
+        logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
+        response.setStatus(SC_OK);
+        response.setPayload(
+            String.format("Launched RDE pipeline: %s", launchResponse.getJob().getId()));
+      } catch (IOException e) {
+        logger.atWarning().withCause(e).log("Pipeline Launch failed");
+        response.setStatus(SC_INTERNAL_SERVER_ERROR);
+        response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
+      }
+    }
   }
 
   private ImmutableSetMultimap<String, PendingDeposit> getStandardPendingDeposits() {

core/src/main/java/google/registry/rde/RdeUploadAction.java

@@ -14,7 +14,6 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static com.google.common.base.Verify.verify;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.jcraft.jsch.ChannelSftp.OVERWRITE;
@@ -24,14 +23,15 @@ import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.model.rde.RdeMode.FULL;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static google.registry.rde.RdeModule.RDE_REPORT_QUEUE;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static java.util.Arrays.asList;
 
-import com.google.appengine.api.taskqueue.Queue;
 import com.google.cloud.storage.BlobId;
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.HashMultimap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteStreams;
 import com.jcraft.jsch.JSch;
@@ -49,14 +49,15 @@ import google.registry.model.tld.Registry;
 import google.registry.rde.EscrowTaskRunner.EscrowTask;
 import google.registry.rde.JSchSshSession.JSchSshSessionFactory;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.HttpException.NoContentException;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import google.registry.util.Retrier;
-import google.registry.util.TaskQueueUtils;
 import google.registry.util.TeeOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
@@ -66,7 +67,6 @@ import java.io.OutputStream;
 import java.net.URI;
 import java.util.Optional;
 import javax.inject.Inject;
-import javax.inject.Named;
 import org.bouncycastle.openpgp.PGPKeyPair;
 import org.bouncycastle.openpgp.PGPPrivateKey;
 import org.bouncycastle.openpgp.PGPPublicKey;
@@ -90,7 +90,7 @@ import org.joda.time.Duration;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public final class RdeUploadAction implements Runnable, EscrowTask {
 
-  static final String PATH = "/_dr/task/rdeUpload";
+  public static final String PATH = "/_dr/task/rdeUpload";
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -109,9 +109,10 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
   @Inject JSchSshSessionFactory jschSshSessionFactory;
   @Inject Response response;
   @Inject SftpProgressMonitor sftpProgressMonitor;
-  @Inject TaskQueueUtils taskQueueUtils;
+  @Inject CloudTasksUtils cloudTasksUtils;
   @Inject Retrier retrier;
   @Inject @Parameter(RequestParameters.PARAM_TLD) String tld;
+  @Inject @Parameter(RdeModule.PARAM_PREFIX) Optional<String> prefix;
   @Inject @Config("rdeBucket") String bucket;
   @Inject @Config("rdeInterval") Duration interval;
   @Inject @Config("rdeUploadLockTimeout") Duration timeout;
@@ -120,15 +121,21 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
   @Inject @Key("rdeReceiverKey") PGPPublicKey receiverKey;
   @Inject @Key("rdeSigningKey") PGPKeyPair signingKey;
   @Inject @Key("rdeStagingDecryptionKey") PGPPrivateKey stagingDecryptionKey;
-  @Inject @Named("rde-report") Queue reportQueue;
   @Inject RdeUploadAction() {}
 
   @Override
   public void run() {
     logger.atInfo().log("Attempting to acquire RDE upload lock for TLD '%s'.", tld);
     runner.lockRunAndRollForward(this, Registry.get(tld), timeout, CursorType.RDE_UPLOAD, interval);
-    taskQueueUtils.enqueue(
-        reportQueue, withUrl(RdeReportAction.PATH).param(RequestParameters.PARAM_TLD, tld));
+    HashMultimap<String, String> params = HashMultimap.create();
+    params.put(RequestParameters.PARAM_TLD, tld);
+    if (prefix.isPresent()) {
+      params.put(RdeModule.PARAM_PREFIX, prefix.get());
+    }
+    cloudTasksUtils.enqueue(
+        RDE_REPORT_QUEUE,
+        CloudTasksUtils.createPostTask(
+            RdeReportAction.PATH, Service.BACKEND.getServiceId(), params));
   }
 
   @Override
@@ -164,7 +171,9 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
         RdeRevision.getCurrentRevision(tld, watermark, FULL)
             .orElseThrow(
                 () -> new IllegalStateException("RdeRevision was not set on generated deposit"));
-    final String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
+    final String nameWithoutPrefix =
+        RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
+    final String name = prefix.orElse("") + nameWithoutPrefix;
     final BlobId xmlFilename = BlobId.of(bucket, name + ".xml.ghostryde");
     final BlobId xmlLengthFilename = BlobId.of(bucket, name + ".xml.length");
     BlobId reportFilename = BlobId.of(bucket, name + "-report.xml.ghostryde");
@@ -174,7 +183,8 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
     logger.atInfo().log("Commencing RDE upload for TLD '%s' to '%s'.", tld, uploadUrl);
     final long xmlLength = readXmlLength(xmlLengthFilename);
     retrier.callWithRetry(
-        () -> upload(xmlFilename, xmlLength, watermark, name), JSchException.class);
+        () -> upload(xmlFilename, xmlLength, watermark, name, nameWithoutPrefix),
+        JSchException.class);
     logger.atInfo().log(
         "Updating RDE cursor '%s' for TLD '%s' following successful upload.", RDE_UPLOAD_SFTP, tld);
     tm().transact(
@@ -210,7 +220,8 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
    * }</pre>
    */
   @VisibleForTesting
-  protected void upload(BlobId xmlFile, long xmlLength, DateTime watermark, String name)
+  protected void upload(
+      BlobId xmlFile, long xmlLength, DateTime watermark, String name, String nameWithoutPrefix)
       throws Exception {
     logger.atInfo().log("Uploading XML file '%s' to remote path '%s'.", xmlFile, uploadUrl);
     try (InputStream gcsInput = gcsUtils.openInputStream(xmlFile);
@@ -218,8 +229,8 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
       try (JSchSshSession session = jschSshSessionFactory.create(lazyJsch.get(), uploadUrl);
           JSchSftpChannel ftpChan = session.openSftpChannel()) {
         ByteArrayOutputStream sigOut = new ByteArrayOutputStream();
-        String rydeFilename = name + ".ryde";
-        BlobId rydeGcsFilename = BlobId.of(bucket, rydeFilename);
+        String rydeFilename = nameWithoutPrefix + ".ryde";
+        BlobId rydeGcsFilename = BlobId.of(bucket, name + ".ryde");
         try (OutputStream ftpOutput =
                 ftpChan.get().put(rydeFilename, sftpProgressMonitor, OVERWRITE);
             OutputStream gcsOutput = gcsUtils.openOutputStream(rydeGcsFilename);
@@ -228,14 +239,15 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
                 new RydeEncoder.Builder()
                     .setRydeOutput(teeOutput, receiverKey)
                     .setSignatureOutput(sigOut, signingKey)
-                    .setFileMetadata(name, xmlLength, watermark)
+                    .setFileMetadata(nameWithoutPrefix, xmlLength, watermark)
                     .build()) {
             long bytesCopied = ByteStreams.copy(ghostrydeDecoder, rydeEncoder);
           logger.atInfo().log("Uploaded %,d bytes to path '%s'.", bytesCopied, rydeFilename);
           }
-        String sigFilename = name + ".sig";
+        String sigFilename = nameWithoutPrefix + ".sig";
+        BlobId sigGcsFilename = BlobId.of(bucket, name + ".sig");
         byte[] signature = sigOut.toByteArray();
-        gcsUtils.createFromBytes(BlobId.of(bucket, sigFilename), signature);
+        gcsUtils.createFromBytes(sigGcsFilename, signature);
         ftpChan.get().put(new ByteArrayInputStream(signature), sigFilename);
         logger.atInfo().log("Uploaded %,d bytes to path '%s'.", signature.length, sigFilename);
       }

core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java

@@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryEnvironment;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
 import google.registry.reporting.ReportingModule;
 import google.registry.request.Action;
@@ -127,7 +128,9 @@ public class GenerateInvoicesAction implements Runnable {
                       "database",
                       database.name(),
                       "billingBucketUrl",
-                      billingBucketUrl));
+                      billingBucketUrl,
+                      "registryEnvironment",
+                      RegistryEnvironment.get().name()));
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/java/google/registry/reporting/icann/ActivityReportingQueryBuilder.java

@@ -16,6 +16,7 @@ package google.registry.reporting.icann;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
+import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
 import static google.registry.reporting.icann.QueryBuilderUtils.getQueryFromFile;
 import static google.registry.reporting.icann.QueryBuilderUtils.getTableName;
 
@@ -23,6 +24,7 @@ import com.google.common.collect.ImmutableMap;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.util.SqlTemplate;
 import javax.inject.Inject;
+import javax.inject.Named;
 import org.joda.time.LocalDate;
 import org.joda.time.YearMonth;
 import org.joda.time.format.DateTimeFormat;
@@ -38,29 +40,32 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
   static final String EPP_METRICS = "epp_metrics";
   static final String WHOIS_COUNTS = "whois_counts";
   static final String ACTIVITY_REPORT_AGGREGATION = "activity_report_aggregation";
-  final String BIGQUERY_DATA_SET = tm().isOfy() ? "icann_reporting" : "cloud_sql_icann_reporting";
+
+  private final String projectId;
+  private final DnsCountQueryCoordinator dnsCountQueryCoordinator;
+  private final String icannReportingDataSet;
 
   @Inject
-  @Config("projectId")
-  String projectId;
-
-  @Inject DnsCountQueryCoordinator dnsCountQueryCoordinator;
-
-  @Inject
-  ActivityReportingQueryBuilder() {}
+  ActivityReportingQueryBuilder(
+      @Config("projectId") String projectId,
+      @Named(ICANN_REPORTING_DATA_SET) String icannReportingDataSet,
+      DnsCountQueryCoordinator dnsCountQueryCoordinator) {
+    this.projectId = projectId;
+    this.dnsCountQueryCoordinator = dnsCountQueryCoordinator;
+    this.icannReportingDataSet = icannReportingDataSet;
+  }
 
   /** Returns the aggregate query which generates the activity report from the saved view. */
   @Override
   public String getReportQuery(YearMonth yearMonth) {
     return String.format(
         "#standardSQL\nSELECT * FROM `%s.%s.%s`",
-        projectId, BIGQUERY_DATA_SET, getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth));
+        projectId, icannReportingDataSet, getTableName(ACTIVITY_REPORT_AGGREGATION, yearMonth));
   }
 
   /** Sets the month we're doing activity reporting for, and returns the view query map. */
   @Override
   public ImmutableMap<String, String> getViewQueryMap(YearMonth yearMonth) {
-
     LocalDate firstDayOfMonth = yearMonth.toLocalDate(1);
     // The pattern-matching is inclusive, so we subtract 1 day to only report that month's data.
     LocalDate lastDayOfMonth = yearMonth.toLocalDate(1).plusMonths(1).minusDays(1);
@@ -102,7 +107,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
     String eppQuery =
         SqlTemplate.create(getQueryFromFile("epp_metrics.sql"))
             .put("PROJECT_ID", projectId)
-            .put("ICANN_REPORTING_DATA_SET", BIGQUERY_DATA_SET)
+            .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
             .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth))
             // All metadata logs for reporting come from google.registry.flows.FlowReporter.
             .put(
@@ -114,7 +119,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
     String whoisQuery =
         SqlTemplate.create(getQueryFromFile("whois_counts.sql"))
             .put("PROJECT_ID", projectId)
-            .put("ICANN_REPORTING_DATA_SET", BIGQUERY_DATA_SET)
+            .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
             .put("MONTHLY_LOGS_TABLE", getTableName(MONTHLY_LOGS, yearMonth))
             .build();
     queriesBuilder.put(getTableName(WHOIS_COUNTS, yearMonth), whoisQuery);
@@ -129,7 +134,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
             .put(
                 "REGISTRAR_OPERATING_STATUS_TABLE",
                 getTableName(REGISTRAR_OPERATING_STATUS, yearMonth))
-            .put("ICANN_REPORTING_DATA_SET", BIGQUERY_DATA_SET)
+            .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
             .put("DNS_COUNTS_TABLE", getTableName(DNS_COUNTS, yearMonth))
             .put("EPP_METRICS_TABLE", getTableName(EPP_METRICS, yearMonth))
             .put("WHOIS_COUNTS_TABLE", getTableName(WHOIS_COUNTS, yearMonth));
@@ -147,7 +152,7 @@ public final class ActivityReportingQueryBuilder implements QueryBuilder {
     return queriesBuilder.build();
   }
 
-  public void prepareForQuery(YearMonth yearMonth) throws InterruptedException {
+  void prepareForQuery(YearMonth yearMonth) throws InterruptedException {
     dnsCountQueryCoordinator.prepareForQuery(yearMonth);
   }
 }

core/src/main/java/google/registry/reporting/icann/DnsCountQueryCoordinator.java

@@ -35,14 +35,19 @@ public interface DnsCountQueryCoordinator {
    * <p>If your report query requires any additional parameters, add them here.
    */
   class Params {
+
     public BigqueryConnection bigquery;
 
     /** The Google Cloud project id. */
     public String projectId;
 
-    public Params(BigqueryConnection bigquery, String projectId) {
+    /** The BigQuery dataset from which to query. */
+    public String icannReportingDataSet;
+
+    public Params(BigqueryConnection bigquery, String projectId, String icannReportingDataSet) {
       this.bigquery = bigquery;
       this.projectId = projectId;
+      this.icannReportingDataSet = icannReportingDataSet;
     }
   }
 

core/src/main/java/google/registry/reporting/icann/DnsCountQueryCoordinatorModule.java

@@ -14,6 +14,7 @@
 
 package google.registry.reporting.icann;
 
+import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
 import static google.registry.util.TypeUtils.getClassFromString;
 import static google.registry.util.TypeUtils.instantiate;
 
@@ -21,6 +22,7 @@ import dagger.Module;
 import dagger.Provides;
 import google.registry.bigquery.BigqueryConnection;
 import google.registry.config.RegistryConfig.Config;
+import javax.inject.Named;
 
 /** Dagger module to provide the DnsCountQueryCoordinator. */
 @Module
@@ -30,9 +32,10 @@ public class DnsCountQueryCoordinatorModule {
   static DnsCountQueryCoordinator provideDnsCountQueryCoordinator(
       @Config("dnsCountQueryCoordinatorClass") String customClass,
       BigqueryConnection bigquery,
-      @Config("projectId") String projectId) {
+      @Config("projectId") String projectId,
+      @Named(ICANN_REPORTING_DATA_SET) String icannReportingDataSet) {
     DnsCountQueryCoordinator.Params params =
-        new DnsCountQueryCoordinator.Params(bigquery, projectId);
+        new DnsCountQueryCoordinator.Params(bigquery, projectId, icannReportingDataSet);
     return instantiate(getClassFromString(customClass, DnsCountQueryCoordinator.class), params);
   }
 }

core/src/main/java/google/registry/reporting/icann/IcannReportingModule.java

@@ -14,7 +14,6 @@
 
 package google.registry.reporting.icann;
 
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static google.registry.request.RequestParameters.extractSetOfEnumParameters;
@@ -24,9 +23,11 @@ import com.google.common.util.concurrent.MoreExecutors;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.bigquery.BigqueryConnection;
+import google.registry.persistence.transaction.TransactionManager;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.Parameter;
 import java.util.Optional;
+import javax.inject.Named;
 import javax.servlet.http.HttpServletRequest;
 import org.joda.time.Duration;
 
@@ -42,8 +43,7 @@ public final class IcannReportingModule {
 
   static final String PARAM_SUBDIR = "subdir";
   static final String PARAM_REPORT_TYPES = "reportTypes";
-  static final String ICANN_REPORTING_DATA_SET =
-      tm().isOfy() ? "icann_reporting" : "cloud_sql_icann_reporting";
+  static final String ICANN_REPORTING_DATA_SET = "icannReportingDataSet";
   static final String DATASTORE_EXPORT_DATA_SET = "latest_datastore_export";
   static final String MANIFEST_FILE_NAME = "MANIFEST.txt";
 
@@ -88,11 +88,12 @@ public final class IcannReportingModule {
    */
   @Provides
   static BigqueryConnection provideBigqueryConnection(
-      BigqueryConnection.Builder bigQueryConnectionBuilder) {
+      BigqueryConnection.Builder bigQueryConnectionBuilder,
+      @Named(ICANN_REPORTING_DATA_SET) String icannReportingDataSet) {
     try {
       return bigQueryConnectionBuilder
           .setExecutorService(MoreExecutors.newDirectExecutorService())
-          .setDatasetId(ICANN_REPORTING_DATA_SET)
+          .setDatasetId(icannReportingDataSet)
           .setOverwrite(true)
           .setPollInterval(Duration.standardSeconds(1))
           .build();
@@ -100,4 +101,10 @@ public final class IcannReportingModule {
       throw new RuntimeException("Could not initialize BigqueryConnection!", e);
     }
   }
+
+  @Provides
+  @Named(ICANN_REPORTING_DATA_SET)
+  static String provideIcannReportingDataSet(TransactionManager tm) {
+    return tm.isOfy() ? "icann_reporting" : "cloud_sql_icann_reporting";
+  }
 }

core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java

@@ -14,7 +14,6 @@
 
 package google.registry.reporting.icann;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
@@ -46,7 +45,6 @@ import google.registry.util.SendEmailService;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Map;
-import java.util.concurrent.Callable;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import javax.inject.Inject;
@@ -78,6 +76,7 @@ public final class IcannReportingUploadAction implements Runnable {
   static final String PATH = "/_dr/task/icannReportingUpload";
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final String LOCK_NAME = "IcannReportingUploadAction";
 
   @Inject
   @Config("reportingBucket")
@@ -98,48 +97,33 @@ public final class IcannReportingUploadAction implements Runnable {
 
   @Override
   public void run() {
-    Runnable transactional =
-        () -> {
-          ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
-
-          ImmutableMap<Cursor, String> cursors = loadCursors();
-
-          // If cursor time is before now, upload the corresponding report
-          cursors.entrySet().stream()
-              .filter(entry -> entry.getKey().getCursorTime().isBefore(clock.nowUtc()))
-              .forEach(
-                  entry -> {
-                    DateTime cursorTime = entry.getKey().getCursorTime();
-                    uploadReport(
-                        cursorTime,
-                        entry.getKey().getType(),
-                        entry.getValue(),
-                        reportSummaryBuilder);
-                  });
-          // Send email of which reports were uploaded
-          emailUploadResults(reportSummaryBuilder.build());
-          response.setStatus(SC_OK);
-          response.setContentType(PLAIN_TEXT_UTF_8);
-        };
-
-    Callable<Void> lockRunner =
-        () -> {
-          tm().transact(transactional);
-          return null;
-        };
-
-    String lockname = "IcannReportingUploadAction";
-    if (!lockHandler.executeWithLocks(lockRunner, null, Duration.standardHours(2), lockname)) {
-      throw new ServiceUnavailableException("Lock for IcannReportingUploadAction already in use");
+    if (!lockHandler.executeWithLocks(
+        this::runWithLock, null, Duration.standardHours(2), LOCK_NAME)) {
+      throw new ServiceUnavailableException(String.format("Lock for %s already in use", LOCK_NAME));
     }
   }
 
+  private Void runWithLock() {
+    ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
+
+    ImmutableMap<Cursor, String> cursors = tm().transact(this::loadCursors);
+
+    // If cursor time is before now, upload the corresponding report
+    cursors.entrySet().stream()
+        .filter(entry -> entry.getKey().getCursorTime().isBefore(clock.nowUtc()))
+        .forEach(entry -> uploadReport(entry.getKey(), entry.getValue(), reportSummaryBuilder));
+    // Send email of which reports were uploaded
+    emailUploadResults(reportSummaryBuilder.build());
+    response.setStatus(SC_OK);
+    response.setContentType(PLAIN_TEXT_UTF_8);
+    return null;
+  }
+
   /** Uploads the report and rolls forward the cursor for that report. */
   private void uploadReport(
-      DateTime cursorTime,
-      CursorType cursorType,
-      String tldStr,
-      ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) {
+      Cursor cursor, String tldStr, ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) {
+    DateTime cursorTime = cursor.getCursorTime();
+    CursorType cursorType = cursor.getType();
     DateTime cursorTimeMinusMonth = cursorTime.withDayOfMonth(1).minusMonths(1);
     String reportSubdir =
         String.format(
@@ -150,17 +134,16 @@ public final class IcannReportingUploadAction implements Runnable {
         BlobId.of(reportingBucket, String.format("%s/%s", reportSubdir, filename));
     logger.atInfo().log("Reading ICANN report %s from bucket '%s'.", filename, reportingBucket);
     // Check that the report exists
-    try {
-      verifyFileExists(gcsFilename);
-    } catch (IllegalArgumentException e) {
+    if (!gcsUtils.existsAndNotEmpty(gcsFilename)) {
       String logMessage =
           String.format(
-              "Could not upload %s report for %s because file %s did not exist.",
-              cursorType, tldStr, filename);
+              "Could not upload %s report for %s because file %s (object %s in bucket %s) did not"
+                  + " exist.",
+              cursorType, tldStr, filename, gcsFilename.getName(), gcsFilename.getBucket());
       if (clock.nowUtc().dayOfMonth().get() == 1) {
-        logger.atInfo().withCause(e).log(logMessage + " This report may not have been staged yet.");
+        logger.atInfo().log(logMessage + " This report may not have been staged yet.");
       } else {
-        logger.atSevere().withCause(e).log(logMessage);
+        logger.atSevere().log(logMessage);
       }
       reportSummaryBuilder.put(filename, false);
       return;
@@ -179,7 +162,6 @@ public final class IcannReportingUploadAction implements Runnable {
     } catch (RuntimeException e) {
       logger.atWarning().withCause(e).log("Upload to %s failed.", gcsFilename);
     }
-    reportSummaryBuilder.put(filename, success);
 
     // Set cursor to first day of next month if the upload succeeded
     if (success) {
@@ -188,8 +170,24 @@ public final class IcannReportingUploadAction implements Runnable {
               cursorType,
               cursorTime.withTimeAtStartOfDay().withDayOfMonth(1).plusMonths(1),
               Registry.get(tldStr));
-      tm().put(newCursor);
+      // In order to keep the transactions short-lived, we load all of the cursors in a single
+      // transaction then later use per-cursor transactions when checking + saving the cursors. We
+      // run behind a lock so the cursors shouldn't be changed, but double check to be sure.
+      success =
+          tm().transact(
+                  () -> {
+                    Cursor fromDb = tm().transact(() -> tm().loadByEntity(cursor));
+                    if (!cursor.equals(fromDb)) {
+                      logger.atSevere().log(
+                          "Expected previously-loaded cursor %s to equal current cursor %s",
+                          cursor, fromDb);
+                      return false;
+                    }
+                    tm().put(newCursor);
+                    return true;
+                  });
     }
+    reportSummaryBuilder.put(filename, success);
   }
 
   private String getFileName(CursorType cursorType, DateTime cursorTime, String tld) {
@@ -303,13 +301,4 @@ public final class IcannReportingUploadAction implements Runnable {
       return ByteStreams.toByteArray(gcsInput);
     }
   }
-
-  private void verifyFileExists(BlobId gcsFilename) {
-    checkArgument(
-        gcsUtils.existsAndNotEmpty(gcsFilename),
-        "Object %s in bucket %s not found",
-        gcsFilename.getName(),
-        gcsFilename.getBucket());
-  }
-
 }

core/src/main/java/google/registry/reporting/icann/TransactionsReportingQueryBuilder.java

@@ -14,6 +14,7 @@
 
 package google.registry.reporting.icann;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.reporting.icann.IcannReportingModule.DATASTORE_EXPORT_DATA_SET;
 import static google.registry.reporting.icann.IcannReportingModule.ICANN_REPORTING_DATA_SET;
 import static google.registry.reporting.icann.QueryBuilderUtils.getQueryFromFile;
@@ -23,6 +24,7 @@ import com.google.common.collect.ImmutableMap;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.util.SqlTemplate;
 import javax.inject.Inject;
+import javax.inject.Named;
 import org.joda.time.DateTime;
 import org.joda.time.LocalTime;
 import org.joda.time.YearMonth;
@@ -34,9 +36,16 @@ import org.joda.time.format.DateTimeFormatter;
  */
 public final class TransactionsReportingQueryBuilder implements QueryBuilder {
 
-  @Inject @Config("projectId") String projectId;
+  final String projectId;
+  private final String icannReportingDataSet;
 
-  @Inject TransactionsReportingQueryBuilder() {}
+  @Inject
+  TransactionsReportingQueryBuilder(
+      @Config("projectId") String projectId,
+      @Named(ICANN_REPORTING_DATA_SET) String icannReportingDataSet) {
+    this.projectId = projectId;
+    this.icannReportingDataSet = icannReportingDataSet;
+  }
 
   static final String TRANSACTIONS_REPORT_AGGREGATION = "transactions_report_aggregation";
   static final String REGISTRAR_IANA_ID = "registrar_iana_id";
@@ -51,9 +60,7 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
   public String getReportQuery(YearMonth yearMonth) {
     return String.format(
         "#standardSQL\nSELECT * FROM `%s.%s.%s`",
-        projectId,
-        ICANN_REPORTING_DATA_SET,
-        getTableName(TRANSACTIONS_REPORT_AGGREGATION, yearMonth));
+        projectId, icannReportingDataSet, getTableName(TRANSACTIONS_REPORT_AGGREGATION, yearMonth));
   }
 
   /** Sets the month we're doing transactions reporting for, and returns the view query map. */
@@ -65,102 +72,180 @@ public final class TransactionsReportingQueryBuilder implements QueryBuilder {
     DateTime latestReportTime = earliestReportTime.plusMonths(1).minusMillis(1);
 
     ImmutableMap.Builder<String, String> queriesBuilder = ImmutableMap.builder();
-    String registrarIanaIdQuery =
-        SqlTemplate.create(getQueryFromFile("registrar_iana_id.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .build();
+    String registrarIanaIdQuery;
+    if (tm().isOfy()) {
+      registrarIanaIdQuery =
+          SqlTemplate.create(getQueryFromFile("registrar_iana_id.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .build();
+    } else {
+      registrarIanaIdQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_registrar_iana_id.sql"))
+              .put("PROJECT_ID", projectId)
+              .build();
+    }
     queriesBuilder.put(getTableName(REGISTRAR_IANA_ID, yearMonth), registrarIanaIdQuery);
 
-    String totalDomainsQuery =
-        SqlTemplate.create(getQueryFromFile("total_domains.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("DOMAINBASE_TABLE", "DomainBase")
-            .put("REGISTRAR_TABLE", "Registrar")
-            .build();
+    String totalDomainsQuery;
+    if (tm().isOfy()) {
+      totalDomainsQuery =
+          SqlTemplate.create(getQueryFromFile("total_domains.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("DOMAINBASE_TABLE", "DomainBase")
+              .put("REGISTRAR_TABLE", "Registrar")
+              .build();
+    } else {
+      totalDomainsQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_total_domains.sql"))
+              .put("PROJECT_ID", projectId)
+              .build();
+    }
     queriesBuilder.put(getTableName(TOTAL_DOMAINS, yearMonth), totalDomainsQuery);
 
     DateTimeFormatter timestampFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS");
-    String totalNameserversQuery =
-        SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("HOSTRESOURCE_TABLE", "HostResource")
-            .put("DOMAINBASE_TABLE", "DomainBase")
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
-            .build();
+    String totalNameserversQuery;
+    if (tm().isOfy()) {
+      totalNameserversQuery =
+          SqlTemplate.create(getQueryFromFile("total_nameservers.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("HOSTRESOURCE_TABLE", "HostResource")
+              .put("DOMAINBASE_TABLE", "DomainBase")
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    } else {
+      totalNameserversQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_total_nameservers.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    }
     queriesBuilder.put(getTableName(TOTAL_NAMESERVERS, yearMonth), totalNameserversQuery);
 
-    String transactionCountsQuery =
-        SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("HISTORYENTRY_TABLE", "HistoryEntry")
-            .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
-            .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
-            .put("CLIENT_ID", "clientId")
-            .put("OTHER_CLIENT_ID", "otherClientId")
-            .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_GAINING_SUCCESSFUL")
-            .put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED")
-            .put("DEFAULT_FIELD", "field")
-            .build();
+    String transactionCountsQuery;
+    if (tm().isOfy()) {
+      transactionCountsQuery =
+          SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("HISTORYENTRY_TABLE", "HistoryEntry")
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .put("CLIENT_ID", "clientId")
+              .put("OTHER_CLIENT_ID", "otherClientId")
+              .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_GAINING_SUCCESSFUL")
+              .put("TRANSFER_NACKED_FIELD", "TRANSFER_GAINING_NACKED")
+              .put("DEFAULT_FIELD", "field")
+              .build();
+    } else {
+      transactionCountsQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_transaction_counts.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    }
     queriesBuilder.put(getTableName(TRANSACTION_COUNTS, yearMonth), transactionCountsQuery);
 
-    String transactionTransferLosingQuery =
-        SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("HISTORYENTRY_TABLE", "HistoryEntry")
-            .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
-            .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
-            // Roles are reversed for losing queries
-            .put("CLIENT_ID", "otherClientId")
-            .put("OTHER_CLIENT_ID", "clientId")
-            .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_LOSING_SUCCESSFUL")
-            .put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED")
-            .put("DEFAULT_FIELD", "NULL")
-            .build();
+    String transactionTransferLosingQuery;
+    if (tm().isOfy()) {
+      transactionTransferLosingQuery =
+          SqlTemplate.create(getQueryFromFile("transaction_counts.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("HISTORYENTRY_TABLE", "HistoryEntry")
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              // Roles are reversed for losing queries
+              .put("CLIENT_ID", "otherClientId")
+              .put("OTHER_CLIENT_ID", "clientId")
+              .put("TRANSFER_SUCCESS_FIELD", "TRANSFER_LOSING_SUCCESSFUL")
+              .put("TRANSFER_NACKED_FIELD", "TRANSFER_LOSING_NACKED")
+              .put("DEFAULT_FIELD", "NULL")
+              .build();
+    } else {
+      transactionTransferLosingQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_transaction_transfer_losing.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("EARLIEST_REPORT_TIME", timestampFormatter.print(earliestReportTime))
+              .put("LATEST_REPORT_TIME", timestampFormatter.print(latestReportTime))
+              .build();
+    }
     queriesBuilder.put(
         getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth), transactionTransferLosingQuery);
 
     // App Engine log table suffixes use YYYYMMDD format
     DateTimeFormatter logTableFormatter = DateTimeFormat.forPattern("yyyyMMdd");
-    String attemptedAddsQuery =
-        SqlTemplate.create(getQueryFromFile("attempted_adds.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRAR_TABLE", "Registrar")
-            .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
-            .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
-            .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
-            .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
-            // All metadata logs for reporting come from google.registry.flows.FlowReporter.
-            .put(
-                "METADATA_LOG_PREFIX",
-                "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
-            .build();
+    String attemptedAddsQuery;
+    if (tm().isOfy()) {
+      attemptedAddsQuery =
+          SqlTemplate.create(getQueryFromFile("attempted_adds.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRAR_TABLE", "Registrar")
+              .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
+              .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
+              .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
+              .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
+              // All metadata logs for reporting come from google.registry.flows.FlowReporter.
+              .put(
+                  "METADATA_LOG_PREFIX",
+                  "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
+              .build();
+    } else {
+      attemptedAddsQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_attempted_adds.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("APPENGINE_LOGS_DATA_SET", "appengine_logs")
+              .put("REQUEST_TABLE", "appengine_googleapis_com_request_log_")
+              .put("FIRST_DAY_OF_MONTH", logTableFormatter.print(earliestReportTime))
+              .put("LAST_DAY_OF_MONTH", logTableFormatter.print(latestReportTime))
+              // All metadata logs for reporting come from google.registry.flows.FlowReporter.
+              .put(
+                  "METADATA_LOG_PREFIX",
+                  "google.registry.flows.FlowReporter recordToLogs: FLOW-LOG-SIGNATURE-METADATA")
+              .build();
+    }
     queriesBuilder.put(getTableName(ATTEMPTED_ADDS, yearMonth), attemptedAddsQuery);
 
-    String aggregateQuery =
-        SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
-            .put("PROJECT_ID", projectId)
-            .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
-            .put("REGISTRY_TABLE", "Registry")
-            .put("ICANN_REPORTING_DATA_SET", ICANN_REPORTING_DATA_SET)
-            .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID, yearMonth))
-            .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS, yearMonth))
-            .put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS, yearMonth))
-            .put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS, yearMonth))
-            .put(
-                "TRANSACTION_TRANSFER_LOSING_TABLE",
-                getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth))
-            .put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS, yearMonth))
-            .build();
+    String aggregateQuery;
+    if (tm().isOfy()) {
+      aggregateQuery =
+          SqlTemplate.create(getQueryFromFile("transactions_report_aggregation.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("DATASTORE_EXPORT_DATA_SET", DATASTORE_EXPORT_DATA_SET)
+              .put("REGISTRY_TABLE", "Registry")
+              .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
+              .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID, yearMonth))
+              .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS, yearMonth))
+              .put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS, yearMonth))
+              .put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS, yearMonth))
+              .put(
+                  "TRANSACTION_TRANSFER_LOSING_TABLE",
+                  getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth))
+              .put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS, yearMonth))
+              .build();
+    } else {
+      aggregateQuery =
+          SqlTemplate.create(getQueryFromFile("cloud_sql_transactions_report_aggregation.sql"))
+              .put("PROJECT_ID", projectId)
+              .put("ICANN_REPORTING_DATA_SET", icannReportingDataSet)
+              .put("REGISTRAR_IANA_ID_TABLE", getTableName(REGISTRAR_IANA_ID, yearMonth))
+              .put("TOTAL_DOMAINS_TABLE", getTableName(TOTAL_DOMAINS, yearMonth))
+              .put("TOTAL_NAMESERVERS_TABLE", getTableName(TOTAL_NAMESERVERS, yearMonth))
+              .put("TRANSACTION_COUNTS_TABLE", getTableName(TRANSACTION_COUNTS, yearMonth))
+              .put(
+                  "TRANSACTION_TRANSFER_LOSING_TABLE",
+                  getTableName(TRANSACTION_TRANSFER_LOSING, yearMonth))
+              .put("ATTEMPTED_ADDS_TABLE", getTableName(ATTEMPTED_ADDS, yearMonth))
+              .build();
+    }
     queriesBuilder.put(getTableName(TRANSACTIONS_REPORT_AGGREGATION, yearMonth), aggregateQuery);
 
     return queriesBuilder.build();

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_attempted_adds.sql

@@ -0,0 +1,73 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Determine the number of attempted adds each registrar made.
+
+  -- Since the specification requests all 'attempted' adds, we regex the
+  -- monthly App Engine logs, searching for all create commands and associating
+  -- them with their corresponding registrars.
+
+  -- Example log generated by FlowReporter in App Engine logs:
+  --google.registry.flows.FlowReporter
+  -- recordToLogs: FLOW-LOG-SIGNATURE-METADATA:
+  --{"serverTrid":"oNwL2J2eRya7bh7c9oHIzg==-2360a","clientId":"ipmirror"
+  -- ,"commandType":"hello", "resourceType":"","flowClassName":"HelloFlow"
+  -- ,"targetId":"","targetIds":[],"tld":"",
+  -- "tlds":[],"icannActivityReportField":""}
+
+-- This outer select just converts the registrar's clientId to their name.
+SELECT
+  tld,
+  registrar_table.registrar_name AS registrar_name,
+  'ATTEMPTED_ADDS' AS metricName,
+  count AS metricValue
+FROM (
+  SELECT
+    JSON_EXTRACT_SCALAR(json, '$.tld') AS tld,
+    JSON_EXTRACT_SCALAR(json, '$.clientId') AS clientId,
+    COUNT(json) AS count
+  FROM (
+      -- Extract JSON metadata package from monthly logs
+    SELECT
+      REGEXP_EXTRACT(logMessages, r'FLOW-LOG-SIGNATURE-METADATA: (.*)\n?$')
+          AS json
+    FROM (
+      SELECT
+        protoPayload.resource AS requestPath,
+        ARRAY(
+          SELECT logMessage
+          FROM UNNEST(protoPayload.line)) AS logMessage
+      FROM
+        `%PROJECT_ID%.%APPENGINE_LOGS_DATA_SET%.%REQUEST_TABLE%*`
+      WHERE _TABLE_SUFFIX
+      BETWEEN '%FIRST_DAY_OF_MONTH%'
+      AND '%LAST_DAY_OF_MONTH%')
+    JOIN UNNEST(logMessage) AS logMessages
+    -- Look for metadata logs from epp and registrar console requests
+    WHERE requestPath IN ('/_dr/epp', '/_dr/epptool', '/registrar-xhr')
+    AND STARTS_WITH(logMessages, "%METADATA_LOG_PREFIX%")
+    -- Look for domain creates
+    AND REGEXP_CONTAINS(
+       logMessages, r'"commandType":"create","resourceType":"domain"')
+    -- Filter prober data
+    AND NOT REGEXP_CONTAINS(
+        logMessages, r'"prober-[a-z]{2}-((any)|(canary))"') )
+  GROUP BY tld, clientId ) AS logs_table
+JOIN
+  EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+              '''SELECT registrar_id, registrar_name FROM "Registrar";''') AS registrar_table
+  ON
+      logs_table.clientId = registrar_table.registrar_id
+  ORDER BY tld, registrar_name

core/src/main/java/google/registry/reporting/icann/sql/cloud_sql_registrar_iana_id.sql

@@ -0,0 +1,37 @@
+#standardSQL
+  -- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+  --
+  -- Licensed under the Apache License, Version 2.0 (the "License");
+  -- you may not use this file except in compliance with the License.
+  -- You may obtain a copy of the License at
+  --
+  --     http://www.apache.org/licenses/LICENSE-2.0
+  --
+  -- Unless required by applicable law or agreed to in writing, software
+  -- distributed under the License is distributed on an "AS IS" BASIS,
+  -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  -- See the License for the specific language governing permissions and
+  -- limitations under the License.
+
+  -- Gather a list of all tld-registrar pairs, with their IANA IDs.
+
+  -- This establishes which registrars will appear in the reports.
+
+SELECT
+  allowedTlds AS tld,
+  registrar_name,
+  iana_identifier AS iana_id
+FROM
+  EXTERNAL_QUERY("projects/%PROJECT_ID%/locations/us/connections/%PROJECT_ID%-sql",
+  '''SELECT
+    allowedTlds,
+    registrar_name,
+    iana_identifier
+   FROM
+    "Registrar" AS r,
+    UNNEST(allowed_tlds) AS allowedTlds
+   WHERE
+    r.type='REAL' OR r.type='INTERNAL';''')
+-- Filter out prober data
+WHERE NOT ENDS_WITH(allowedTlds, "test")
+ORDER BY tld, registrar_name