Provide useful error messages on flows run during read-only mode (#1425)

We want to keep the read-only-mode-exception as an unchecked exception,
so we introduce a temporary check in the EppController that provides a
specific error message for this situation (rather than letting it fall
through to the generic "command failed" messaging

core/build.gradle

@@ -470,7 +470,7 @@ task soyToJava {
 
     outputs.each { file ->
       exec {
-        commandLine 'sed', '-i', 's/@link/LINK/g', file.getCanonicalPath()
+        commandLine 'sed', '-i""', '-e', 's/@link/LINK/g', file.getCanonicalPath()
       }
     }
   }

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -41,11 +41,12 @@ public class BackupUtils {
   }
 
   /**
-   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an
-   * {@link OutputStream} in delimited protocol buffer format.
+   * Converts the given {@link ImmutableObject} to a raw Datastore entity and write it to an {@link
+   * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().save().toEntity(entity)).writeDelimitedTo(stream);
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnly().toEntity(entity))
+        .writeDelimitedTo(stream);
   }
 
   /**

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -74,7 +74,7 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveWithoutBackup()
+                  .saveIgnoringReadOnly()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -140,7 +140,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
       response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,10 +24,8 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
@@ -84,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -96,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -114,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -129,14 +125,13 @@ public final class AsyncTaskEnqueuer {
       String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingRegistrarId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))

core/src/main/java/google/registry/batch/BatchModule.java

@@ -79,6 +79,7 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
+  // TODO(b/207363014): figure out if this needs to be modified for vkey string replacement
   static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
     return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
   }

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -523,18 +523,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -353,8 +353,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
       VKey<HostResource> hostKey =
-          VKey.fromWebsafeKey(
-              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
           tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
               .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -76,13 +76,15 @@ public class ResaveEntityAction implements Runnable {
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
     tm().transact(
             () -> {
+              // TODO(/207363014): figure out if this should modified for vkey string replacement
               ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
               tm().put(
                       (entity instanceof EppResource)
                           ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
                           : entity);
               if (!resaveTimes.isEmpty()) {
-                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.from(resourceKey), requestedTime, resaveTimes);
               }
             });
     response.setPayload("Entity re-saved.");

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -57,7 +57,8 @@ import org.apache.beam.sdk.values.TypeDescriptor;
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=invoicing}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -75,6 +75,8 @@ public class RdeIO {
   abstract static class Write
       extends PTransform<PCollection<KV<PendingDeposit, Iterable<DepositFragment>>>, PDone> {
 
+    private static final long serialVersionUID = 3334807737227087760L;
+
     abstract GcsUtils gcsUtils();
 
     abstract CloudTasksUtils cloudTasksUtils();
@@ -113,8 +115,9 @@ public class RdeIO {
           .apply(
               "Write to GCS",
               ParDo.of(new RdeWriter(gcsUtils(), rdeBucket(), stagingKeyBytes(), validationMode())))
-          .apply("Update cursors", ParDo.of(new CursorUpdater()))
-          .apply("Enqueue upload action", ParDo.of(new UploadEnqueuer(cloudTasksUtils())));
+          .apply(
+              "Update cursor and enqueue next action",
+              ParDo.of(new CursorUpdater(cloudTasksUtils())));
       return PDone.in(input.getPipeline());
     }
   }
@@ -123,6 +126,7 @@ public class RdeIO {
       extends DoFn<KV<PendingDeposit, Iterable<DepositFragment>>, KV<PendingDeposit, Integer>> {
 
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5496375923068400382L;
 
     private final GcsUtils gcsUtils;
     private final String rdeBucket;
@@ -169,7 +173,7 @@ public class RdeIO {
         checkState(key.directoryWithTrailingSlash() != null, "Manual subdirectory not specified");
         prefix = prefix + "/manual/" + key.directoryWithTrailingSlash() + basename;
       } else {
-        prefix = prefix + "/" + basename;
+        prefix = prefix + '/' + basename;
       }
       BlobId xmlFilename = BlobId.of(rdeBucket, prefix + ".xml.ghostryde");
       // This file will contain the byte length (ASCII) of the raw unencrypted XML.
@@ -250,12 +254,20 @@ public class RdeIO {
     }
   }
 
-  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, PendingDeposit> {
+  private static class CursorUpdater extends DoFn<KV<PendingDeposit, Integer>, Void> {
+
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+    private static final long serialVersionUID = 5822176227753327224L;
+
+    private final CloudTasksUtils cloudTasksUtils;
+
+    private CursorUpdater(CloudTasksUtils cloudTasksUtils) {
+      this.cloudTasksUtils = cloudTasksUtils;
+    }
 
     @ProcessElement
     public void processElement(
-        @Element KV<PendingDeposit, Integer> input, OutputReceiver<PendingDeposit> outputReceiver) {
+        @Element KV<PendingDeposit, Integer> input, PipelineOptions options) {
       tm().transact(
               () -> {
                 PendingDeposit key = input.getKey();
@@ -282,46 +294,32 @@ public class RdeIO {
                 logger.atInfo().log(
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                if (key.mode() == RdeMode.FULL) {
+                  cloudTasksUtils.enqueue(
+                      RDE_UPLOAD_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          RdeUploadAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                } else {
+                  cloudTasksUtils.enqueue(
+                      BRDA_QUEUE,
+                      CloudTasksUtils.createPostTask(
+                          BrdaCopyAction.PATH,
+                          Service.BACKEND.getServiceId(),
+                          ImmutableMultimap.of(
+                              RequestParameters.PARAM_TLD,
+                              key.tld(),
+                              RdeModule.PARAM_WATERMARK,
+                              key.watermark().toString(),
+                              RdeModule.PARAM_PREFIX,
+                              options.getJobName() + '/')));
+                }
               });
-      outputReceiver.output(input.getKey());
-    }
-  }
-
-  private static class UploadEnqueuer extends DoFn<PendingDeposit, Void> {
-
-    private final CloudTasksUtils cloudTasksUtils;
-
-    private UploadEnqueuer(CloudTasksUtils cloudTasksUtils) {
-      this.cloudTasksUtils = cloudTasksUtils;
-    }
-
-    @ProcessElement
-    public void processElement(@Element PendingDeposit input, PipelineOptions options) {
-      if (input.mode() == RdeMode.FULL) {
-        cloudTasksUtils.enqueue(
-            RDE_UPLOAD_QUEUE,
-            CloudTasksUtils.createPostTask(
-                RdeUploadAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      } else {
-        cloudTasksUtils.enqueue(
-            BRDA_QUEUE,
-            CloudTasksUtils.createPostTask(
-                BrdaCopyAction.PATH,
-                Service.BACKEND.getServiceId(),
-                ImmutableMultimap.of(
-                    RequestParameters.PARAM_TLD,
-                    input.tld(),
-                    RdeModule.PARAM_WATERMARK,
-                    input.watermark().toString(),
-                    RdeModule.PARAM_PREFIX,
-                    options.getJobName() + '/')));
-      }
     }
   }
 }

core/src/main/java/google/registry/beam/rde/RdePipeline.java

@@ -14,36 +14,50 @@
 
 package google.registry.beam.rde;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.EppResourceUtils.loadAtPointInTimeAsync;
+import static google.registry.beam.rde.RdePipeline.TupleTags.DOMAIN_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.EXTERNAL_HOST_FRAGMENTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.HOST_TO_PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.PENDING_DEPOSIT;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_CONTACTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REFERENCED_HOSTS;
+import static google.registry.beam.rde.RdePipeline.TupleTags.REVISION_ID;
+import static google.registry.beam.rde.RdePipeline.TupleTags.SUPERORDINATE_DOMAINS;
+import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import dagger.BindsInstance;
 import dagger.Component;
 import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
 import google.registry.config.CloudTasksUtilsModule;
 import google.registry.config.CredentialModule;
 import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.Type;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.VKey;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.PendingDeposit;
 import google.registry.rde.PendingDeposit.PendingDepositCoder;
-import google.registry.rde.RdeFragmenter;
 import google.registry.rde.RdeMarshaller;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.UtilsModule;
@@ -54,74 +68,158 @@ import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Supplier;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.persistence.Entity;
+import javax.persistence.IdClass;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineResult;
 import org.apache.beam.sdk.coders.KvCoder;
 import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.coders.StringUtf8Coder;
+import org.apache.beam.sdk.coders.VarLongCoder;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Filter;
 import org.apache.beam.sdk.transforms.FlatMapElements;
 import org.apache.beam.sdk.transforms.Flatten;
 import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.join.CoGbkResult;
+import org.apache.beam.sdk.transforms.join.CoGroupByKey;
+import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
 import org.apache.beam.sdk.values.TypeDescriptor;
-import org.apache.beam.sdk.values.TypeDescriptors;
 import org.joda.time.DateTime;
 
 /**
  * Definition of a Dataflow Flex template, which generates RDE/BRDA deposits.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=rde}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *
+ * <p>This pipeline only works for pending deposits with the same watermark, the {@link
+ * google.registry.rde.RdeStagingAction} will batch such pending deposits together and launch
+ * multiple pipelines if multiple watermarks exist.
+ *
+ * <p>The pipeline is broadly divided into two parts -- creating the {@link DepositFragment}s, and
+ * processing them.
+ *
+ * <h1>Creating {@link DepositFragment}</h1>
+ *
+ * <h2>{@link Registrar}</h2>
+ *
+ * Non-test registrar entities are loaded from Cloud SQL and marshalled into deposit fragments. They
+ * are <b>NOT</b> rewound to the watermark.
+ *
+ * <h2>{@link EppResource}</h2>
+ *
+ * All EPP resources are loaded from the corresponding {@link HistoryEntry}, which has the resource
+ * embedded. In general we find most recent history entry before watermark and filter out the ones
+ * that are soft-deleted by watermark. The history is emitted as pairs of (resource repo ID: history
+ * revision ID) from the SQL query.
+ *
+ * <h3>{@link DomainBase}</h3>
+ *
+ * After the most recent (live) domain resources are loaded from the corresponding history objects,
+ * we marshall them to deposit fragments and emit the (pending deposit: deposit fragment) pairs for
+ * further processing. We also find all the contacts and hosts referenced by a given domain and emit
+ * pairs of (contact/host repo ID: pending deposit) for all RDE pending deposits for further
+ * processing.
+ *
+ * <h3>{@link ContactResource}</h3>
+ *
+ * We first join most recent contact histories, represented by (contact repo ID: contact history
+ * revision ID) pairs, with referenced contacts, represented by (contact repo ID: pending deposit)
+ * pairs, on the contact repo ID, to remove unreferenced contact histories. Contact resources are
+ * then loaded from the remaining referenced contact histories, and marshalled into (pending
+ * deposit: deposit fragment) pairs.
+ *
+ * <h3>{@link HostResource}</h3>
+ *
+ * Similar to {@link ContactResource}, we join the most recent host history with referenced hosts to
+ * find most recent referenced hosts. For external hosts we do the same treatment as we did on
+ * contacts and obtain the (pending deposit: deposit fragment) pairs. For subordinate hosts, we need
+ * to find the superordinate domain in order to properly handle pending transfer in the deposit as
+ * well. So we first find the superordinate domain repo ID from the host and join the (superordinate
+ * domain repo ID: (subordinate host repo ID: (pending deposit: revision ID))) pair with the (domain
+ * repo ID: revision ID) pair obtained from the domain history query in order to map the host at
+ * watermark to the domain at watermark. We then proceed to create the (pending deposit: deposit
+ * fragment) pair for subordinate hosts using the added domain information.
+ *
+ * <h1>Processing {@link DepositFragment}</h1>
+ *
+ * The (pending deposit: deposit fragment) pairs from different resources are combined and grouped
+ * by pending deposit. For each pending deposit, all the relevant deposit fragments are written into
+ * a encrypted file stored on GCS. The filename is uniquely determined by the Beam job ID so there
+ * is no need to lock the GCS write operation to prevent stomping. The cursor for staging the
+ * pending deposit is then rolled forward, and the next action is enqueued. The latter two
+ * operations are performed in a transaction so the cursor is rolled back if enqueueing failed.
+ *
  * @see <a href="https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates">Using
  *     Flex Templates</a>
  */
 @Singleton
 public class RdePipeline implements Serializable {
 
+  private static final long serialVersionUID = -4866795928854754666L;
   private final transient RdePipelineOptions options;
   private final ValidationMode mode;
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings;
+  private final ImmutableSet<PendingDeposit> pendingDeposits;
+  private final DateTime watermark;
   private final String rdeBucket;
   private final byte[] stagingKeyBytes;
   private final GcsUtils gcsUtils;
   private final CloudTasksUtils cloudTasksUtils;
+  private final RdeMarshaller marshaller;
 
   // Registrars to be excluded from data escrow. Not including the sandbox-only OTE type so that
   // if sneaks into production we would get an extra signal.
   private static final ImmutableSet<Type> IGNORED_REGISTRAR_TYPES =
       Sets.immutableEnumSet(Registrar.Type.MONITORING, Registrar.Type.TEST);
 
-  private static final String EPP_RESOURCE_QUERY =
-      "SELECT id FROM %entity% "
-          + "WHERE COALESCE(creationClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(currentSponsorClientId, '') NOT LIKE 'prober-%' "
-          + "AND COALESCE(lastEppUpdateClientId, '') NOT LIKE 'prober-%'";
-
-  public static String createEppResourceQuery(Class<? extends EppResource> clazz) {
-    return EPP_RESOURCE_QUERY.replace("%entity%", clazz.getAnnotation(Entity.class).name())
-        + (clazz.equals(DomainBase.class) ? " AND tld in (:tlds)" : "");
-  }
+  // The field name of the EPP resource embedded in its corresponding history entry.
+  private static final ImmutableMap<Class<? extends HistoryEntry>, String> EPP_RESOURCE_FIELD_NAME =
+      ImmutableMap.of(
+          DomainHistory.class,
+          "domainContent",
+          ContactHistory.class,
+          "contactBase",
+          HostHistory.class,
+          "hostBase");
 
   @Inject
   RdePipeline(RdePipelineOptions options, GcsUtils gcsUtils, CloudTasksUtils cloudTasksUtils) {
     this.options = options;
     this.mode = ValidationMode.valueOf(options.getValidationMode());
-    this.pendings = decodePendings(options.getPendings());
+    this.pendingDeposits = decodePendingDeposits(options.getPendings());
+    ImmutableSet<DateTime> potentialWatermarks =
+        pendingDeposits.stream()
+            .map(PendingDeposit::watermark)
+            .distinct()
+            .collect(toImmutableSet());
+    checkArgument(
+        potentialWatermarks.size() == 1,
+        String.format(
+            "RDE pipeline should only work on pending deposits "
+                + "with the same watermark, but %d were given: %s",
+            potentialWatermarks.size(), potentialWatermarks));
+    this.watermark = potentialWatermarks.asList().get(0);
     this.rdeBucket = options.getRdeStagingBucket();
     this.stagingKeyBytes = BaseEncoding.base64Url().decode(options.getStagingKey());
     this.gcsUtils = gcsUtils;
     this.cloudTasksUtils = cloudTasksUtils;
+    this.marshaller = new RdeMarshaller(mode);
   }
 
   PipelineResult run() {
@@ -133,13 +231,46 @@ public class RdePipeline implements Serializable {
   }
 
   PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> createFragments(Pipeline pipeline) {
-    return PCollectionList.of(processRegistrars(pipeline))
-        .and(processNonRegistrarEntities(pipeline, DomainBase.class))
-        .and(processNonRegistrarEntities(pipeline, ContactResource.class))
-        .and(processNonRegistrarEntities(pipeline, HostResource.class))
-        .apply(Flatten.pCollections())
+    PCollection<KV<PendingDeposit, DepositFragment>> registrarFragments =
+        processRegistrars(pipeline);
+
+    PCollection<KV<String, Long>> domainHistories =
+        getMostRecentHistoryEntries(pipeline, DomainHistory.class);
+
+    PCollection<KV<String, Long>> contactHistories =
+        getMostRecentHistoryEntries(pipeline, ContactHistory.class);
+
+    PCollection<KV<String, Long>> hostHistories =
+        getMostRecentHistoryEntries(pipeline, HostHistory.class);
+
+    PCollectionTuple processedDomainHistories = processDomainHistories(domainHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> domainFragments =
+        processedDomainHistories.get(DOMAIN_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> contactFragments =
+        processContactHistories(
+            processedDomainHistories.get(REFERENCED_CONTACTS), contactHistories);
+
+    PCollectionTuple processedHosts =
+        processHostHistories(processedDomainHistories.get(REFERENCED_HOSTS), hostHistories);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> externalHostFragments =
+        processedHosts.get(EXTERNAL_HOST_FRAGMENTS);
+
+    PCollection<KV<PendingDeposit, DepositFragment>> subordinateHostFragments =
+        processSubordinateHosts(processedHosts.get(SUPERORDINATE_DOMAINS), domainHistories);
+
+    return PCollectionList.of(registrarFragments)
+        .and(domainFragments)
+        .and(contactFragments)
+        .and(externalHostFragments)
+        .and(subordinateHostFragments)
+        .apply(
+            "Combine PendingDeposit:DepositFragment pairs from all entities",
+            Flatten.pCollections())
         .setCoder(KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)))
-        .apply("Group by PendingDeposit", GroupByKey.create());
+        .apply("Group DepositFragment by PendingDeposit", GroupByKey.create());
   }
 
   void persistData(PCollection<KV<PendingDeposit, Iterable<DepositFragment>>> input) {
@@ -154,127 +285,398 @@ public class RdePipeline implements Serializable {
             .build());
   }
 
-  PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+  private PCollection<KV<PendingDeposit, DepositFragment>> processRegistrars(Pipeline pipeline) {
+    // Note that the namespace in the metric is not being used by Stackdriver, it just has to be
+    // non-empty.
+    // See:
+    // https://stackoverflow.com/questions/48530496/google-dataflow-custom-metrics-not-showing-on-stackdriver
+    Counter includedRegistrarCounter = Metrics.counter("RDE", "IncludedRegistrar");
+    Counter registrarFragmentCounter = Metrics.counter("RDE", "RegistrarFragment");
     return pipeline
         .apply(
-            "Read all production Registrar entities",
+            "Read all production Registrars",
             RegistryJpaIO.read(
                 "SELECT clientIdentifier FROM Registrar WHERE type NOT IN (:types)",
                 ImmutableMap.of("types", IGNORED_REGISTRAR_TYPES),
                 String.class,
-                // TODO: consider adding coders for entities and pass them directly instead of using
-                // VKeys.
                 id -> VKey.createSql(Registrar.class, id)))
         .apply(
-            "Marshal Registrar into DepositFragment",
+            "Marshall Registrar into DepositFragment",
             FlatMapElements.into(
-                    TypeDescriptors.kvs(
+                    kvs(
                         TypeDescriptor.of(PendingDeposit.class),
                         TypeDescriptor.of(DepositFragment.class)))
                 .via(
                     (VKey<Registrar> key) -> {
+                      includedRegistrarCounter.inc();
                       Registrar registrar = jpaTm().transact(() -> jpaTm().loadByKey(key));
-                      DepositFragment fragment =
-                          new RdeMarshaller(mode).marshalRegistrar(registrar);
-                      return pendings.values().stream()
-                          .map(pending -> KV.of(pending, fragment))
-                          .collect(toImmutableSet());
+                      DepositFragment fragment = marshaller.marshalRegistrar(registrar);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          pendingDeposits.stream()
+                              .map(pending -> KV.of(pending, fragment))
+                              .collect(toImmutableSet());
+                      registrarFragmentCounter.inc(fragments.size());
+                      return fragments;
                     }));
   }
 
-  <T extends EppResource>
-      PCollection<KV<PendingDeposit, DepositFragment>> processNonRegistrarEntities(
-          Pipeline pipeline, Class<T> clazz) {
-    return createInputs(pipeline, clazz)
-        .apply("Marshal " + clazz.getSimpleName() + " into DepositFragment", mapToFragments(clazz))
-        .setCoder(
-            KvCoder.of(PendingDepositCoder.of(), SerializableCoder.of(DepositFragment.class)));
+  /**
+   * Load the most recent history entry before the watermark for a given history entry type.
+   *
+   * <p>Note that deleted and non-production resources are not included.
+   *
+   * @return A KV pair of (repoId, revisionId), used to reconstruct the composite key for the
+   *     history entry.
+   */
+  private <T extends HistoryEntry> PCollection<KV<String, Long>> getMostRecentHistoryEntries(
+      Pipeline pipeline, Class<T> historyClass) {
+    String repoIdFieldName = HistoryEntryDao.REPO_ID_FIELD_NAMES.get(historyClass);
+    String resourceFieldName = EPP_RESOURCE_FIELD_NAME.get(historyClass);
+    return pipeline
+        .apply(
+            String.format("Load most recent %s", historyClass.getSimpleName()),
+            RegistryJpaIO.read(
+                ("SELECT %repoIdField%, id FROM %entity% WHERE (%repoIdField%, modificationTime)"
+                     + " IN (SELECT %repoIdField%, MAX(modificationTime) FROM %entity% WHERE"
+                     + " modificationTime <= :watermark GROUP BY %repoIdField%) AND"
+                     + " %resourceField%.deletionTime > :watermark AND"
+                     + " COALESCE(%resourceField%.creationClientId, '') NOT LIKE 'prober-%' AND"
+                     + " COALESCE(%resourceField%.currentSponsorClientId, '') NOT LIKE 'prober-%'"
+                     + " AND COALESCE(%resourceField%.lastEppUpdateClientId, '') NOT LIKE"
+                     + " 'prober-%' "
+                        + (historyClass == DomainHistory.class
+                            ? "AND %resourceField%.tld IN "
+                                + "(SELECT id FROM Tld WHERE tldType = 'REAL')"
+                            : ""))
+                    .replace("%entity%", historyClass.getSimpleName())
+                    .replace("%repoIdField%", repoIdFieldName)
+                    .replace("%resourceField%", resourceFieldName),
+                ImmutableMap.of("watermark", watermark),
+                Object[].class,
+                row -> KV.of((String) row[0], (long) row[1])))
+        .setCoder(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()));
   }
 
-  <T extends EppResource> PCollection<VKey<T>> createInputs(Pipeline pipeline, Class<T> clazz) {
-    return pipeline.apply(
-        "Read all production " + clazz.getSimpleName() + " entities",
-        RegistryJpaIO.read(
-            createEppResourceQuery(clazz),
-            clazz.equals(DomainBase.class)
-                ? ImmutableMap.of("tlds", pendings.keySet())
-                : ImmutableMap.of(),
-            String.class,
-            // TODO: consider adding coders for entities and pass them directly instead of using
-            // VKeys.
-            x -> VKey.createSql(clazz, x)));
+  private <T extends HistoryEntry> EppResource loadResourceByHistoryEntryId(
+      Class<T> historyEntryClazz, String repoId, long revisionId) {
+    try {
+      Class<?> idClazz = historyEntryClazz.getAnnotation(IdClass.class).value();
+      Serializable idObject =
+          (Serializable)
+              idClazz.getConstructor(String.class, long.class).newInstance(repoId, revisionId);
+      return jpaTm()
+          .transact(() -> jpaTm().loadByKey(VKey.createSql(historyEntryClazz, idObject)))
+          .getResourceAtPointInTime()
+          .map(resource -> resource.cloneProjectedAtTime(watermark))
+          .get();
+    } catch (NoSuchMethodException
+        | InvocationTargetException
+        | InstantiationException
+        | IllegalAccessException e) {
+      throw new RuntimeException(
+          String.format(
+              "Cannot load resource from %s with repoId %s and revisionId %s",
+              historyEntryClazz.getSimpleName(), repoId, revisionId),
+          e);
+    }
   }
 
-  <T extends EppResource>
-      FlatMapElements<VKey<T>, KV<PendingDeposit, DepositFragment>> mapToFragments(Class<T> clazz) {
-    return FlatMapElements.into(
-            TypeDescriptors.kvs(
-                TypeDescriptor.of(PendingDeposit.class), TypeDescriptor.of(DepositFragment.class)))
-        .via(
-            (VKey<T> key) -> {
-              T resource = jpaTm().transact(() -> jpaTm().loadByKey(key));
-              // The set of all TLDs to which this resource should be emitted.
-              ImmutableSet<String> tlds =
-                  clazz.equals(DomainBase.class)
-                      ? ImmutableSet.of(((DomainBase) resource).getTld())
-                      : pendings.keySet();
-              // Get the set of all point-in-time watermarks we need, to minimize rewinding.
-              ImmutableSet<DateTime> dates =
-                  tlds.stream()
-                      .map(pendings::get)
-                      .flatMap(ImmutableSet::stream)
-                      .map(PendingDeposit::watermark)
-                      .collect(toImmutableSet());
-              // Launch asynchronous fetches of point-in-time representations of resource.
-              ImmutableMap<DateTime, Supplier<EppResource>> resourceAtTimes =
-                  ImmutableMap.copyOf(
-                      Maps.asMap(dates, input -> loadAtPointInTimeAsync(resource, input)));
-              // Convert resource to an XML fragment for each watermark/mode pair lazily and cache
-              // the result.
-              RdeFragmenter fragmenter =
-                  new RdeFragmenter(resourceAtTimes, new RdeMarshaller(mode));
-              List<KV<PendingDeposit, DepositFragment>> results = new ArrayList<>();
-              for (String tld : tlds) {
-                for (PendingDeposit pending : pendings.get(tld)) {
-                  // Hosts and contacts don't get included in BRDA deposits.
-                  if (pending.mode() == RdeMode.THIN && !clazz.equals(DomainBase.class)) {
-                    continue;
+  /**
+   * Remove unreferenced resources by joining the (repoId, pendingDeposit) pair with the (repoId,
+   * revisionId) on the repoId.
+   *
+   * <p>The (repoId, pendingDeposit) pairs denote resources (contact, host) that are referenced from
+   * a domain, that are to be included in the corresponding pending deposit.
+   *
+   * <p>The (repoId, revisionId) paris come from the most recent history entry query, which can be
+   * used to load the embedded resources themselves.
+   *
+   * @return a pair of (repoId, ([pendingDeposit], [revisionId])) where neither the pendingDeposit
+   *     nor the revisionId list is empty.
+   */
+  private static PCollection<KV<String, CoGbkResult>> removeUnreferencedResource(
+      PCollection<KV<String, PendingDeposit>> referencedResources,
+      PCollection<KV<String, Long>> historyEntries,
+      Class<? extends EppResource> resourceClazz) {
+    String resourceName = resourceClazz.getSimpleName();
+    Class<? extends HistoryEntry> historyEntryClazz =
+        RESOURCE_TYPES_TO_HISTORY_TYPES.get(resourceClazz);
+    String historyEntryName = historyEntryClazz.getSimpleName();
+    Counter referencedResourceCounter = Metrics.counter("RDE", "Referenced" + resourceName);
+    return KeyedPCollectionTuple.of(PENDING_DEPOSIT, referencedResources)
+        .and(REVISION_ID, historyEntries)
+        .apply(
+            String.format(
+                "Join PendingDeposit with %s revision ID on %s", historyEntryName, resourceName),
+            CoGroupByKey.create())
+        .apply(
+            String.format("Remove unreferenced %s", resourceName),
+            Filter.by(
+                (KV<String, CoGbkResult> kv) -> {
+                  boolean toInclude =
+                      // If a resource does not have corresponding pending deposit, it is not
+                      // referenced and should not be included.
+                      kv.getValue().getAll(PENDING_DEPOSIT).iterator().hasNext()
+                          // If a resource does not have revision id (this should not happen, as
+                          // every referenced resource must be valid at watermark time, therefore
+                          // be embedded in a history entry valid at watermark time, otherwise
+                          // the domain cannot reference it), there is no way for us to find the
+                          // history entry and load the embedded resource. So we ignore the resource
+                          // to keep the downstream process simple.
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedResourceCounter.inc();
                   }
-                  Optional<DepositFragment> fragment =
-                      fragmenter.marshal(pending.watermark(), pending.mode());
-                  fragment.ifPresent(
-                      depositFragment -> results.add(KV.of(pending, depositFragment)));
-                }
-              }
-              return results;
-            });
+                  return toInclude;
+                }));
+  }
+
+  private PCollectionTuple processDomainHistories(PCollection<KV<String, Long>> domainHistories) {
+    Counter activeDomainCounter = Metrics.counter("RDE", "ActiveDomainBase");
+    Counter domainFragmentCounter = Metrics.counter("RDE", "DomainFragment");
+    Counter referencedContactCounter = Metrics.counter("RDE", "ReferencedContactResource");
+    Counter referencedHostCounter = Metrics.counter("RDE", "ReferencedHostResource");
+    return domainHistories.apply(
+        "Map DomainHistory to DepositFragment "
+            + "and emit referenced ContactResource and HostResource",
+        ParDo.of(
+                new DoFn<KV<String, Long>, KV<PendingDeposit, DepositFragment>>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Long> kv, MultiOutputReceiver receiver) {
+                    activeDomainCounter.inc();
+                    DomainBase domain =
+                        (DomainBase)
+                            loadResourceByHistoryEntryId(
+                                DomainHistory.class, kv.getKey(), kv.getValue());
+                    pendingDeposits.stream()
+                        .filter(pendingDeposit -> pendingDeposit.tld().equals(domain.getTld()))
+                        .forEach(
+                            pendingDeposit -> {
+                              // Domains are always deposited in both modes.
+                              domainFragmentCounter.inc();
+                              receiver
+                                  .get(DOMAIN_FRAGMENTS)
+                                  .output(
+                                      KV.of(
+                                          pendingDeposit,
+                                          marshaller.marshalDomain(domain, pendingDeposit.mode())));
+                              // Contacts and hosts are only deposited in RDE, not BRDA.
+                              if (pendingDeposit.mode() == RdeMode.FULL) {
+                                HashSet<Serializable> contacts = new HashSet<>();
+                                contacts.add(domain.getAdminContact().getSqlKey());
+                                contacts.add(domain.getTechContact().getSqlKey());
+                                contacts.add(domain.getRegistrant().getSqlKey());
+                                // Billing contact is not mandatory.
+                                if (domain.getBillingContact() != null) {
+                                  contacts.add(domain.getBillingContact().getSqlKey());
+                                }
+                                referencedContactCounter.inc(contacts.size());
+                                contacts.forEach(
+                                    contactRepoId ->
+                                        receiver
+                                            .get(REFERENCED_CONTACTS)
+                                            .output(KV.of((String) contactRepoId, pendingDeposit)));
+                                if (domain.getNsHosts() != null) {
+                                  referencedHostCounter.inc(domain.getNsHosts().size());
+                                  domain
+                                      .getNsHosts()
+                                      .forEach(
+                                          hostKey ->
+                                              receiver
+                                                  .get(REFERENCED_HOSTS)
+                                                  .output(
+                                                      KV.of(
+                                                          (String) hostKey.getSqlKey(),
+                                                          pendingDeposit)));
+                                }
+                              }
+                            });
+                  }
+                })
+            .withOutputTags(
+                DOMAIN_FRAGMENTS, TupleTagList.of(REFERENCED_CONTACTS).and(REFERENCED_HOSTS)));
+  }
+
+  private PCollection<KV<PendingDeposit, DepositFragment>> processContactHistories(
+      PCollection<KV<String, PendingDeposit>> referencedContacts,
+      PCollection<KV<String, Long>> contactHistories) {
+    Counter contactFragmentCounter = Metrics.counter("RDE", "ContactFragment");
+    return removeUnreferencedResource(referencedContacts, contactHistories, ContactResource.class)
+        .apply(
+            "Map ContactResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      ContactResource contact =
+                          (ContactResource)
+                              loadResourceByHistoryEntryId(
+                                  ContactHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      DepositFragment fragment = marshaller.marshalContact(contact);
+                      ImmutableSet<KV<PendingDeposit, DepositFragment>> fragments =
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same contact could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .map(pendingDeposit -> KV.of(pendingDeposit, fragment))
+                              .collect(toImmutableSet());
+                      contactFragmentCounter.inc(fragments.size());
+                      return fragments;
+                    }));
+  }
+
+  private PCollectionTuple processHostHistories(
+      PCollection<KV<String, PendingDeposit>> referencedHosts,
+      PCollection<KV<String, Long>> hostHistories) {
+    Counter subordinateHostCounter = Metrics.counter("RDE", "SubordinateHostResource");
+    Counter externalHostCounter = Metrics.counter("RDE", "ExternalHostResource");
+    Counter externalHostFragmentCounter = Metrics.counter("RDE", "ExternalHostFragment");
+    return removeUnreferencedResource(referencedHosts, hostHistories, HostResource.class)
+        .apply(
+            "Map external DomainResource to DepositFragment and process subordinate domains",
+            ParDo.of(
+                    new DoFn<KV<String, CoGbkResult>, KV<PendingDeposit, DepositFragment>>() {
+                      @ProcessElement
+                      public void processElement(
+                          @Element KV<String, CoGbkResult> kv, MultiOutputReceiver receiver) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    kv.getKey(),
+                                    kv.getValue().getOnly(REVISION_ID));
+                        // When a host is subordinate, we need to find it's superordinate domain and
+                        // include it in the deposit as well.
+                        if (host.isSubordinate()) {
+                          subordinateHostCounter.inc();
+                          receiver
+                              .get(SUPERORDINATE_DOMAINS)
+                              .output(
+                                  // The output are pairs of
+                                  // (superordinateDomainRepoId,
+                                  //   (subordinateHostRepoId, (pendingDeposit, revisionId))).
+                                  KV.of((String) host.getSuperordinateDomain().getSqlKey(), kv));
+                        } else {
+                          externalHostCounter.inc();
+                          DepositFragment fragment = marshaller.marshalExternalHost(host);
+                          Streams.stream(kv.getValue().getAll(PENDING_DEPOSIT))
+                              // The same host could be used by multiple domains, therefore
+                              // matched to the same pending deposit multiple times.
+                              .distinct()
+                              .forEach(
+                                  pendingDeposit -> {
+                                    externalHostFragmentCounter.inc();
+                                    receiver
+                                        .get(EXTERNAL_HOST_FRAGMENTS)
+                                        .output(KV.of(pendingDeposit, fragment));
+                                  });
+                        }
+                      }
+                    })
+                .withOutputTags(EXTERNAL_HOST_FRAGMENTS, TupleTagList.of(SUPERORDINATE_DOMAINS)));
+  }
+
+  /**
+   * Process subordinate hosts by making a deposit fragment with pending transfer information
+   * obtained from its superordinate domain.
+   *
+   * @param superordinateDomains Pairs of (superordinateDomainRepoId, (subordinateHostRepoId,
+   *     (pendingDeposit, revisionId))). This collection maps the subordinate host and the pending
+   *     deposit to include it to its superordinate domain.
+   * @param domainHistories Pairs of (domainRepoId, revisionId). This collection helps us find the
+   *     historical superordinate domain from its history entry and is obtained from calling {@link
+   *     #getMostRecentHistoryEntries} for domains.
+   */
+  private PCollection<KV<PendingDeposit, DepositFragment>> processSubordinateHosts(
+      PCollection<KV<String, KV<String, CoGbkResult>>> superordinateDomains,
+      PCollection<KV<String, Long>> domainHistories) {
+    Counter subordinateHostFragmentCounter = Metrics.counter("RDE", "SubordinateHostFragment");
+    Counter referencedSubordinateHostCounter = Metrics.counter("RDE", "ReferencedSubordinateHost");
+    return KeyedPCollectionTuple.of(HOST_TO_PENDING_DEPOSIT, superordinateDomains)
+        .and(REVISION_ID, domainHistories)
+        .apply(
+            "Join HostResource:PendingDeposits with DomainHistory on DomainResource",
+            CoGroupByKey.create())
+        .apply(
+            " Remove unreferenced DomainResource",
+            Filter.by(
+                kv -> {
+                  boolean toInclude =
+                      kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT).iterator().hasNext()
+                          && kv.getValue().getAll(REVISION_ID).iterator().hasNext();
+                  if (toInclude) {
+                    referencedSubordinateHostCounter.inc();
+                  }
+                  return toInclude;
+                }))
+        .apply(
+            "Map subordinate HostResource to DepositFragment",
+            FlatMapElements.into(
+                    kvs(
+                        TypeDescriptor.of(PendingDeposit.class),
+                        TypeDescriptor.of(DepositFragment.class)))
+                .via(
+                    (KV<String, CoGbkResult> kv) -> {
+                      DomainBase superordinateDomain =
+                          (DomainBase)
+                              loadResourceByHistoryEntryId(
+                                  DomainHistory.class,
+                                  kv.getKey(),
+                                  kv.getValue().getOnly(REVISION_ID));
+                      ImmutableSet.Builder<KV<PendingDeposit, DepositFragment>> results =
+                          new ImmutableSet.Builder<>();
+                      for (KV<String, CoGbkResult> hostToPendingDeposits :
+                          kv.getValue().getAll(HOST_TO_PENDING_DEPOSIT)) {
+                        HostResource host =
+                            (HostResource)
+                                loadResourceByHistoryEntryId(
+                                    HostHistory.class,
+                                    hostToPendingDeposits.getKey(),
+                                    hostToPendingDeposits.getValue().getOnly(REVISION_ID));
+                        DepositFragment fragment =
+                            marshaller.marshalSubordinateHost(host, superordinateDomain);
+                        Streams.stream(hostToPendingDeposits.getValue().getAll(PENDING_DEPOSIT))
+                            .distinct()
+                            .forEach(
+                                pendingDeposit -> {
+                                  subordinateHostFragmentCounter.inc();
+                                  results.add(KV.of(pendingDeposit, fragment));
+                                });
+                      }
+                      return results.build();
+                    }));
   }
 
   /**
    * Decodes the pipeline option extracted from the URL parameter sent by the pipeline launcher to
-   * the original TLD to pending deposit map.
+   * the original pending deposit set.
    */
   @SuppressWarnings("unchecked")
-  static ImmutableSetMultimap<String, PendingDeposit> decodePendings(String encodedPending) {
+  static ImmutableSet<PendingDeposit> decodePendingDeposits(String encodedPendingDeposits) {
     try (ObjectInputStream ois =
         new ObjectInputStream(
             new ByteArrayInputStream(
-                BaseEncoding.base64Url().omitPadding().decode(encodedPending)))) {
-      return (ImmutableSetMultimap<String, PendingDeposit>) ois.readObject();
+                BaseEncoding.base64Url().omitPadding().decode(encodedPendingDeposits)))) {
+      return (ImmutableSet<PendingDeposit>) ois.readObject();
     } catch (IOException | ClassNotFoundException e) {
       throw new IllegalArgumentException("Unable to parse encoded pending deposit map.", e);
     }
   }
 
   /**
-   * Encodes the TLD to pending deposit map in an URL safe string that is sent to the pipeline
-   * worker by the pipeline launcher as a pipeline option.
+   * Encodes the pending deposit set in an URL safe string that is sent to the pipeline worker by
+   * the pipeline launcher as a pipeline option.
    */
-  public static String encodePendings(ImmutableSetMultimap<String, PendingDeposit> pendings)
+  public static String encodePendingDeposits(ImmutableSet<PendingDeposit> pendingDeposits)
       throws IOException {
     try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
       ObjectOutputStream oos = new ObjectOutputStream(baos);
-      oos.writeObject(pendings);
+      oos.writeObject(pendingDeposits);
       oos.flush();
       return BaseEncoding.base64Url().omitPadding().encode(baos.toByteArray());
     }
@@ -284,14 +686,40 @@ public class RdePipeline implements Serializable {
     PipelineOptionsFactory.register(RdePipelineOptions.class);
     RdePipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(RdePipelineOptions.class);
-    // RegistryPipelineWorkerInitializer only initializes before pipeline executions, after the
-    // main() function constructed the graph. We need the registry environment set up so that we
-    // can create a CloudTasksUtils which uses the environment-dependent config file.
-    options.getRegistryEnvironment().setup();
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
     options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
     DaggerRdePipeline_RdePipelineComponent.builder().options(options).build().rdePipeline().run();
   }
 
+  /**
+   * A utility class that contains {@link TupleTag}s when {@link PCollectionTuple}s and {@link
+   * CoGbkResult}s are used.
+   */
+  protected abstract static class TupleTags {
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> DOMAIN_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_CONTACTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, PendingDeposit>> REFERENCED_HOSTS =
+        new TupleTag<KV<String, PendingDeposit>>() {};
+
+    protected static final TupleTag<KV<String, KV<String, CoGbkResult>>> SUPERORDINATE_DOMAINS =
+        new TupleTag<KV<String, KV<String, CoGbkResult>>>() {};
+
+    protected static final TupleTag<KV<PendingDeposit, DepositFragment>> EXTERNAL_HOST_FRAGMENTS =
+        new TupleTag<KV<PendingDeposit, DepositFragment>>() {};
+
+    protected static final TupleTag<PendingDeposit> PENDING_DEPOSIT =
+        new TupleTag<PendingDeposit>() {};
+
+    protected static final TupleTag<KV<String, CoGbkResult>> HOST_TO_PENDING_DEPOSIT =
+        new TupleTag<KV<String, CoGbkResult>>() {};
+
+    protected static final TupleTag<Long> REVISION_ID = new TupleTag<Long>() {};
+  }
+
   @Singleton
   @Component(
       modules = {

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -61,7 +61,8 @@ import org.json.JSONObject;
 /**
  * Definition of a Dataflow Flex template, which generates a given month's spec11 report.
  *
- * <p>To stage this template locally, run the {@code stage_beam_pipeline.sh} shell script.
+ * <p>To stage this template locally, run {@code ./nom_build :core:sBP --environment=alpha
+ * --pipeline=spec11}.
  *
  * <p>Then, you can run the staged template via the API client library, gCloud or a raw REST call.
  *

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -557,11 +557,23 @@ public final class RegistryConfig {
       return config.registryPolicy.requireSslCertificates;
     }
 
+    /**
+     * Returns the GCE machine type that a CPU-demanding pipeline should use.
+     *
+     * @see google.registry.beam.rde.RdePipeline
+     */
+    @Provides
+    @Config("highPerformanceMachineType")
+    public static String provideHighPerformanceMachineType(RegistryConfigSettings config) {
+      return config.beam.highPerformanceMachineType;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *
      * @see google.registry.beam.invoicing.InvoicingPipeline
      * @see google.registry.beam.spec11.Spec11Pipeline
+     * @see google.registry.beam.invoicing.InvoicingPipeline
      */
     @Provides
     @Config("defaultJobRegion")

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -133,6 +133,7 @@ public class RegistryConfigSettings {
   /** Configuration for Apache Beam (Cloud Dataflow). */
   public static class Beam {
     public String defaultJobRegion;
+    public String highPerformanceMachineType;
     public String stagingBucketUrl;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -419,6 +419,14 @@ misc:
 beam:
   # The default region to run Apache Beam (Cloud Dataflow) jobs in.
   defaultJobRegion: us-east1
+  # The GCE machine type to use when a job is CPU-intensive (e. g. RDE). Be sure
+  # to check the VM CPU quota for the job region. In a massively parallel
+  # pipeline this quota can be easily reached and needs to be raised, otherwise
+  # the job will run very slowly. Also note that there is a separate quota for
+  # external IPv4 address in a region, which means that machine type with higher
+  # core count per machine may be preferable in order to preserve IP addresses.
+  # See: https://cloud.google.com/compute/quotas#cpu_quota
+  highPerformanceMachineType: n2-standard-4
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:

core/src/main/java/google/registry/cron/CommitLogFanoutAction.java

@@ -14,18 +14,15 @@
 
 package google.registry.cron;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskOptions;
+import com.google.common.collect.ImmutableMultimap;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.request.Action;
+import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
 import google.registry.request.auth.Auth;
-import google.registry.util.TaskQueueUtils;
-import java.time.Duration;
+import google.registry.util.Clock;
+import google.registry.util.CloudTasksUtils;
 import java.util.Optional;
-import java.util.Random;
 import javax.inject.Inject;
 
 /** Action for fanning out cron tasks for each commit log bucket. */
@@ -38,25 +35,27 @@ public final class CommitLogFanoutAction implements Runnable {
 
   public static final String BUCKET_PARAM = "bucket";
 
-  private static final Random random = new Random();
+  @Inject Clock clock;
+  @Inject CloudTasksUtils cloudTasksUtils;
 
-  @Inject TaskQueueUtils taskQueueUtils;
   @Inject @Parameter("endpoint") String endpoint;
   @Inject @Parameter("queue") String queue;
   @Inject @Parameter("jitterSeconds") Optional<Integer> jitterSeconds;
   @Inject CommitLogFanoutAction() {}
 
+
+
   @Override
   public void run() {
-    Queue taskQueue = getQueue(queue);
     for (int bucketId : CommitLogBucket.getBucketIds()) {
-      long delay =
-          jitterSeconds.map(i -> random.nextInt((int) Duration.ofSeconds(i).toMillis())).orElse(0);
-      TaskOptions taskOptions =
-          TaskOptions.Builder.withUrl(endpoint)
-              .param(BUCKET_PARAM, Integer.toString(bucketId))
-              .countdownMillis(delay);
-      taskQueueUtils.enqueue(taskQueue, taskOptions);
+      cloudTasksUtils.enqueue(
+          queue,
+          CloudTasksUtils.createPostTask(
+              endpoint,
+              Service.BACKEND.toString(),
+              ImmutableMultimap.of(BUCKET_PARAM, Integer.toString(bucketId)),
+              clock,
+              jitterSeconds));
     }
   }
 }

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -157,6 +157,12 @@
     <url-pattern>/_dr/cron/readDnsQueue</url-pattern>
   </servlet-mapping>
 
+  <!-- Replicates SQL transactions to Datastore during the Registry 3.0 migration. -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/cron/replicateToDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Publishes DNS updates. -->
   <servlet-mapping>
     <servlet-name>backend-servlet</servlet-name>

core/src/main/java/google/registry/flows/EppController.java

@@ -143,9 +143,6 @@ public final class EppController {
   /** Creates a response indicating an EPP failure. */
   @VisibleForTesting
   static EppOutput getErrorResponse(Result result, Trid trid) {
-    return EppOutput.create(new EppResponse.Builder()
-        .setResult(result)
-        .setTrid(trid)
-        .build());
+    return EppOutput.create(new EppResponse.Builder().setResult(result).setTrid(trid).build());
   }
 }

core/src/main/java/google/registry/flows/EppException.java

@@ -25,10 +25,12 @@ import google.registry.model.eppinput.EppInput.InnerCommand;
 import google.registry.model.eppinput.EppInput.ResourceCommandWrapper;
 import google.registry.model.eppoutput.Result;
 import google.registry.model.eppoutput.Result.Code;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Inherited;
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
+import javax.annotation.Nullable;
 
 /** Exception used to propagate all failures containing one or more EPP responses. */
 public abstract class EppException extends Exception {
@@ -37,7 +39,12 @@ public abstract class EppException extends Exception {
 
   /** Create an EppException with a custom message. */
   private EppException(String message) {
-    super(message);
+    this(message, null);
+  }
+
+  /** Create an EppException with a custom message and cause. */
+  private EppException(String message, @Nullable Throwable cause) {
+    super(message, cause);
     Code code = getClass().getAnnotation(EppResultCode.class).value();
     Preconditions.checkState(!code.isSuccess());
     this.result = Result.create(code, message);
@@ -255,4 +262,12 @@ public abstract class EppException extends Exception {
       super("Specified protocol version is not implemented");
     }
   }
+
+  /** Registry is currently undergoing maintenance and is in read-only mode. */
+  @EppResultCode(Code.COMMAND_FAILED)
+  public static class ReadOnlyModeEppException extends EppException {
+    ReadOnlyModeEppException(ReadOnlyModeException cause) {
+      super("Registry is currently undergoing maintenance and is in read-only mode", cause);
+    }
+  }
 }

core/src/main/java/google/registry/flows/FlowRunner.java

@@ -19,6 +19,7 @@ import static google.registry.xml.XmlTransformer.prettyPrint;
 
 import com.google.common.base.Strings;
 import com.google.common.flogger.FluentLogger;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.FlowModule.DryRun;
 import google.registry.flows.FlowModule.InputXml;
 import google.registry.flows.FlowModule.RegistrarId;
@@ -28,6 +29,7 @@ import google.registry.flows.session.LoginFlow;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppOutput;
 import google.registry.monitoring.whitebox.EppMetric;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import javax.inject.Inject;
 import javax.inject.Provider;
 
@@ -97,6 +99,8 @@ public class FlowRunner {
       return e.output;
     } catch (EppRuntimeException e) {
       throw e.getCause();
+    } catch (ReadOnlyModeException e) {
+      throw new ReadOnlyModeEppException(e);
     }
   }
 

core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java

@@ -50,6 +50,7 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that creates a new contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link ContactFlowUtils.BadInternationalizedPostalInfoException}

core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java

@@ -60,6 +60,7 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java

@@ -54,6 +54,7 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * explicitly approve the transfer request, which then becomes effective immediately.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}

core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java

@@ -54,6 +54,7 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the gaining client to
  * withdraw the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java

@@ -53,6 +53,7 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * reject the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java

@@ -63,6 +63,7 @@ import org.joda.time.Duration;
  * by the losing registrar or rejected, and the gaining registrar can also cancel the transfer
  * request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.AlreadyPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java

@@ -55,6 +55,7 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that updates a contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -133,6 +133,7 @@ import org.joda.time.Duration;
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException}
  * @error {@link
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException}
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -103,6 +103,7 @@ import org.joda.time.Duration;
 /**
  * An EPP flow that deletes a domain.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
@@ -187,7 +188,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
     } else {
       DateTime redemptionTime = now.plus(redemptionGracePeriodLength);
       asyncTaskEnqueuer.enqueueAsyncResave(
-          existingDomain, now, ImmutableSortedSet.of(redemptionTime, deletionTime));
+          existingDomain.createVKey(), now, ImmutableSortedSet.of(redemptionTime, deletionTime));
       builder
           .setDeletionTime(deletionTime)
           .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE))

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -94,6 +94,7 @@ import org.joda.time.Duration;
  * longer than 10 years unless it comes in at the exact millisecond that the domain would have
  * expired.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -93,6 +93,7 @@ import org.joda.time.DateTime;
  * restored to a single year expiration starting at the restore time, regardless of what the
  * original expiration time was.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -110,7 +111,7 @@ import org.joda.time.DateTime;
  * @error {@link DomainRestoreRequestFlow.RestoreCommandIncludesChangesException}
  */
 @ReportingSpec(ActivityReportField.DOMAIN_RGP_RESTORE_REQUEST)
-public final class DomainRestoreRequestFlow implements TransactionalFlow  {
+public final class DomainRestoreRequestFlow implements TransactionalFlow {
 
   @Inject ResourceCommand resourceCommand;
   @Inject ExtensionManager extensionManager;

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -78,6 +78,7 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted and replaced with new ones with the correct approval time.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -65,6 +65,7 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java

@@ -67,6 +67,7 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -93,6 +93,7 @@ import org.joda.time.DateTime;
  * rejection or cancellation of the request, they will be deleted (and in the approval case,
  * replaced with new ones with the correct approval time).
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -102,7 +103,8 @@ import org.joda.time.DateTime;
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}
  * @error {@link google.registry.flows.exceptions.TransferPeriodMustBeOneYearException}
  * @error {@link InvalidTransferPeriodValueException}
- * @error {@link google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
+ * @error {@link
+ *     google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
  * @error {@link DomainFlowUtils.BadPeriodUnitException}
  * @error {@link DomainFlowUtils.CurrencyUnitMismatchException}
  * @error {@link DomainFlowUtils.CurrencyValueScaleException}
@@ -235,7 +237,7 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
             .build();
     DomainHistory domainHistory = buildDomainHistory(newDomain, registry, now, period);
 
-    asyncTaskEnqueuer.enqueueAsyncResave(newDomain, now, automaticTransferTime);
+    asyncTaskEnqueuer.enqueueAsyncResave(newDomain.createVKey(), now, automaticTransferTime);
     tm().putAll(
             new ImmutableSet.Builder<>()
                 .add(newDomain, domainHistory, requestPollMessage)

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -16,6 +16,7 @@ package google.registry.flows.domain;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.collect.ImmutableSortedSet.toImmutableSortedSet;
 import static com.google.common.collect.Sets.symmetricDifference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.flows.FlowUtils.persistEntityChanges;
@@ -43,6 +44,9 @@ import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_UPDATE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
 import com.google.common.net.InternetDomainName;
 import google.registry.dns.DnsQueue;
 import google.registry.flows.EppException;
@@ -76,6 +80,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
+import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
 import google.registry.model.tld.Registry;
 import java.util.Optional;
@@ -92,6 +97,7 @@ import org.joda.time.DateTime;
  * superuser. As such, adding or removing these statuses incurs a billing event. There will be only
  * one charge per update, even if several such statuses are updated at once.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -175,6 +181,9 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     Optional<BillingEvent.OneTime> statusUpdateBillingEvent =
         createBillingEventForStatusUpdates(existingDomain, newDomain, domainHistory, now);
     statusUpdateBillingEvent.ifPresent(entitiesToSave::add);
+    Optional<PollMessage.OneTime> serverStatusUpdatePollMessage =
+        createPollMessageForServerStatusUpdates(existingDomain, newDomain, domainHistory, now);
+    serverStatusUpdatePollMessage.ifPresent(entitiesToSave::add);
     EntityChanges entityChanges =
         flowCustomLogic.beforeSave(
             BeforeSaveParameters.newBuilder()
@@ -306,4 +315,50 @@ public final class DomainUpdateFlow implements TransactionalFlow {
     }
     return Optional.empty();
   }
+
+  /** Enqueues a poll message iff a superuser is adding/removing server statuses. */
+  private Optional<PollMessage.OneTime> createPollMessageForServerStatusUpdates(
+      DomainBase existingDomain, DomainBase newDomain, DomainHistory historyEntry, DateTime now) {
+    if (registrarId.equals(existingDomain.getPersistedCurrentSponsorRegistrarId())) {
+      // Don't send a poll message when a superuser registrar is updating its own domain.
+      return Optional.empty();
+    }
+    ImmutableSortedSet<String> addedServerStatuses =
+        Sets.difference(newDomain.getStatusValues(), existingDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+    ImmutableSortedSet<String> removedServerStatuses =
+        Sets.difference(existingDomain.getStatusValues(), newDomain.getStatusValues()).stream()
+            .filter(StatusValue::isServerSettable)
+            .map(StatusValue::getXmlName)
+            .collect(toImmutableSortedSet(Ordering.natural()));
+
+    String msg = "";
+    if (addedServerStatuses.size() > 0 && removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s and removed the status(es)"
+                  + " %s.",
+              addedServerStatuses, removedServerStatuses);
+    } else if (addedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has added the status(es) %s.", addedServerStatuses);
+    } else if (removedServerStatuses.size() > 0) {
+      msg =
+          String.format(
+              "The registry administrator has removed the status(es) %s.", removedServerStatuses);
+    } else {
+      return Optional.empty();
+    }
+
+    return Optional.ofNullable(
+        new PollMessage.OneTime.Builder()
+            .setParent(historyEntry)
+            .setEventTime(now)
+            .setRegistrarId(existingDomain.getCurrentSponsorRegistrarId())
+            .setMsg(msg)
+            .build());
+  }
 }

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -65,6 +65,7 @@ import org.joda.time.DateTime;
  * hosts cannot have any. This flow allows creating a host name, and if necessary enqueues tasks to
  * update DNS.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.IpAddressVersionMismatchException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -58,6 +58,7 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -73,11 +73,12 @@ import org.joda.time.DateTime;
  * hosts. Internal hosts must have at least one IP address associated with them, whereas external
  * hosts cannot have any.
  *
- * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When
- * a host is renamed from internal to external all IP addresses must be simultaneously removed, and
+ * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When a
+ * host is renamed from internal to external all IP addresses must be simultaneously removed, and
  * when it is renamed from external to internal at least one must be added. If the host is renamed
  * or IP addresses are added, tasks are enqueued to update DNS accordingly.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/model/CreateAutoTimestamp.java

@@ -14,8 +14,20 @@
 
 package google.registry.model;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.OnLoad;
 import google.registry.model.translators.CreateAutoTimestampTranslatorFactory;
+import google.registry.util.DateTimeUtils;
+import java.time.ZonedDateTime;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.PostLoad;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -23,9 +35,37 @@ import org.joda.time.DateTime;
  *
  * @see CreateAutoTimestampTranslatorFactory
  */
+@Embeddable
 public class CreateAutoTimestamp extends ImmutableObject implements UnsafeSerializable {
 
-  DateTime timestamp;
+  @Transient DateTime timestamp;
+
+  @Column(nullable = false)
+  @Ignore
+  ZonedDateTime creationTime;
+
+  @PrePersist
+  @PreUpdate
+  void setTimestamp() {
+    if (creationTime == null) {
+      timestamp = jpaTm().getTransactionTime();
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @OnLoad
+  void onLoad() {
+    if (timestamp != null) {
+      creationTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (creationTime != null) {
+      timestamp = DateTimeUtils.toJodaDateTime(creationTime);
+    }
+  }
 
   /** Returns the timestamp. */
   @Nullable
@@ -36,6 +76,7 @@ public class CreateAutoTimestamp extends ImmutableObject implements UnsafeSerial
   public static CreateAutoTimestamp create(@Nullable DateTime timestamp) {
     CreateAutoTimestamp instance = new CreateAutoTimestamp();
     instance.timestamp = timestamp;
+    instance.creationTime = (timestamp == null) ? null : DateTimeUtils.toZonedDateTime(timestamp);
     return instance;
   }
 }

core/src/main/java/google/registry/model/EntityClasses.java

@@ -43,7 +43,6 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.server.Lock;
 import google.registry.model.server.ServerSecret;
 import google.registry.model.tld.Registry;
-import google.registry.model.tmch.TmchCrl;
 
 /** Sets of classes of the Objectify-registered entities in use throughout the model. */
 public final class EntityClasses {
@@ -85,8 +84,7 @@ public final class EntityClasses {
           Registrar.class,
           RegistrarContact.class,
           Registry.class,
-          ServerSecret.class,
-          TmchCrl.class);
+          ServerSecret.class);
 
   private EntityClasses() {}
 }

core/src/main/java/google/registry/model/EppResource.java

@@ -50,6 +50,8 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
@@ -112,7 +114,9 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
    * <p>This can be null in the case of pre-Registry-3.0-migration history objects with null
    * resource fields.
    */
-  @Index CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
+  @AttributeOverrides({@AttributeOverride(name = "creationTime", column = @Column())})
+  @Index
+  CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /**
    * The time when this resource was or will be deleted.
@@ -224,10 +228,8 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /** Used when replaying from SQL to DS to populate the Datastore indexes. */
   protected void saveIndexesToDatastore() {
-    ofyTm()
-        .putAll(
-            ForeignKeyIndex.create(this, getDeletionTime()),
-            EppResourceIndex.create(Key.create(this)));
+    ofyTm().putIgnoringReadOnly(ForeignKeyIndex.create(this, getDeletionTime()));
+    ofyTm().putIgnoringReadOnly(EppResourceIndex.create(Key.create(this)));
   }
 
   /** EppResources that are loaded via foreign keys should implement this marker interface. */

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -27,7 +27,6 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -77,14 +76,29 @@ public abstract class BillingEvent extends ImmutableObject
 
   /** The reason for the bill, which maps 1:1 to skus in go/registry-billing-skus. */
   public enum Reason {
-    CREATE,
+    CREATE(true),
     @Deprecated // TODO(b/31676071): remove this legacy value once old data is cleaned up.
-    ERROR,
-    FEE_EARLY_ACCESS,
-    RENEW,
-    RESTORE,
-    SERVER_STATUS,
-    TRANSFER
+    ERROR(false),
+    FEE_EARLY_ACCESS(true),
+    RENEW(true),
+    RESTORE(true),
+    SERVER_STATUS(false),
+    TRANSFER(true);
+
+    private final boolean requiresPeriod;
+
+    Reason(boolean requiresPeriod) {
+      this.requiresPeriod = requiresPeriod;
+    }
+
+    /**
+     * Returns whether billing events with this reason have a period years associated with them.
+     *
+     * <p>Note that this is an "if an only if" condition.
+     */
+    public boolean hasPeriodYears() {
+      return requiresPeriod;
+    }
   }
 
   /** Set of flags that can be applied to billing events. */
@@ -268,7 +282,7 @@ public abstract class BillingEvent extends ImmutableObject
     public T build() {
       T instance = getInstance();
       checkNotNull(instance.reason, "Reason must be set");
-      checkNotNull(instance.clientId, "Client ID must be set");
+      checkNotNull(instance.clientId, "Registrar ID must be set");
       checkNotNull(instance.eventTime, "Event time must be set");
       checkNotNull(instance.targetId, "Target ID must be set");
       checkNotNull(instance.parent, "Parent must be set");
@@ -462,17 +476,14 @@ public abstract class BillingEvent extends ImmutableObject
         checkNotNull(instance.billingTime);
         checkNotNull(instance.cost);
         checkState(!instance.cost.isNegative(), "Costs should be non-negative.");
-        ImmutableSet<Reason> reasonsWithPeriods =
-            Sets.immutableEnumSet(
-                Reason.CREATE,
-                Reason.FEE_EARLY_ACCESS,
-                Reason.RENEW,
-                Reason.RESTORE,
-                Reason.TRANSFER);
-        checkState(
-            reasonsWithPeriods.contains(instance.reason) == (instance.periodYears != null),
-            "Period years must be set if and only if reason is "
-                + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        // TODO(mcilwain): Enforce this check on all billing events (not just more recent ones)
+        //                 post-migration after we add the missing period years values in SQL.
+        if (instance.eventTime.isAfter(DateTime.parse("2019-01-01T00:00:00Z"))) {
+          checkState(
+              instance.reason.hasPeriodYears() == (instance.periodYears != null),
+              "Period years must be set if and only if reason is "
+                  + "CREATE, FEE_EARLY_ACCESS, RENEW, RESTORE or TRANSFER.");
+        }
         checkState(
             instance.getFlags().contains(Flag.SYNTHETIC)
                 == (instance.syntheticCreationTime != null),

core/src/main/java/google/registry/model/domain/RegistryLock.java

@@ -28,6 +28,8 @@ import google.registry.util.DateTimeUtils;
 import java.time.ZonedDateTime;
 import java.util.Optional;
 import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
@@ -100,7 +102,11 @@ public final class RegistryLock extends ImmutableObject implements Buildable, Sq
   private String registrarPocId;
 
   /** When the lock is first requested. */
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "lockRequestTime", nullable = false))
+  })
   private CreateAutoTimestamp lockRequestTime = CreateAutoTimestamp.create(null);
 
   /** When the unlock is first requested. */

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -122,7 +122,6 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @Nullable @Index String domainName;
 
   /** When this token was created. */
-  @Column(nullable = false)
   CreateAutoTimestamp creationTime = CreateAutoTimestamp.create(null);
 
   /** Allowed registrar client IDs for this token, or null if all registrars are allowed. */

core/src/main/java/google/registry/model/eppcommon/StatusValue.java

@@ -112,7 +112,6 @@ public enum StatusValue implements EppEnum {
    */
   PENDING_UPDATE(AllowedOn.NONE),
 
-
   /** A non-client-settable status that prevents deletes of EPP resources. */
   SERVER_DELETE_PROHIBITED(AllowedOn.ALL),
 
@@ -162,6 +161,10 @@ public enum StatusValue implements EppEnum {
     return xmlName.startsWith("client");
   }
 
+  public boolean isServerSettable() {
+    return xmlName.startsWith("server");
+  }
+
   public boolean isChargedStatus() {
     return xmlName.startsWith("server") && xmlName.endsWith("Prohibited");
   }

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -64,6 +64,7 @@ public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEn
     EppResourceIndex instance = instantiate(EppResourceIndex.class);
     instance.reference = resourceKey;
     instance.kind = resourceKey.getKind();
+    // TODO(b/207368050): figure out if this value has ever been used other than test cases
     instance.id = resourceKey.getString(); // creates a web-safe key string
     instance.bucket = bucket;
     return instance;

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -82,6 +82,7 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
       com.google.appengine.api.datastore.Entity rawEntity) {
     CommitLogMutation instance = new CommitLogMutation();
     instance.parent = checkNotNull(parent);
+    // TODO(b/207516684): figure out if this should be converted to a vkey string via stringify()
     // Creates a web-safe key string.
     instance.entityKey = KeyFactory.keyToString(rawEntity.getKey());
     instance.entityProtoBytes = convertToPb(rawEntity).toByteArray();
@@ -91,6 +92,8 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
   /** Returns the key of a mutation based on the {@code entityKey} of the entity it stores. */
   public static
       Key<CommitLogMutation> createKey(Key<CommitLogManifest> parent, Key<?> entityKey) {
+    // TODO(b/207516684): figure out if the return type needs to be VKey and
+    //  if the string used to create a key should remain the same
     return Key.create(parent, CommitLogMutation.class, entityKey.getString());
   }
 }

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -148,7 +148,9 @@ public class ReplicateToDatastoreAction implements Runnable {
 
                 // Write the updated last transaction id to Datastore as part of this Datastore
                 // transaction.
-                auditedOfy().save().entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
+                auditedOfy()
+                    .saveIgnoringReadOnly()
+                    .entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
                 logger.atInfo().log(
                     "Finished applying single transaction Cloud SQL -> Cloud Datastore.");
               });
@@ -194,7 +196,7 @@ public class ReplicateToDatastoreAction implements Runnable {
       response.setStatus(SC_INTERNAL_SERVER_ERROR);
       response.setPayload(message);
     } finally {
-      lock.ifPresent(Lock::release);
+      lock.ifPresent(Lock::releaseSql);
     }
   }
 

core/src/main/java/google/registry/model/tmch/ClaimsList.java

@@ -27,6 +27,8 @@ import google.registry.model.replay.SqlOnlyEntity;
 import google.registry.model.tld.label.ReservedList.ReservedListEntry;
 import java.util.Map;
 import java.util.Optional;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
@@ -56,7 +58,11 @@ public class ClaimsList extends ImmutableObject implements SqlOnlyEntity {
   @GeneratedValue(strategy = GenerationType.IDENTITY)
   Long revisionId;
 
-  @Column(nullable = false)
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "creationTime",
+        column = @Column(name = "creationTimestamp", nullable = false))
+  })
   CreateAutoTimestamp creationTimestamp = CreateAutoTimestamp.create(null);
 
   /**

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -16,25 +16,18 @@ package google.registry.model.tmch;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
-import com.googlecode.objectify.annotation.Entity;
-import google.registry.model.annotations.NotBackedUp;
-import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
-import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
 import javax.persistence.Column;
 import org.joda.time.DateTime;
 
-/** Datastore singleton for ICANN's TMCH CA certificate revocation list (CRL). */
-@Entity
+/** Singleton for ICANN's TMCH CA certificate revocation list (CRL). */
 @javax.persistence.Entity
 @Immutable
-@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEntity {
+public final class TmchCrl extends CrossTldSingleton implements SqlOnlyEntity {
 
   @Column(name = "certificateRevocations", nullable = false)
   String crl;
@@ -47,25 +40,23 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
 
   /** Returns the singleton instance of this entity, without memoization. */
   public static Optional<TmchCrl> get() {
-    return tm().transact(() -> tm().loadSingleton(TmchCrl.class));
+    return jpaTm().transact(() -> jpaTm().loadSingleton(TmchCrl.class));
   }
 
   /**
-   * Change the Datastore singleton to a new ASCII-armored X.509 CRL.
+   * Change the singleton to a new ASCII-armored X.509 CRL.
    *
    * <p>Please do not call this function unless your CRL is properly formatted, signed by the root,
    * and actually newer than the one currently in Datastore.
-   *
-   * <p>During the dual-write period, we write to both Datastore and SQL
    */
   public static void set(final String crl, final String url) {
-    tm().transact(
+    jpaTm()
+        .transact(
             () -> {
               TmchCrl tmchCrl = new TmchCrl();
-              tmchCrl.updated = tm().getTransactionTime();
+              tmchCrl.updated = jpaTm().getTransactionTime();
               tmchCrl.crl = checkNotNull(crl, "crl");
               tmchCrl.url = checkNotNull(url, "url");
-              ofyTm().transactNew(() -> ofyTm().putWithoutBackup(tmchCrl));
               jpaTm().transactNew(() -> jpaTm().putWithoutBackup(tmchCrl));
             });
   }
@@ -80,7 +71,7 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
     return crl;
   }
 
-  /** Time we last updated the Datastore with a newer ICANN CRL. */
+  /** Time we last updated the Database with a newer ICANN CRL. */
   public final DateTime getUpdated() {
     return updated;
   }

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -60,6 +60,7 @@ import google.registry.export.sheet.SheetModule;
 import google.registry.export.sheet.SyncRegistrarsSheetAction;
 import google.registry.flows.FlowComponent;
 import google.registry.mapreduce.MapreduceModule;
+import google.registry.model.replay.ReplicateToDatastoreAction;
 import google.registry.monitoring.whitebox.WhiteboxModule;
 import google.registry.rdap.UpdateRegistrarRdapBaseUrlsAction;
 import google.registry.rde.BrdaCopyAction;
@@ -190,6 +191,8 @@ interface BackendRequestComponent {
 
   ReplayCommitLogsToSqlAction replayCommitLogsToSqlAction();
 
+  ReplicateToDatastoreAction replicateToDatastoreAction();
+
   ResaveAllEppResourcesAction resaveAllEppResourcesAction();
 
   ResaveEntityAction resaveEntityAction();

core/src/main/java/google/registry/persistence/converter/CreateAutoTimestampConverter.java

@@ -1,53 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package google.registry.persistence.converter;
-
-import static com.google.common.base.MoreObjects.firstNonNull;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-
-import google.registry.model.CreateAutoTimestamp;
-import google.registry.util.DateTimeUtils;
-import java.sql.Timestamp;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-import javax.annotation.Nullable;
-import javax.persistence.AttributeConverter;
-import javax.persistence.Converter;
-import org.joda.time.DateTime;
-
-/** JPA converter to for storing/retrieving CreateAutoTimestamp objects. */
-@Converter(autoApply = true)
-public class CreateAutoTimestampConverter
-    implements AttributeConverter<CreateAutoTimestamp, Timestamp> {
-
-  @Override
-  @Nullable
-  public Timestamp convertToDatabaseColumn(@Nullable CreateAutoTimestamp entity) {
-    if (entity == null) {
-      return null;
-    }
-    DateTime dateTime = firstNonNull(entity.getTimestamp(), jpaTm().getTransactionTime());
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(dateTime).toInstant());
-  }
-
-  @Override
-  @Nullable
-  public CreateAutoTimestamp convertToEntityAttribute(@Nullable Timestamp columnValue) {
-    if (columnValue == null) {
-      return null;
-    }
-    ZonedDateTime zdt = ZonedDateTime.ofInstant(columnValue.toInstant(), ZoneOffset.UTC);
-    return CreateAutoTimestamp.create(DateTimeUtils.toJodaDateTime(zdt));
-  }
-}

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -157,7 +157,7 @@ public class TransactionManagerFactory {
 
   /** Registry is currently undergoing maintenance and is in read-only mode. */
   public static class ReadOnlyModeException extends IllegalStateException {
-    ReadOnlyModeException() {
+    public ReadOnlyModeException() {
       super("Registry is currently undergoing maintenance and is in read-only mode");
     }
   }

core/src/main/java/google/registry/privileges/secretmanager/SqlUser.java

@@ -54,7 +54,10 @@ public abstract class SqlUser {
      * Credential for RegistryTool. This is temporary, and will be removed when tool users are
      * assigned their personal credentials.
      */
-    TOOL
+    TOOL,
+
+    /** The Postgres admin account. */
+    POSTGRES
   }
 
   /** Information of a RobotUser for privilege management purposes. */

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -14,12 +14,14 @@
 
 package google.registry.rde;
 
+import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
 import static google.registry.beam.BeamUtils.createJobName;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.xml.ValidationMode.LENIENT;
 import static google.registry.xml.ValidationMode.STRICT;
+import static java.util.function.Function.identity;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -29,6 +31,7 @@ import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
 import com.google.common.base.Ascii;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -48,6 +51,7 @@ import google.registry.model.EppResource;
 import google.registry.model.common.Cursor;
 import google.registry.model.common.Cursor.CursorType;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.EppResourceIndex;
 import google.registry.model.rde.RdeMode;
@@ -67,14 +71,17 @@ import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /**
- * MapReduce that idempotently stages escrow deposit XML files on GCS for RDE/BRDA for all TLDs.
+ * Action that kicks off either a MapReduce (for Datastore) or Dataflow (for Cloud SQL) job to stage
+ * escrow deposit XML files on GCS for RDE/BRDA for all TLDs.
  *
- * <h3>MapReduce Operation</h3>
+ * <h3>Pending Deposits</h3>
  *
  * <p>This task starts by asking {@link PendingDepositChecker} which deposits need to be generated.
  * If there's nothing to deposit, we return 204 No Content; otherwise, we fire off a MapReduce job
  * and redirect to its status GUI. The task can also be run in manual operation, as described below.
  *
+ * <h3>MapReduce</h3>
+ *
  * <p>The mapreduce job scans every {@link EppResource} in Datastore. It maps a point-in-time
  * representation of each entity to the escrow XML files in which it should appear.
  *
@@ -88,11 +95,26 @@ import org.joda.time.Duration;
  * <p>{@link Registrar} entities, both active and inactive, are included in all deposits. They are
  * not rewinded point-in-time.
  *
+ * <h3>Dataflow</h3>
+ *
+ * The Dataflow job finds the most recent history entry on or before watermark for each resource
+ * type and loads the embedded resource from it, which is then projected to watermark time to
+ * account for things like pending transfer.
+ *
+ * <p>Only {@link ContactResource}s and {@link HostResource}s that are referenced by an included
+ * {@link DomainBase} will be included in the corresponding pending deposit.
+ *
+ * <p>{@link Registrar} entities, both active and inactive, are included in all deposits. They are
+ * not rewinded point-in-time.
+ *
+ * <h3>Afterward</h3>
+ *
  * <p>The XML deposit files generated by this job are humongous. A tiny XML report file is generated
  * for each deposit, telling us how much of what it contains.
  *
- * <p>Once a deposit is successfully generated, an {@link RdeUploadAction} is enqueued which will
- * upload it via SFTP to the third-party escrow provider.
+ * <p>Once a deposit is successfully generated, For RDE an {@link RdeUploadAction} is enqueued which
+ * will upload it via SFTP to the third-party escrow provider; for BRDA an {@link BrdaCopyAction} is
+ * enqueued which will copy it to a GCS bucket and be rsynced to a third-party escrow provider.
  *
  * <p>To generate escrow deposits manually and locally, use the {@code nomulus} tool command {@code
  * GenerateEscrowDepositCommand}.
@@ -106,7 +128,7 @@ import org.joda.time.Duration;
  *
  * <p>Valid model objects might not be valid to the RDE XML schema. A single invalid object will
  * cause the whole deposit to fail. You need to check the logs, find out which entities are broken,
- * and perform Datastore surgery.
+ * and perform database surgery.
  *
  * <p>If a deposit fails, an error is emitted to the logs for each broken entity. It tells you the
  * key and shows you its representation in lenient XML.
@@ -137,33 +159,40 @@ import org.joda.time.Duration;
  * <p>The deposit and report are encrypted using {@link Ghostryde}. Administrators can use the
  * {@code GhostrydeCommand} command in the {@code nomulus} tool to view them.
  *
- * <p>Unencrypted XML fragments are stored temporarily between the map and reduce steps. The
- * ghostryde encryption on the full archived deposits makes life a little more difficult for an
- * attacker. But security ultimately depends on the bucket.
+ * <p>Unencrypted XML fragments are stored temporarily between the map and reduce steps and between
+ * Dataflow transforms. The ghostryde encryption on the full archived deposits makes life a little
+ * more difficult for an attacker. But security ultimately depends on the bucket.
  *
  * <h3>Idempotency</h3>
  *
- * <p>We lock the reduce tasks. This is necessary because: a) App Engine tasks might get double
- * executed; and b) Cloud Storage file handles get committed on close <i>even if our code throws an
- * exception.</i>
+ * <p>We lock the reduce tasks for the MapReduce job. This is necessary because: a) App Engine tasks
+ * might get double executed; and b) Cloud Storage file handles get committed on close <i>even if
+ * our code throws an exception.</i>
+ *
+ * <p>For the Dataflow job we do not employ a lock because it is difficult to span a lock across
+ * three subsequent transforms (save to GCS, roll forward cursor, enqueue next action). Instead, we
+ * get around the issue by saving the deposit to a unique folder named after the job name so there
+ * is no possibility of overwriting.
  *
  * <p>Deposits are generated serially for a given (watermark, mode) pair. A deposit is never started
  * beyond the cursor. Once a deposit is completed, its cursor is rolled forward transactionally.
  * Duplicate jobs may exist {@code <=cursor}. So a transaction will not bother changing the cursor
  * if it's already been rolled forward.
  *
- * <p>Enqueuing {@code RdeUploadAction} is also part of the cursor transaction. This is necessary
- * because the first thing the upload task does is check the staging cursor to verify it's been
- * completed, so we can't enqueue before we roll. We also can't enqueue after the roll, because then
- * if enqueuing fails, the upload might never be enqueued.
+ * <p>Enqueuing {@code RdeUploadAction} or {@code BrdaCopyAction} is also part of the cursor
+ * transaction. This is necessary because the first thing the upload task does is check the staging
+ * cursor to verify it's been completed, so we can't enqueue before we roll. We also can't enqueue
+ * after the roll, because then if enqueuing fails, the upload might never be enqueued.
  *
  * <h3>Determinism</h3>
  *
  * <p>The filename of an escrow deposit is determistic for a given (TLD, watermark, {@linkplain
  * RdeMode mode}) triplet. Its generated contents is deterministic in all the ways that we care
- * about. Its view of the database is strongly consistent.
+ * about. Its view of the database is strongly consistent in Cloud SQL automatically by nature of
+ * the initial query for the history entry running at {@code READ_COMMITTED} transaction isolation
+ * level.
  *
- * <p>This is because:
+ * <p>This is also true in Datastore because:
  *
  * <ol>
  *   <li>{@code EppResource} queries are strongly consistent thanks to {@link EppResourceIndex}
@@ -226,6 +255,11 @@ public final class RdeStagingAction implements Runnable {
   @Inject MapreduceRunner mrRunner;
   @Inject @Config("projectId") String projectId;
   @Inject @Config("defaultJobRegion") String jobRegion;
+
+  @Inject
+  @Config("highPerformanceMachineType")
+  String machineType;
+
   @Inject @Config("transactionCooldown") Duration transactionCooldown;
   @Inject @Config("beamStagingBucketUrl") String stagingBucketUrl;
   @Inject @Config("rdeBucket") String rdeBucket;
@@ -270,43 +304,65 @@ public final class RdeStagingAction implements Runnable {
                   new NullInput<>(), EppResourceInputs.createEntityInput(EppResource.class)))
           .sendLinkToMapreduceConsole(response);
     } else {
-      try {
-        LaunchFlexTemplateParameter parameter =
-            new LaunchFlexTemplateParameter()
-                .setJobName(createJobName("rde", clock))
-                .setContainerSpecGcsPath(
-                    String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-                .setParameters(
-                    ImmutableMap.of(
-                        "pendings",
-                        RdePipeline.encodePendings(pendings),
-                        "validationMode",
-                        validationMode.name(),
-                        "rdeStagingBucket",
-                        rdeBucket,
-                        "stagingKey",
-                        BaseEncoding.base64Url().omitPadding().encode(stagingKeyBytes),
-                        "registryEnvironment",
-                        RegistryEnvironment.get().name()));
-        LaunchFlexTemplateResponse launchResponse =
-            dataflow
-                .projects()
-                .locations()
-                .flexTemplates()
-                .launch(
-                    projectId,
-                    jobRegion,
-                    new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
-                .execute();
-        logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
-        response.setStatus(SC_OK);
-        response.setPayload(
-            String.format("Launched RDE pipeline: %s", launchResponse.getJob().getId()));
-      } catch (IOException e) {
-        logger.atWarning().withCause(e).log("Pipeline Launch failed");
-        response.setStatus(SC_INTERNAL_SERVER_ERROR);
-        response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
-      }
+      ImmutableList.Builder<String> jobNameBuilder = new ImmutableList.Builder<>();
+      pendings.values().stream()
+          .collect(toImmutableSetMultimap(PendingDeposit::watermark, identity()))
+          .asMap()
+          .forEach(
+              (watermark, pendingDeposits) -> {
+                try {
+                  LaunchFlexTemplateParameter parameter =
+                      new LaunchFlexTemplateParameter()
+                          .setJobName(
+                              createJobName(
+                                  String.format(
+                                      "rde-%s", watermark.toString("yyyy-MM-dd't'HH-mm-ss'z'")),
+                                  clock))
+                          .setContainerSpecGcsPath(
+                              String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+                          .setParameters(
+                              new ImmutableMap.Builder<String, String>()
+                                  .put(
+                                      "pendings",
+                                      RdePipeline.encodePendingDeposits(
+                                          ImmutableSet.copyOf(pendingDeposits)))
+                                  .put("validationMode", validationMode.name())
+                                  .put("rdeStagingBucket", rdeBucket)
+                                  .put(
+                                      "stagingKey",
+                                      BaseEncoding.base64Url()
+                                          .omitPadding()
+                                          .encode(stagingKeyBytes))
+                                  .put("registryEnvironment", RegistryEnvironment.get().name())
+                                  .put("workerMachineType", machineType)
+                                  // TODO (jianglai): Investigate turning off public IPs (for which
+                                  // there is a quota) in order to increase the total number of
+                                  // workers allowed (also under quota).
+                                  // See:
+                                  // https://cloud.google.com/dataflow/docs/guides/routes-firewall
+                                  .put("usePublicIps", "true")
+                                  .build());
+                  LaunchFlexTemplateResponse launchResponse =
+                      dataflow
+                          .projects()
+                          .locations()
+                          .flexTemplates()
+                          .launch(
+                              projectId,
+                              jobRegion,
+                              new LaunchFlexTemplateRequest().setLaunchParameter(parameter))
+                          .execute();
+                  logger.atInfo().log("Got response: %s", launchResponse.getJob().toPrettyString());
+                  jobNameBuilder.add(launchResponse.getJob().getId());
+                } catch (IOException e) {
+                  logger.atWarning().withCause(e).log("Pipeline Launch failed");
+                  response.setStatus(SC_INTERNAL_SERVER_ERROR);
+                  response.setPayload(String.format("Pipeline launch failed: %s", e.getMessage()));
+                }
+              });
+      response.setStatus(SC_OK);
+      response.setPayload(
+          String.format("Launched RDE pipeline: %s", Joiner.on(", ").join(jobNameBuilder.build())));
     }
   }
 
@@ -349,7 +405,7 @@ public final class RdeStagingAction implements Runnable {
       throw new BadRequestException("Directory must not start with a slash");
     }
     String directoryWithTrailingSlash =
-        directory.get().endsWith("/") ? directory.get() : (directory.get() + '/');
+        directory.get().endsWith("/") ? directory.get() : directory.get() + '/';
 
     if (modeStrings.isEmpty()) {
       throw new BadRequestException("Mode parameter required in manual operation");
@@ -381,7 +437,7 @@ public final class RdeStagingAction implements Runnable {
       }
     }
 
-    if (revision.isPresent() && (revision.get() < 0)) {
+    if (revision.isPresent() && revision.get() < 0) {
       throw new BadRequestException("Revision must be greater than or equal to zero");
     }
 
@@ -394,11 +450,7 @@ public final class RdeStagingAction implements Runnable {
           pendingsBuilder.put(
               tld,
               PendingDeposit.createInManualOperation(
-                  tld,
-                  watermark,
-                  mode,
-                  directoryWithTrailingSlash,
-                  revision.orElse(null)));
+                  tld, watermark, mode, directoryWithTrailingSlash, revision.orElse(null)));
         }
       }
     }

core/src/main/java/google/registry/tools/GetEppResourceCommand.java

@@ -19,7 +19,6 @@ import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import java.util.Optional;
 import org.joda.time.DateTime;
@@ -56,7 +55,7 @@ abstract class GetEppResourceCommand implements CommandWithRemoteApi {
             ? String.format(
                 "%s\n\nWebsafe key: %s",
                 expand ? resource.get().toHydratedString() : resource.get(),
-                Key.create(resource.get()).getString())
+                resource.get().createVKey().getOfyKey().getString())
             : String.format("%s '%s' does not exist or is deleted\n", resourceType, uniqueId));
   }
 

core/src/main/java/google/registry/tools/GetResourceByKeyCommand.java

@@ -19,8 +19,8 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
+import google.registry.persistence.VKey;
 import java.util.List;
 
 /**
@@ -40,15 +40,15 @@ final class GetResourceByKeyCommand implements CommandWithRemoteApi {
   boolean expand;
 
   @Override
-  public void run() {
+  public void run() throws Exception {
     for (String keyString : mainParameters) {
       System.out.println("\n");
-      Key<EppResource> resourceKey =
-          checkNotNull(Key.create(keyString), "Could not parse key string: " + keyString);
+      VKey<EppResource> resourceKey =
+          checkNotNull(VKey.create(keyString), "Could not parse key string: " + keyString);
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(),
-              "Could not load resource for key: " + resourceKey);
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Could not load resource for key: " + resourceKey.getOfyKey());
       System.out.println(expand ? resource.toHydratedString() : resource.toString());
     }
   }

core/src/main/java/google/registry/tools/ResaveEntitiesCommand.java

@@ -19,8 +19,8 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
 import java.util.List;
 
 /**
@@ -37,15 +37,16 @@ public final class ResaveEntitiesCommand extends MutatingCommand {
   /** The number of resaves to do in a single transaction. */
   private static final int BATCH_SIZE = 10;
 
+  // TODO(b/207376744): figure out if there's a guide that shows how a websafe key should look like
   @Parameter(description = "Websafe keys", required = true)
   List<String> mainParameters;
 
   @Override
-  protected void init() {
+  protected void init() throws Exception {
     for (List<String> batch : partition(mainParameters, BATCH_SIZE)) {
       for (String websafeKey : batch) {
         ImmutableObject entity =
-            auditedOfy().load().key(Key.<ImmutableObject>create(websafeKey)).now();
+            (ImmutableObject) auditedOfy().load().key(VKey.create(websafeKey).getOfyKey()).now();
         stageEntityChange(entity, entity);
       }
       flushTransaction();

core/src/main/resources/META-INF/persistence.xml

@@ -83,7 +83,6 @@
     <class>google.registry.persistence.converter.BillingEventFlagSetConverter</class>
     <class>google.registry.persistence.converter.BloomFilterConverter</class>
     <class>google.registry.persistence.converter.CidrAddressBlockListConverter</class>
-    <class>google.registry.persistence.converter.CreateAutoTimestampConverter</class>
     <class>google.registry.persistence.converter.CurrencyToBillingConverter</class>
     <class>google.registry.persistence.converter.CurrencyUnitConverter</class>
     <class>google.registry.persistence.converter.DatabaseMigrationScheduleTransitionConverter</class>

core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json

@@ -43,6 +43,22 @@
       "regexes": [
         "[A-Za-z0-9\\-_]+"
       ]
+    },
+    {
+      "name": "workerMachineType",
+      "label": "The GCE machine type for the dataflow job workers.",
+      "helpText": "See https://cloud.google.com/dataflow/quotas#compute-engine-quotas for available machine types.",
+      "regexes": [
+        "[a-z0-9\\-]+"
+      ]
+    },
+    {
+      "name": "usePublicIps",
+      "label": "Whether the GCE workers are assigned public IPs",
+      "helpText": "Public IPs have an associated cost and there's a quota per region on the total number of public IPs assigned at a given time. If the service only needs to access GCP APIs, it's better to not use public IP, but one needs to configure the network accordingly. See https://cloud.google.com/dataflow/docs/guides/routes-firewall.",
+      "regexes": [
+        "true|false"
+      ]
     }
   ]
 }

core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java

@@ -67,7 +67,6 @@ import google.registry.model.replay.SqlReplayCheckpoint;
 import google.registry.model.server.Lock;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.PremiumList.PremiumEntry;
-import google.registry.model.tmch.TmchCrl;
 import google.registry.model.translators.VKeyTranslatorFactory;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -482,7 +481,8 @@ public class ReplayCommitLogsToSqlActionTest {
 
     jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
     // Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
-    Key<TmchCrl> tmchCrlKey = Key.create(TmchCrl.class, 1L);
+    Key<CommitLogManifest> manifestKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
     saveDiffFile(
         gcsUtils,
         createCheckpoint(now.minusMinutes(1)),
@@ -490,7 +490,7 @@ public class ReplayCommitLogsToSqlActionTest {
             getBucketKey(1),
             now.minusMinutes(1),
             // one object only exists in Datastore, one is dually-written (so isn't replicated)
-            ImmutableSet.of(getCrossTldKey(), tmchCrlKey)));
+            ImmutableSet.of(getCrossTldKey(), manifestKey)));
 
     runAndAssertSuccess(now.minusMinutes(1), 1, 1);
     verify(spy, times(0)).delete(any(VKey.class));

core/src/test/java/google/registry/batch/AsyncTaskEnqueuerTest.java

@@ -35,7 +35,6 @@ import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.LoggerConfig;
-import com.googlecode.objectify.Key;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.RegistryLock;
 import google.registry.testing.AppEngineExtension;
@@ -95,7 +94,8 @@ public class AsyncTaskEnqueuerTest {
   @Test
   void test_enqueueAsyncResave_success() {
     ContactResource contact = persistActiveContact("jd23456");
-    asyncTaskEnqueuer.enqueueAsyncResave(contact, clock.nowUtc(), clock.nowUtc().plusDays(5));
+    asyncTaskEnqueuer.enqueueAsyncResave(
+        contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(5));
     assertTasksEnqueued(
         QUEUE_ASYNC_ACTIONS,
         new TaskMatcher()
@@ -103,7 +103,7 @@ public class AsyncTaskEnqueuerTest {
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, Key.create(contact).getString())
+            .param(PARAM_RESOURCE_KEY, contact.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .etaDelta(
                 standardDays(5).minus(standardSeconds(30)),
@@ -115,7 +115,7 @@ public class AsyncTaskEnqueuerTest {
     ContactResource contact = persistActiveContact("jd23456");
     DateTime now = clock.nowUtc();
     asyncTaskEnqueuer.enqueueAsyncResave(
-        contact,
+        contact.createVKey(),
         now,
         ImmutableSortedSet.of(now.plusHours(24), now.plusHours(50), now.plusHours(75)));
     assertTasksEnqueued(
@@ -125,7 +125,7 @@ public class AsyncTaskEnqueuerTest {
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, Key.create(contact).getString())
+            .param(PARAM_RESOURCE_KEY, contact.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, now.toString())
             .param(PARAM_RESAVE_TIMES, "2015-05-20T14:34:56.000Z,2015-05-21T15:34:56.000Z")
             .etaDelta(
@@ -137,7 +137,8 @@ public class AsyncTaskEnqueuerTest {
   @Test
   void test_enqueueAsyncResave_ignoresTasksTooFarIntoFuture() {
     ContactResource contact = persistActiveContact("jd23456");
-    asyncTaskEnqueuer.enqueueAsyncResave(contact, clock.nowUtc(), clock.nowUtc().plusDays(31));
+    asyncTaskEnqueuer.enqueueAsyncResave(
+        contact.createVKey(), clock.nowUtc(), clock.nowUtc().plusDays(31));
     assertNoTasksEnqueued(QUEUE_ASYNC_ACTIONS);
     assertLogMessage(logHandler, Level.INFO, "Ignoring async re-save");
   }

core/src/test/java/google/registry/batch/DeleteContactsAndHostsActionTest.java

@@ -64,7 +64,6 @@ import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
-import com.googlecode.objectify.Key;
 import google.registry.batch.AsyncTaskMetrics.OperationResult;
 import google.registry.batch.AsyncTaskMetrics.OperationType;
 import google.registry.batch.DeleteContactsAndHostsAction.DeleteEppResourceReducer;
@@ -497,7 +496,7 @@ public class DeleteContactsAndHostsActionTest
         QUEUE_ASYNC_DELETE,
         new TaskMatcher()
             .etaDelta(standardHours(23), standardHours(25))
-            .param("resourceKey", Key.create(contactNotSaved).getString())
+            .param("resourceKey", contactNotSaved.createVKey().getOfyKey().getString())
             .param("requestingClientId", "TheRegistrar")
             .param("clientTransactionId", "fakeClientTrid")
             .param("serverTransactionId", "fakeServerTrid")
@@ -505,7 +504,7 @@ public class DeleteContactsAndHostsActionTest
             .param("requestedTime", timeBeforeRun.toString()),
         new TaskMatcher()
             .etaDelta(standardHours(23), standardHours(25))
-            .param("resourceKey", Key.create(hostNotSaved).getString())
+            .param("resourceKey", hostNotSaved.createVKey().getOfyKey().getString())
             .param("requestingClientId", "TheRegistrar")
             .param("clientTransactionId", "fakeClientTrid")
             .param("serverTransactionId", "fakeServerTrid")

core/src/test/java/google/registry/batch/RefreshDnsOnHostRenameActionTest.java

@@ -44,7 +44,6 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
-import com.googlecode.objectify.Key;
 import google.registry.batch.AsyncTaskMetrics.OperationResult;
 import google.registry.batch.RefreshDnsOnHostRenameAction.RefreshDnsOnHostRenameReducer;
 import google.registry.dns.DnsQueue;
@@ -238,7 +237,7 @@ public class RefreshDnsOnHostRenameActionTest
         QUEUE_ASYNC_HOST_RENAME,
         new TaskMatcher()
             .etaDelta(standardHours(23), standardHours(25))
-            .param("hostKey", Key.create(host).getString()));
+            .param("hostKey", host.createVKey().getOfyKey().getString()));
     assertThat(acquireLock()).isPresent();
   }
 

core/src/test/java/google/registry/batch/ResaveEntityActionTest.java

@@ -148,7 +148,7 @@ public class ResaveEntityActionTest {
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, Key.create(resavedDomain).getString())
+            .param(PARAM_RESOURCE_KEY, resavedDomain.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, requestedTime.toString())
             .etaDelta(
                 standardDays(5).minus(standardSeconds(30)),

core/src/test/java/google/registry/beam/BeamActionTestBase.java

@@ -29,8 +29,10 @@ import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
 import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
 import google.registry.testing.FakeResponse;
 import org.junit.jupiter.api.BeforeEach;
+import org.mockito.invocation.InvocationOnMock;
 import org.mockito.junit.jupiter.MockitoSettings;
 import org.mockito.quality.Strictness;
+import org.mockito.stubbing.Answer;
 
 /** Base class for all actions that launches a Dataflow Flex template. */
 @MockitoSettings(strictness = Strictness.STRICT_STUBS)
@@ -42,8 +44,19 @@ public abstract class BeamActionTestBase {
   private Locations locations = mock(Locations.class);
   protected FlexTemplates templates = mock(FlexTemplates.class);
   protected Launch launch = mock(Launch.class);
-  private LaunchFlexTemplateResponse launchResponse =
-      new LaunchFlexTemplateResponse().setJob(new Job().setId("jobid"));
+  private Answer<LaunchFlexTemplateResponse> answer =
+      new Answer<LaunchFlexTemplateResponse>() {
+        private Integer times = 0;
+
+        @Override
+        public LaunchFlexTemplateResponse answer(InvocationOnMock invocation) throws Throwable {
+          LaunchFlexTemplateResponse response =
+              new LaunchFlexTemplateResponse()
+                  .setJob(new Job().setId("jobid" + (times == 0 ? "" : times.toString())));
+          times = times + 1;
+          return response;
+        }
+      };
 
   @BeforeEach
   protected void beforeEach() throws Exception {
@@ -52,6 +65,6 @@ public abstract class BeamActionTestBase {
     when(locations.flexTemplates()).thenReturn(templates);
     when(templates.launch(anyString(), anyString(), any(LaunchFlexTemplateRequest.class)))
         .thenReturn(launch);
-    when(launch.execute()).thenReturn(launchResponse);
+    when(launch.execute()).thenAnswer(answer);
   }
 }

core/src/test/java/google/registry/beam/rde/RdePipelineTest.java

@@ -17,8 +17,8 @@ package google.registry.beam.rde;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.beam.rde.RdePipeline.decodePendings;
-import static google.registry.beam.rde.RdePipeline.encodePendings;
+import static google.registry.beam.rde.RdePipeline.decodePendingDeposits;
+import static google.registry.beam.rde.RdePipeline.encodePendingDeposits;
 import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
 import static google.registry.model.rde.RdeMode.FULL;
 import static google.registry.model.rde.RdeMode.THIN;
@@ -27,25 +27,26 @@ import static google.registry.rde.RdeResourceType.CONTACT;
 import static google.registry.rde.RdeResourceType.DOMAIN;
 import static google.registry.rde.RdeResourceType.HOST;
 import static google.registry.rde.RdeResourceType.REGISTRAR;
-import static google.registry.testing.AppEngineExtension.loadInitialData;
+import static google.registry.testing.AppEngineExtension.makeRegistrar1;
+import static google.registry.testing.AppEngineExtension.makeRegistrar2;
 import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.DatabaseHelper.newHostResource;
+import static google.registry.testing.DatabaseHelper.insertSimpleResources;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
-import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
+import static google.registry.testing.DatabaseHelper.persistActiveHost;
 import static google.registry.testing.DatabaseHelper.persistEppResource;
 import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
+import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.joda.time.Duration.standardDays;
+import static org.junit.Assert.assertThrows;
 
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
-import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.Iterables;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.common.io.BaseEncoding;
 import google.registry.beam.TestPipelineExtension;
@@ -53,12 +54,26 @@ import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.PgpHelper;
 import google.registry.model.common.Cursor;
 import google.registry.model.common.Cursor.CursorType;
+import google.registry.model.contact.ContactBase;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DesignatedContact;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.Period;
+import google.registry.model.eppcommon.Trid;
+import google.registry.model.host.HostBase;
+import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeRevision;
 import google.registry.model.rde.RdeRevision.RdeRevisionId;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.State;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
+import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTestExtensions;
@@ -111,14 +126,11 @@ public class RdePipelineTest {
   // This is teh default as-of time the RDE/BRDA job.
   private final DateTime now = DateTime.parse("2000-01-01TZ");
 
-  private final ImmutableSetMultimap<String, PendingDeposit> pendings =
-      ImmutableSetMultimap.of(
-          "soy",
-          PendingDeposit.create("soy", now, FULL, RDE_STAGING, standardDays(1)),
-          "soy",
-          PendingDeposit.create("soy", now, THIN, RDE_STAGING, standardDays(1)),
-          "fun",
-          PendingDeposit.create("fun", now, FULL, RDE_STAGING, standardDays(1)));
+  private final ImmutableSet<PendingDeposit> pendings =
+      ImmutableSet.of(
+          PendingDeposit.create("soy", now, FULL, RDE_STAGING, Duration.standardDays(1)),
+          PendingDeposit.create("soy", now, THIN, RDE_STAGING, Duration.standardDays(1)),
+          PendingDeposit.create("fun", now, FULL, RDE_STAGING, Duration.standardDays(1)));
 
   private final ImmutableList<DepositFragment> brdaFragments =
       ImmutableList.of(
@@ -129,8 +141,8 @@ public class RdePipelineTest {
       ImmutableList.of(
           DepositFragment.create(RdeResourceType.DOMAIN, "<rdeDomain:domain/>\n", ""),
           DepositFragment.create(RdeResourceType.REGISTRAR, "<rdeRegistrar:registrar/>\n", ""),
-          DepositFragment.create(RdeResourceType.CONTACT, "<rdeContact:contact/>\n", ""),
-          DepositFragment.create(RdeResourceType.HOST, "<rdeHost:host/>\n", ""));
+          DepositFragment.create(CONTACT, "<rdeContact:contact/>\n", ""),
+          DepositFragment.create(HOST, "<rdeHost:host/>\n", ""));
 
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
 
@@ -164,9 +176,68 @@ public class RdePipelineTest {
 
   private RdePipeline rdePipeline;
 
+  private ContactHistory persistContactHistory(ContactBase contact) {
+    return persistResource(
+        new ContactHistory.Builder()
+            .setType(HistoryEntry.Type.HOST_CREATE)
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .setModificationTime(clock.nowUtc())
+            .setRegistrarId("TheRegistrar")
+            .setTrid(Trid.create("ABC-123", "server-trid"))
+            .setBySuperuser(false)
+            .setReason("reason")
+            .setRequestedByRegistrar(true)
+            .setContact(contact)
+            .setContactRepoId(contact.getRepoId())
+            .build());
+  }
+
+  private DomainHistory persistDomainHistory(DomainContent domain) {
+    DomainTransactionRecord transactionRecord =
+        new DomainTransactionRecord.Builder()
+            .setTld("soy")
+            .setReportingTime(clock.nowUtc())
+            .setReportField(TransactionReportField.NET_ADDS_1_YR)
+            .setReportAmount(1)
+            .build();
+
+    return persistResource(
+        new DomainHistory.Builder()
+            .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .setModificationTime(clock.nowUtc())
+            .setRegistrarId("TheRegistrar")
+            .setTrid(Trid.create("ABC-123", "server-trid"))
+            .setBySuperuser(false)
+            .setReason("reason")
+            .setRequestedByRegistrar(true)
+            .setDomain(domain)
+            .setDomainRepoId(domain.getRepoId())
+            .setDomainTransactionRecords(ImmutableSet.of(transactionRecord))
+            .setOtherRegistrarId("otherClient")
+            .setPeriod(Period.create(1, Period.Unit.YEARS))
+            .build());
+  }
+
+  private HostHistory persistHostHistory(HostBase hostBase) {
+    return persistResource(
+        new HostHistory.Builder()
+            .setType(HistoryEntry.Type.HOST_CREATE)
+            .setXmlBytes("<xml></xml>".getBytes(UTF_8))
+            .setModificationTime(clock.nowUtc())
+            .setRegistrarId("TheRegistrar")
+            .setTrid(Trid.create("ABC-123", "server-trid"))
+            .setBySuperuser(false)
+            .setReason("reason")
+            .setRequestedByRegistrar(true)
+            .setHost(hostBase)
+            .setHostRepoId(hostBase.getRepoId())
+            .build());
+  }
+
   @BeforeEach
   void beforeEach() throws Exception {
-    loadInitialData();
+    insertSimpleResources(ImmutableList.of(makeRegistrar1(), makeRegistrar2()));
 
     // Two real registrars have been created by loadInitialData(), named "New Registrar" and "The
     // Registrar". Create one included registrar (external_monitoring) and two excluded ones.
@@ -192,26 +263,89 @@ public class RdePipelineTest {
     tm().transact(
             () -> {
               tm().put(Cursor.create(CursorType.BRDA, now, Registry.get("soy")));
-              tm().put(Cursor.create(CursorType.RDE_STAGING, now, Registry.get("soy")));
+              tm().put(Cursor.create(RDE_STAGING, now, Registry.get("soy")));
               RdeRevision.saveRevision("soy", now, THIN, 0);
               RdeRevision.saveRevision("soy", now, FULL, 0);
             });
 
-    // Also persists a "contact1234" contact in the process.
-    persistActiveDomain("hello.soy");
-    persistActiveDomain("kitty.fun");
-    // Should not appear anywhere.
-    persistActiveDomain("lol.cat");
-    persistDeletedDomain("deleted.soy", DateTime.parse("1999-12-30TZ"));
+    // This contact is never referenced.
+    persistContactHistory(persistActiveContact("contactX"));
+    ContactResource contact1 = persistActiveContact("contact1234");
+    persistContactHistory(contact1);
+    ContactResource contact2 = persistActiveContact("contact456");
+    persistContactHistory(contact2);
 
-    persistActiveContact("contact456");
+    // This host is never referenced.
+    persistHostHistory(persistActiveHost("ns0.domain.tld"));
+    HostResource host1 = persistActiveHost("ns1.external.tld");
+    persistHostHistory(host1);
+    DomainBase helloDomain =
+        persistEppResource(
+            newDomainBase("hello.soy", contact1)
+                .asBuilder()
+                .addNameserver(host1.createVKey())
+                .build());
+    persistDomainHistory(helloDomain);
+    persistHostHistory(persistActiveHost("not-used-subordinate.hello.soy"));
+    HostResource host2 = persistActiveHost("ns1.hello.soy");
+    persistHostHistory(host2);
+    DomainBase kittyDomain =
+        persistEppResource(
+            newDomainBase("kitty.fun", contact2)
+                .asBuilder()
+                .addNameservers(ImmutableSet.of(host1.createVKey(), host2.createVKey()))
+                .build());
+    persistDomainHistory(kittyDomain);
+    // Should not appear because the TLD is not included in a pending deposit.
+    persistDomainHistory(persistEppResource(newDomainBase("lol.cat", contact1)));
+    // To be deleted.
+    DomainBase deletedDomain = persistActiveDomain("deleted.soy");
+    persistDomainHistory(deletedDomain);
 
-    HostResource host = persistEppResource(newHostResource("old.host.test"));
-    // Set the clock to 2000-01-02, the updated host should NOT show up in RDE.
+    // Advance time
+    clock.advanceOneMilli();
+    persistDomainHistory(deletedDomain.asBuilder().setDeletionTime(clock.nowUtc()).build());
+    kittyDomain = kittyDomain.asBuilder().setDomainName("cat.fun").build();
+    persistDomainHistory(kittyDomain);
+    ContactResource contact3 = persistActiveContact("contact789");
+    persistContactHistory(contact3);
+    // This is a subordinate domain in TLD .cat, which is not included in any pending deposit. But
+    // it should still be included as a subordinate host in the pendign deposit for .soy.
+    HostResource host3 = persistActiveHost("ns1.lol.cat");
+    persistHostHistory(host3);
+    persistDomainHistory(
+        helloDomain
+            .asBuilder()
+            .addContacts(
+                ImmutableSet.of(
+                    DesignatedContact.create(DesignatedContact.Type.ADMIN, contact3.createVKey())))
+            .addNameserver(host3.createVKey())
+            .build());
+    // contact456 is renamed to contactABC.
+    persistContactHistory(contact2.asBuilder().setContactId("contactABC").build());
+    // ns1.hello.soy is renamed to ns2.hello.soy
+    persistHostHistory(host2.asBuilder().setHostName("ns2.hello.soy").build());
+
+    // Set the clock to 2000-01-02, any change after hereafter should not show up in the
+    // resulting deposit fragments.
     clock.advanceBy(Duration.standardDays(2));
-    persistEppResource(host.asBuilder().setHostName("new.host.test").build());
+    persistDomainHistory(kittyDomain.asBuilder().setDeletionTime(clock.nowUtc()).build());
+    ContactResource futureContact = persistActiveContact("future-contact");
+    persistContactHistory(futureContact);
+    HostResource futureHost = persistActiveHost("ns1.future.tld");
+    persistHostHistory(futureHost);
+    persistDomainHistory(
+        persistEppResource(
+            newDomainBase("future.soy", futureContact)
+                .asBuilder()
+                .setNameservers(futureHost.createVKey())
+                .build()));
+    // contactABC is renamed to contactXYZ.
+    persistContactHistory(contact2.asBuilder().setContactId("contactXYZ").build());
+    // ns2.hello.soy is renamed to ns3.hello.soy
+    persistHostHistory(host2.asBuilder().setHostName("ns3.hello.soy").build());
 
-    options.setPendings(encodePendings(pendings));
+    options.setPendings(encodePendingDeposits(pendings));
     // The EPP resources created in tests do not have all the fields populated, using a STRICT
     // validation mode will result in a lot of warnings during marshalling.
     options.setValidationMode("LENIENT");
@@ -223,9 +357,22 @@ public class RdePipelineTest {
   }
 
   @Test
-  void testSuccess_encodeAndDecodePendingsMap() throws Exception {
-    String encodedString = encodePendings(pendings);
-    assertThat(decodePendings(encodedString)).isEqualTo(pendings);
+  void testSuccess_encodeAndDecodePendingDeposits() throws Exception {
+    String encodedString = encodePendingDeposits(pendings);
+    assertThat(decodePendingDeposits(encodedString)).isEqualTo(pendings);
+  }
+
+  @Test
+  void testFailure_pendingDepositsWithDifferentWatermarks() throws Exception {
+    options.setPendings(
+        encodePendingDeposits(
+            ImmutableSet.of(
+                PendingDeposit.create("soy", now, FULL, RDE_STAGING, Duration.standardDays(1)),
+                PendingDeposit.create(
+                    "soy", now.plusSeconds(1), THIN, RDE_STAGING, Duration.standardDays(1)))));
+    assertThrows(
+        IllegalArgumentException.class,
+        () -> new RdePipeline(options, gcsUtils, cloudTasksHelper.getTestCloudTasksUtils()));
   }
 
   @Test
@@ -243,36 +390,47 @@ public class RdePipelineTest {
                         // The same registrars are attached to all the pending deposits.
                         .containsExactly("New Registrar", "The Registrar", "external_monitoring");
                     // Domain fragments.
-                    assertThat(
-                            getFragmentForType(kv, DOMAIN)
-                                .map(getXmlElement(DOMAIN_NAME_PATTERN))
-                                .anyMatch(
-                                    domain ->
-                                        // Deleted domain should not be included
-                                        domain.equals("deleted.soy")
-                                            // Only domains on the pending deposit's tld should
-                                            // appear.
-                                            || !kv.getKey()
-                                                .tld()
-                                                .equals(
-                                                    Iterables.get(
-                                                        Splitter.on('.').split(domain), 1))))
-                        .isFalse();
+                    if (kv.getKey().tld().equals("soy")) {
+                      assertThat(
+                              getFragmentForType(kv, DOMAIN)
+                                  .map(getXmlElement(DOMAIN_NAME_PATTERN))
+                                  .collect(toImmutableSet()))
+                          .containsExactly("hello.soy");
+                    } else {
+                      assertThat(
+                              getFragmentForType(kv, DOMAIN)
+                                  .map(getXmlElement(DOMAIN_NAME_PATTERN))
+                                  .collect(toImmutableSet()))
+                          .containsExactly("cat.fun");
+                    }
                     if (kv.getKey().mode().equals(FULL)) {
-                      // Contact fragments.
-                      assertThat(
-                              getFragmentForType(kv, CONTACT)
-                                  .map(getXmlElement(CONTACT_ID_PATTERN))
-                                  .collect(toImmutableSet()))
-                          // The same contacts are attached too all pending deposits.
-                          .containsExactly("contact1234", "contact456");
-                      // Host fragments.
-                      assertThat(
-                              getFragmentForType(kv, HOST)
-                                  .map(getXmlElement(HOST_NAME_PATTERN))
-                                  .collect(toImmutableSet()))
-                          // Should load the resource before update.
-                          .containsExactly("old.host.test");
+                      // Contact fragments for hello.soy.
+                      if (kv.getKey().tld().equals("soy")) {
+                        assertThat(
+                                getFragmentForType(kv, CONTACT)
+                                    .map(getXmlElement(CONTACT_ID_PATTERN))
+                                    .collect(toImmutableSet()))
+                            .containsExactly("contact1234", "contact789");
+                        // Host fragments for hello.soy.
+                        assertThat(
+                                getFragmentForType(kv, HOST)
+                                    .map(getXmlElement(HOST_NAME_PATTERN))
+                                    .collect(toImmutableSet()))
+                            .containsExactly("ns1.external.tld", "ns1.lol.cat");
+                      } else {
+                        // Contact fragments for cat.fun.
+                        assertThat(
+                                getFragmentForType(kv, CONTACT)
+                                    .map(getXmlElement(CONTACT_ID_PATTERN))
+                                    .collect(toImmutableSet()))
+                            .containsExactly("contactABC");
+                        // Host fragments for cat.soy.
+                        assertThat(
+                                getFragmentForType(kv, HOST)
+                                    .map(getXmlElement(HOST_NAME_PATTERN))
+                                    .collect(toImmutableSet()))
+                            .containsExactly("ns1.external.tld", "ns2.hello.soy");
+                      }
                     } else {
                       // BRDA does not contain contact or hosts.
                       assertThat(
@@ -295,7 +453,7 @@ public class RdePipelineTest {
     PendingDeposit brdaKey =
         PendingDeposit.create("soy", now, THIN, CursorType.BRDA, Duration.standardDays(1));
     PendingDeposit rdeKey =
-        PendingDeposit.create("soy", now, FULL, CursorType.RDE_STAGING, Duration.standardDays(1));
+        PendingDeposit.create("soy", now, FULL, RDE_STAGING, Duration.standardDays(1));
 
     verifyFiles(ImmutableMap.of(brdaKey, brdaFragments, rdeKey, rdeFragments), false);
 
@@ -311,7 +469,7 @@ public class RdePipelineTest {
         .isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
     assertThat(loadRevision(now, THIN)).isEqualTo(1);
 
-    assertThat(loadCursorTime(CursorType.RDE_STAGING))
+    assertThat(loadCursorTime(RDE_STAGING))
         .isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
     assertThat(loadRevision(now, FULL)).isEqualTo(1);
     cloudTasksHelper.assertTasksEnqueued(
@@ -350,7 +508,7 @@ public class RdePipelineTest {
     assertThat(loadCursorTime(CursorType.BRDA)).isEquivalentAccordingToCompareTo(now);
     assertThat(loadRevision(now, THIN)).isEqualTo(0);
 
-    assertThat(loadCursorTime(CursorType.RDE_STAGING)).isEquivalentAccordingToCompareTo(now);
+    assertThat(loadCursorTime(RDE_STAGING)).isEquivalentAccordingToCompareTo(now);
     assertThat(loadRevision(now, FULL)).isEqualTo(0);
     cloudTasksHelper.assertNoTasksEnqueued("brda", "rde-upload");
   }

core/src/test/java/google/registry/cron/CommitLogFanoutActionTest.java

@@ -15,14 +15,13 @@
 package google.registry.cron;
 
 import static google.registry.cron.CommitLogFanoutAction.BUCKET_PARAM;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 
 import com.google.common.base.Joiner;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.testing.AppEngineExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
-import google.registry.util.Retrier;
-import google.registry.util.TaskQueueUtils;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
+import google.registry.testing.FakeClock;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -34,6 +33,7 @@ class CommitLogFanoutActionTest {
 
   private static final String ENDPOINT = "/the/servlet";
   private static final String QUEUE = "the-queue";
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
 
   @RegisterExtension
   final AppEngineExtension appEngineExtension =
@@ -54,15 +54,16 @@ class CommitLogFanoutActionTest {
   @Test
   void testSuccess() {
     CommitLogFanoutAction action = new CommitLogFanoutAction();
-    action.taskQueueUtils = new TaskQueueUtils(new Retrier(null, 1));
+    action.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
     action.endpoint = ENDPOINT;
     action.queue = QUEUE;
     action.jitterSeconds = Optional.empty();
+    action.clock = new FakeClock();
     action.run();
     List<TaskMatcher> matchers = new ArrayList<>();
     for (int bucketId : CommitLogBucket.getBucketIds()) {
       matchers.add(new TaskMatcher().url(ENDPOINT).param(BUCKET_PARAM, Integer.toString(bucketId)));
     }
-    assertTasksEnqueued(QUEUE, matchers);
+    cloudTasksHelper.assertTasksEnqueued(QUEUE, matchers);
   }
 }

core/src/test/java/google/registry/flows/EppLifecycleDomainTest.java

@@ -1449,4 +1449,39 @@ class EppLifecycleDomainTest extends EppTestCase {
         .atTime("2001-01-08T00:00:00Z")
         .hasResponse("domain_transfer_query_response_completed_fakesite.xml");
   }
+
+  @TestOfyAndSql
+  void testDomainUpdateBySuperuser_sendsPollMessage() throws Exception {
+    setIsSuperuser(false);
+    assertThatLoginSucceeds("NewRegistrar", "foo-BAR2");
+    createContacts(DateTime.parse("2000-06-01T00:00:00Z"));
+
+    // Create domain example.tld
+    assertThatCommand(
+            "domain_create_no_hosts_or_dsdata.xml", ImmutableMap.of("DOMAIN", "example.tld"))
+        .atTime("2000-06-01T00:02:00Z")
+        .hasResponse(
+            "domain_create_response.xml",
+            ImmutableMap.of(
+                "DOMAIN", "example.tld",
+                "CRDATE", "2000-06-01T00:02:00Z",
+                "EXDATE", "2002-06-01T00:02:00Z"));
+    assertThatLogoutSucceeds();
+
+    setIsSuperuser(true);
+    assertThatLoginSucceeds("TheRegistrar", "password2");
+    // Run domain update that adds server status as superuser
+    assertThatCommand("domain_update_server_hold.xml")
+        .atTime("2000-06-02T13:00:00Z")
+        .hasResponse("generic_success_response.xml");
+    assertThatLogoutSucceeds();
+    setIsSuperuser(false);
+
+    assertThatLoginSucceeds("NewRegistrar", "foo-BAR2");
+    // Verify that the owning registrar now has the correct poll message
+    assertThatCommand("poll.xml")
+        .atTime("2000-06-02T13:00:01Z")
+        .hasResponse("poll_response_server_hold.xml");
+    assertThatLogoutSucceeds();
+  }
 }

core/src/test/java/google/registry/flows/ResourceFlowTestCase.java

@@ -154,13 +154,14 @@ public abstract class ResourceFlowTestCase<F extends Flow, R extends EppResource
   /** Asserts the presence of a single enqueued async contact or host deletion */
   protected <T extends EppResource> void assertAsyncDeletionTaskEnqueued(
       T resource, String requestingClientId, Trid trid, boolean isSuperuser) {
-    TaskMatcher expected = new TaskMatcher()
-        .etaDelta(Duration.standardSeconds(75), Duration.standardSeconds(105)) // expected: 90
-        .param("resourceKey", Key.create(resource).getString())
-        .param("requestingClientId", requestingClientId)
-        .param("serverTransactionId", trid.getServerTransactionId())
-        .param("isSuperuser", Boolean.toString(isSuperuser))
-        .param("requestedTime", clock.nowUtc().toString());
+    TaskMatcher expected =
+        new TaskMatcher()
+            .etaDelta(Duration.standardSeconds(75), Duration.standardSeconds(105)) // expected: 90
+            .param("resourceKey", resource.createVKey().getOfyKey().getString())
+            .param("requestingClientId", requestingClientId)
+            .param("serverTransactionId", trid.getServerTransactionId())
+            .param("isSuperuser", Boolean.toString(isSuperuser))
+            .param("requestedTime", clock.nowUtc().toString());
     trid.getClientTransactionId()
         .ifPresent(clTrid -> expected.param("clientTransactionId", clTrid));
     assertTasksEnqueued("async-delete-pull", expected);

core/src/test/java/google/registry/flows/contact/ContactCreateFlowTest.java

@@ -26,15 +26,18 @@ import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptio
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.contact.ContactFlowUtils.BadInternationalizedPostalInfoException;
 import google.registry.flows.contact.ContactFlowUtils.DeclineContactDisclosureFieldDisallowedPolicyException;
 import google.registry.flows.exceptions.ResourceAlreadyExistsForThisClientException;
 import google.registry.flows.exceptions.ResourceCreateContentionException;
 import google.registry.model.contact.ContactResource;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -137,4 +140,12 @@ class ContactCreateFlowTest extends ResourceFlowTestCase<ContactCreateFlow, Cont
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-cont-create");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/contact/ContactDeleteFlowTest.java

@@ -35,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException;
@@ -51,9 +52,11 @@ import google.registry.model.tld.Registry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import google.registry.testing.TestSqlOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
@@ -243,6 +246,15 @@ class ContactDeleteFlowTest extends ResourceFlowTestCase<ContactDeleteFlow, Cont
     assertIcannReportingActivityFieldLogged("srs-cont-delete");
   }
 
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() throws Exception {
+    persistActiveContact(getUniqueIdFromCommand());
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   private void assertOfyDeleteSuccess(String registrarId, String clientTrid, boolean isSuperuser)
       throws Exception {
     ContactResource deletedContact = reloadResourceByForeignKey();

core/src/test/java/google/registry/flows/contact/ContactTransferApproveFlowTest.java

@@ -26,6 +26,7 @@ import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptio
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException;
@@ -40,9 +41,11 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -265,4 +268,12 @@ class ContactTransferApproveFlowTest
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-cont-transfer-approve");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/contact/ContactTransferCancelFlowTest.java

@@ -25,6 +25,7 @@ import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptio
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.exceptions.NotPendingTransferException;
@@ -37,9 +38,11 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -251,4 +254,12 @@ class ContactTransferCancelFlowTest
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-cont-transfer-cancel");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/contact/ContactTransferRejectFlowTest.java

@@ -25,6 +25,7 @@ import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptio
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException;
@@ -39,9 +40,11 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -264,4 +267,12 @@ class ContactTransferRejectFlowTest
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-cont-transfer-reject");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/contact/ContactTransferRequestFlowTest.java

@@ -35,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.exceptions.AlreadyPendingTransferException;
@@ -50,9 +51,11 @@ import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.transfer.ContactTransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
@@ -313,4 +316,12 @@ class ContactTransferRequestFlowTest
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-cont-transfer-request");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/contact/ContactUpdateFlowTest.java

@@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
@@ -41,9 +42,11 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.contact.PostalInfo;
 import google.registry.model.contact.PostalInfo.Type;
 import google.registry.model.eppcommon.StatusValue;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -426,4 +429,13 @@ class ContactUpdateFlowTest extends ResourceFlowTestCase<ContactUpdateFlow, Cont
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-cont-update");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() throws Exception {
+    persistActiveContact(getUniqueIdFromCommand());
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -72,6 +72,7 @@ import com.google.common.collect.Ordering;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.EppException.UnimplementedExtensionException;
 import google.registry.flows.EppRequestSource;
 import google.registry.flows.ExtensionManager.UndeclaredServiceExtensionException;
@@ -164,10 +165,12 @@ import google.registry.model.tld.Registry.TldState;
 import google.registry.model.tld.Registry.TldType;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.persistence.VKey;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.math.BigDecimal;
 import java.util.Map;
 import javax.annotation.Nullable;
@@ -2517,4 +2520,13 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     EppMetric eppMetric = getEppMetric();
     assertThat(eppMetric.getCommandName()).hasValue("DomainCreate");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    persistContactsAndHosts();
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -66,9 +66,9 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
-import com.googlecode.objectify.Key;
 import google.registry.batch.ResaveEntityAction;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.EppException.UnimplementedExtensionException;
 import google.registry.flows.EppRequestSource;
 import google.registry.flows.ResourceFlowTestCase;
@@ -101,10 +101,12 @@ import google.registry.model.tld.Registry.TldType;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.Map;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -311,7 +313,7 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, Key.create(domain).getString())
+            .param(PARAM_RESOURCE_KEY, domain.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .param(PARAM_RESAVE_TIMES, clock.nowUtc().plusDays(5).toString())
             .etaDelta(when.minus(standardSeconds(30)), when.plus(standardSeconds(30))));
@@ -1249,4 +1251,13 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
     EppException thrown = assertThrows(UnimplementedExtensionException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() throws Exception {
+    setUpSuccessfulTest();
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainRenewFlowTest.java

@@ -42,6 +42,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.EppRequestSource;
 import google.registry.flows.FlowUtils.UnknownCurrencyEppException;
 import google.registry.flows.ResourceFlowTestCase;
@@ -73,10 +74,12 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.SetClockExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.Map;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -870,4 +873,20 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
                 TransactionReportField.netRenewsFieldFromYears(5),
                 1));
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() throws Exception {
+    persistDomain();
+    DomainBase domain = reloadResourceByForeignKey();
+    persistResource(
+        domain
+            .asBuilder()
+            .setRegistrationExpirationTime(domain.getRegistrationExpirationTime().minusYears(1))
+            .build());
+    clock.setTo(clock.nowUtc().minusSeconds(2));
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainRestoreRequestFlowTest.java

@@ -41,6 +41,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.EppException.UnimplementedExtensionException;
 import google.registry.flows.FlowUtils.UnknownCurrencyEppException;
 import google.registry.flows.ResourceFlowTestCase;
@@ -72,9 +73,11 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.Map;
 import java.util.Optional;
 import org.joda.money.Money;
@@ -771,4 +774,13 @@ class DomainRestoreRequestFlowTest
     assertThat(thrown).hasMessageThat().contains("domain restore reports are not supported");
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() throws Exception {
+    persistPendingDeleteDomain();
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainTransferApproveFlowTest.java

@@ -43,6 +43,7 @@ import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Ordering;
 import com.google.common.collect.Streams;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException;
@@ -72,9 +73,11 @@ import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.Arrays;
 import java.util.stream.Stream;
 import org.joda.money.Money;
@@ -94,7 +97,7 @@ class DomainTransferApproveFlowTest
   final ReplayExtension replayExtension = ReplayExtension.createWithDoubleReplay(clock);
 
   @BeforeEach
-  void setUp() {
+  void beforeEach() {
     setEppInput("domain_transfer_approve.xml");
     // Change the registry so that the renew price changes a day minus 1 millisecond before the
     // transfer (right after there will be an autorenew in the test case that has one) and then
@@ -674,4 +677,12 @@ class DomainTransferApproveFlowTest
         domain.getRegistrationExpirationTime());
     assertHistoryEntriesDoNotContainTransferBillingEventsOrGracePeriods();
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainTransferCancelFlowTest.java

@@ -37,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.domain.DomainFlowUtils.NotAuthorizedForTldException;
@@ -55,9 +56,11 @@ import google.registry.model.tld.Registry;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse.DomainTransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
@@ -74,7 +77,7 @@ class DomainTransferCancelFlowTest
   final ReplayExtension replayExtension = ReplayExtension.createWithDoubleReplay(clock);
 
   @BeforeEach
-  void setUp() {
+  void beforeEach() {
     setEppInput("domain_transfer_cancel.xml");
     setRegistrarIdForFlow("NewRegistrar");
     setupDomainWithPendingTransfer("example", "tld");
@@ -416,4 +419,12 @@ class DomainTransferCancelFlowTest
     assertThat(persistedEntry.getDomainTransactionRecords())
         .containsExactly(previousSuccessRecord.asBuilder().setReportAmount(-1).build());
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainTransferRejectFlowTest.java

@@ -37,6 +37,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.google.common.collect.ImmutableSet;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException;
@@ -57,9 +58,11 @@ import google.registry.model.tld.Registry;
 import google.registry.model.transfer.TransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
@@ -76,7 +79,7 @@ class DomainTransferRejectFlowTest
   final ReplayExtension replayExtension = ReplayExtension.createWithDoubleReplay(clock);
 
   @BeforeEach
-  void setUp() {
+  void beforeEach() {
     setEppInput("domain_transfer_reject.xml");
     setRegistrarIdForFlow("TheRegistrar");
     setupDomainWithPendingTransfer("example", "tld");
@@ -384,4 +387,12 @@ class DomainTransferRejectFlowTest
             previousSuccessRecord.asBuilder().setReportAmount(-1).build(),
             DomainTransactionRecord.create("tld", clock.nowUtc(), TRANSFER_NACKED, 1));
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainTransferRequestFlowTest.java

@@ -58,9 +58,9 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Streams;
-import com.googlecode.objectify.Key;
 import google.registry.batch.ResaveEntityAction;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.EppRequestSource;
 import google.registry.flows.FlowUtils.UnknownCurrencyEppException;
 import google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException;
@@ -105,10 +105,12 @@ import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Stream;
@@ -164,7 +166,7 @@ class DomainTransferRequestFlowTest
           .build();
 
   @BeforeEach
-  void setUp() {
+  void beforeEach() {
     setEppInput("domain_transfer_request.xml");
     setRegistrarIdForFlow("NewRegistrar");
   }
@@ -518,7 +520,7 @@ class DomainTransferRequestFlowTest
             .method("POST")
             .header("Host", "backend.hostname.fake")
             .header("content-type", "application/x-www-form-urlencoded")
-            .param(PARAM_RESOURCE_KEY, Key.create(domain).getString())
+            .param(PARAM_RESOURCE_KEY, domain.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, clock.nowUtc().toString())
             .etaDelta(
                 registry.getAutomaticTransferLength().minus(standardSeconds(30)),
@@ -1541,4 +1543,15 @@ class DomainTransferRequestFlowTest
             DomainTransactionRecord.create(
                 "tld", clock.nowUtc().plusDays(5), TRANSFER_SUCCESSFUL, 1));
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    setupDomain("example", "tld");
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown =
+        assertThrows(
+            ReadOnlyModeEppException.class, () -> doFailingTest("domain_transfer_request.xml"));
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -19,12 +19,22 @@ import static com.google.common.collect.Sets.union;
 import static com.google.common.io.BaseEncoding.base16;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
+import static google.registry.model.eppcommon.StatusValue.CLIENT_DELETE_PROHIBITED;
+import static google.registry.model.eppcommon.StatusValue.CLIENT_HOLD;
+import static google.registry.model.eppcommon.StatusValue.CLIENT_RENEW_PROHIBITED;
+import static google.registry.model.eppcommon.StatusValue.SERVER_DELETE_PROHIBITED;
+import static google.registry.model.eppcommon.StatusValue.SERVER_HOLD;
+import static google.registry.model.eppcommon.StatusValue.SERVER_TRANSFER_PROHIBITED;
 import static google.registry.model.eppcommon.StatusValue.SERVER_UPDATE_PROHIBITED;
+import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_CREATE;
+import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_UPDATE;
 import static google.registry.model.tld.Registry.TldState.QUIET_PERIOD;
 import static google.registry.testing.DatabaseHelper.assertBillingEvents;
 import static google.registry.testing.DatabaseHelper.assertNoBillingEvents;
+import static google.registry.testing.DatabaseHelper.assertPollMessagesForResource;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.getOnlyHistoryEntryOfType;
+import static google.registry.testing.DatabaseHelper.getPollMessages;
 import static google.registry.testing.DatabaseHelper.loadByKey;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
@@ -49,6 +59,7 @@ import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.EppException.UnimplementedExtensionException;
 import google.registry.flows.EppRequestSource;
 import google.registry.flows.ResourceFlowTestCase;
@@ -88,12 +99,14 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.secdns.DelegationSignerData;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
-import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.poll.PollMessage;
 import google.registry.model.tld.Registry;
 import google.registry.persistence.VKey;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -123,7 +136,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   final ReplayExtension replayExtension = ReplayExtension.createWithDoubleReplay(clock);
 
   @BeforeEach
-  void initDomainTest() {
+  void beforeEach() {
     createTld("tld");
     // Note that "domain_update.xml" tests adding and removing the same contact type.
     setEppInput("domain_update.xml");
@@ -155,7 +168,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
                 .build());
     persistResource(
         new DomainHistory.Builder()
-            .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setType(DOMAIN_CREATE)
             .setModificationTime(clock.nowUtc())
             .setRegistrarId(domain.getCreationRegistrarId())
             .setDomain(domain)
@@ -179,7 +192,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
                 .build());
     persistResource(
         new DomainHistory.Builder()
-            .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setType(DOMAIN_CREATE)
             .setModificationTime(clock.nowUtc())
             .setRegistrarId(domain.getCreationRegistrarId())
             .setDomain(domain)
@@ -202,8 +215,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         .and()
         .hasAuthInfoPwd("2BARfoo")
         .and()
-        .hasOneHistoryEntryEachOfTypes(
-            HistoryEntry.Type.DOMAIN_CREATE, HistoryEntry.Type.DOMAIN_UPDATE)
+        .hasOneHistoryEntryEachOfTypes(DOMAIN_CREATE, DOMAIN_UPDATE)
         .and()
         .hasLastEppUpdateTime(clock.nowUtc())
         .and()
@@ -329,10 +341,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     assertTransactionalFlow(true);
     runFlowAssertResponse(loadFile("generic_success_response.xml"));
     DomainBase domain = reloadResourceByForeignKey();
-    assertAboutDomains()
-        .that(domain)
-        .hasOneHistoryEntryEachOfTypes(
-            HistoryEntry.Type.DOMAIN_CREATE, HistoryEntry.Type.DOMAIN_UPDATE);
+    assertAboutDomains().that(domain).hasOneHistoryEntryEachOfTypes(DOMAIN_CREATE, DOMAIN_UPDATE);
     assertThat(domain.getNameservers()).hasSize(13);
     // getContacts does not return contacts of type REGISTRANT, so check these separately.
     assertThat(domain.getContacts()).hasSize(3);
@@ -349,12 +358,9 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     persistDomain();
     runFlow();
     DomainBase domain = reloadResourceByForeignKey();
-    assertAboutDomains()
-        .that(domain)
-        .hasOneHistoryEntryEachOfTypes(
-            HistoryEntry.Type.DOMAIN_CREATE, HistoryEntry.Type.DOMAIN_UPDATE);
+    assertAboutDomains().that(domain).hasOneHistoryEntryEachOfTypes(DOMAIN_CREATE, DOMAIN_UPDATE);
     assertAboutHistoryEntries()
-        .that(getOnlyHistoryEntryOfType(domain, HistoryEntry.Type.DOMAIN_UPDATE))
+        .that(getOnlyHistoryEntryOfType(domain, DOMAIN_UPDATE))
         .hasMetadataReason("domain-update-test")
         .and()
         .hasMetadataRequestedByRegistrar(true);
@@ -482,10 +488,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     clock.advanceOneMilli();
     runFlowAssertResponse(loadFile("generic_success_response.xml"));
     DomainBase resource = reloadResourceByForeignKey();
-    assertAboutDomains()
-        .that(resource)
-        .hasOnlyOneHistoryEntryWhich()
-        .hasType(HistoryEntry.Type.DOMAIN_UPDATE);
+    assertAboutDomains().that(resource).hasOnlyOneHistoryEntryWhich().hasType(DOMAIN_UPDATE);
     assertThat(resource.getDsData())
         .isEqualTo(
             expectedDsData.stream()
@@ -687,9 +690,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
               .setBillingTime(clock.nowUtc())
               .setParent(
                   getOnlyHistoryEntryOfType(
-                      reloadResourceByForeignKey(),
-                      HistoryEntry.Type.DOMAIN_UPDATE,
-                      DomainHistory.class))
+                      reloadResourceByForeignKey(), DOMAIN_UPDATE, DomainHistory.class))
               .build());
     } else {
       assertNoBillingEvents();
@@ -768,7 +769,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         .that(reloadResourceByForeignKey())
         .hasStatusValue(StatusValue.CLIENT_UPDATE_PROHIBITED)
         .and()
-        .hasStatusValue(StatusValue.SERVER_HOLD);
+        .hasStatusValue(SERVER_HOLD);
   }
 
   private void doSecDnsFailingTest(
@@ -910,12 +911,103 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   }
 
   @TestOfyAndSql
-  void testSuccess_superuserStatusValueNotClientSettable() throws Exception {
+  void testSuccess_superuserCanSetServerStatusValues() throws Exception {
     setEppInput("domain_update_prohibited_status.xml");
     persistReferencedEntities();
     persistDomain();
     runFlowAssertResponse(
         CommitMode.LIVE, UserPrivileges.SUPERUSER, loadFile("generic_success_response.xml"));
+    // No poll message because the server status was added by the owning registrar.
+    assertThat(getPollMessages()).isEmpty();
+  }
+
+  @TestOfyAndSql
+  void testSuccess_addingServerStatusValue_sendsPollMessage() throws Exception {
+    setEppInput("domain_update_prohibited_status.xml");
+    persistReferencedEntities();
+    persistDomain();
+    persistResource(
+        reloadResourceByForeignKey()
+            .asBuilder()
+            .setPersistedCurrentSponsorRegistrarId("NewRegistrar")
+            .build());
+    runFlowAssertResponse(
+        CommitMode.LIVE, UserPrivileges.SUPERUSER, loadFile("generic_success_response.xml"));
+    DomainBase updatedDomain = reloadResourceByForeignKey();
+    assertPollMessagesForResource(
+        updatedDomain,
+        new PollMessage.OneTime.Builder()
+            .setEventTime(clock.nowUtc())
+            .setParent(getOnlyHistoryEntryOfType(updatedDomain, DOMAIN_UPDATE))
+            .setRegistrarId("NewRegistrar")
+            .setMsg("The registry administrator has added the status(es) [serverHold].")
+            .build());
+  }
+
+  @TestOfyAndSql
+  void testSuccess_removingServerStatusValue_sendsPollMessage() throws Exception {
+    setEppInput("domain_update_remove_server_statuses.xml");
+    persistReferencedEntities();
+    persistDomain();
+    persistResource(
+        reloadResourceByForeignKey()
+            .asBuilder()
+            .setPersistedCurrentSponsorRegistrarId("NewRegistrar")
+            .setStatusValues(
+                ImmutableSet.of(
+                    SERVER_DELETE_PROHIBITED,
+                    SERVER_TRANSFER_PROHIBITED,
+                    SERVER_UPDATE_PROHIBITED,
+                    CLIENT_DELETE_PROHIBITED,
+                    CLIENT_RENEW_PROHIBITED,
+                    CLIENT_HOLD,
+                    SERVER_HOLD))
+            .build());
+    runFlowAssertResponse(
+        CommitMode.LIVE, UserPrivileges.SUPERUSER, loadFile("generic_success_response.xml"));
+    DomainBase updatedDomain = reloadResourceByForeignKey();
+    assertPollMessagesForResource(
+        updatedDomain,
+        new PollMessage.OneTime.Builder()
+            .setEventTime(clock.nowUtc())
+            .setParent(getOnlyHistoryEntryOfType(updatedDomain, DOMAIN_UPDATE))
+            .setRegistrarId("NewRegistrar")
+            .setMsg(
+                "The registry administrator has removed the status(es) [serverHold,"
+                    + " serverTransferProhibited, serverUpdateProhibited].")
+            .build());
+  }
+
+  @TestOfyAndSql
+  void testSuccess_addingAndRemovingServerStatusValues_sendsPollMessage() throws Exception {
+    setEppInput("domain_update_change_server_statuses.xml");
+    persistReferencedEntities();
+    persistDomain();
+    persistResource(
+        reloadResourceByForeignKey()
+            .asBuilder()
+            .setPersistedCurrentSponsorRegistrarId("NewRegistrar")
+            .setStatusValues(
+                ImmutableSet.of(
+                    SERVER_DELETE_PROHIBITED,
+                    SERVER_TRANSFER_PROHIBITED,
+                    CLIENT_DELETE_PROHIBITED,
+                    CLIENT_RENEW_PROHIBITED))
+            .build());
+    runFlowAssertResponse(
+        CommitMode.LIVE, UserPrivileges.SUPERUSER, loadFile("generic_success_response.xml"));
+    DomainBase updatedDomain = reloadResourceByForeignKey();
+    assertPollMessagesForResource(
+        updatedDomain,
+        new PollMessage.OneTime.Builder()
+            .setEventTime(clock.nowUtc())
+            .setParent(getOnlyHistoryEntryOfType(updatedDomain, DOMAIN_UPDATE))
+            .setRegistrarId("NewRegistrar")
+            .setMsg(
+                "The registry administrator has added the status(es) [serverHold,"
+                    + " serverRenewProhibited] and removed the status(es)"
+                    + " [serverTransferProhibited].")
+            .build());
   }
 
   @TestOfyAndSql
@@ -1406,4 +1498,14 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     runFlowAsSuperuser();
     assertAboutDomains().that(reloadResourceByForeignKey()).hasNoAutorenewEndTime();
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() throws Exception {
+    persistReferencedEntities();
+    persistDomain();
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/host/HostCreateFlowTest.java

@@ -36,6 +36,7 @@ import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.FlowUtils.IpAddressVersionMismatchException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.exceptions.ResourceAlreadyExistsForThisClientException;
@@ -54,9 +55,11 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.extension.RegisterExtension;
@@ -329,4 +332,12 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-host-create");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/flows/host/HostDeleteFlowTest.java

@@ -35,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
 import google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException;
@@ -51,9 +52,11 @@ import google.registry.model.reporting.HistoryEntry.Type;
 import google.registry.model.tld.Registry;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
@@ -338,6 +341,15 @@ class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, HostResour
     assertIcannReportingActivityFieldLogged("srs-host-delete");
   }
 
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() {
+    persistActiveHost("ns1.example.tld");
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   private void assertOfyDeleteSuccess(String registrarId, String clientTrid, boolean isSuperuser)
       throws Exception {
     HostResource deletedHost = reloadResourceByForeignKey();

core/src/test/java/google/registry/flows/host/HostUpdateFlowTest.java

@@ -46,8 +46,8 @@ import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.EppRequestSource;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException;
@@ -78,10 +78,12 @@ import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.tld.Registry;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import javax.annotation.Nullable;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Order;
@@ -220,7 +222,7 @@ class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, HostResour
     assertTasksEnqueued(
         QUEUE_ASYNC_HOST_RENAME,
         new TaskMatcher()
-            .param("hostKey", Key.create(renamedHost).getString())
+            .param("hostKey", renamedHost.createVKey().getOfyKey().getString())
             .param("requestedTime", clock.nowUtc().toString()));
   }
 
@@ -1342,4 +1344,14 @@ class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, HostResour
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-host-update");
   }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase() throws Exception {
+    createTld("tld");
+    persistActiveSubordinateHost(oldHostName(), persistActiveDomain("example.tld"));
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
 }

core/src/test/java/google/registry/model/billing/BillingEventTest.java

@@ -416,4 +416,62 @@ public class BillingEventTest extends EntityTestCase {
     assertThat(recurring.getParentKey()).isEqualTo(Key.create(domainHistory));
     new BillingEvent.OneTime.Builder().setParent(Key.create(domainHistory));
   }
+
+  @TestOfyAndSql
+  void testReasonRequiringPeriodYears_missingPeriodYears_throwsException() {
+    IllegalStateException thrown =
+        assertThrows(
+            IllegalStateException.class,
+            () ->
+                new BillingEvent.OneTime.Builder()
+                    .setBillingTime(DateTime.parse("2020-02-05T15:33:11Z"))
+                    .setEventTime(DateTime.parse("2020-01-05T15:33:11Z"))
+                    .setCost(Money.of(USD, 10))
+                    .setReason(Reason.RENEW)
+                    .setCost(Money.of(USD, 10))
+                    .setRegistrarId("TheRegistrar")
+                    .setTargetId("example.tld")
+                    .setParent(domainHistory)
+                    .build());
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains("Period years must be set if and only if reason is");
+  }
+
+  @TestOfyAndSql
+  void testReasonNotRequiringPeriodYears_havingPeriodYears_throwsException() {
+    IllegalStateException thrown =
+        assertThrows(
+            IllegalStateException.class,
+            () ->
+                new BillingEvent.OneTime.Builder()
+                    .setBillingTime(DateTime.parse("2020-02-05T15:33:11Z"))
+                    .setEventTime(DateTime.parse("2020-01-05T15:33:11Z"))
+                    .setCost(Money.of(USD, 10))
+                    .setPeriodYears(2)
+                    .setReason(Reason.SERVER_STATUS)
+                    .setCost(Money.of(USD, 10))
+                    .setRegistrarId("TheRegistrar")
+                    .setTargetId("example.tld")
+                    .setParent(domainHistory)
+                    .build());
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains("Period years must be set if and only if reason is");
+  }
+
+  @TestOfyAndSql
+  void testReasonRequiringPeriodYears_missingPeriodYears_isAllowedOnOldData() {
+    // This won't throw even though periodYears is missing on a RESTORE because the event time
+    // is before 2019.
+    new BillingEvent.OneTime.Builder()
+        .setBillingTime(DateTime.parse("2018-02-05T15:33:11Z"))
+        .setEventTime(DateTime.parse("2018-01-05T15:33:11Z"))
+        .setReason(Reason.RESTORE)
+        .setCost(Money.of(USD, 10))
+        .setRegistrarId("TheRegistrar")
+        .setTargetId("example.tld")
+        .setParent(domainHistory)
+        .build();
+  }
 }

core/src/test/java/google/registry/model/domain/DomainBaseSqlTest.java

@@ -657,20 +657,18 @@ public class DomainBaseSqlTest {
   }
 
   private void assertEqualDomainExcept(DomainBase thatDomain, String... excepts) {
-    // Fix the original creation timestamp (this gets initialized on first write)
-    DomainBase org = domain.asBuilder().setCreationTime(thatDomain.getCreationTime()).build();
-
     ImmutableList<String> moreExcepts =
         new ImmutableList.Builder<String>()
             .addAll(Arrays.asList(excepts))
+            .add("creationTime")
             .add("updateTimestamp")
             .add("transferData")
             .build();
     // Note that the equality comparison forces a lazy load of all fields.
-    assertAboutImmutableObjects().that(thatDomain).isEqualExceptFields(org, moreExcepts);
+    assertAboutImmutableObjects().that(thatDomain).isEqualExceptFields(domain, moreExcepts);
     // Transfer data cannot be directly compared due to serverApproveEtities inequalities
     assertAboutImmutableObjects()
         .that(domain.getTransferData())
-        .isEqualExceptFields(org.getTransferData(), "serverApproveEntities");
+        .isEqualExceptFields(thatDomain.getTransferData(), "serverApproveEntities");
   }
 }

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -24,6 +24,7 @@ import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.junit.Assert.assertThrows;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -33,6 +34,8 @@ import com.google.common.testing.TestLogHandler;
 import com.google.common.truth.Truth8;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.ofy.CommitLogBucket;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.server.Lock;
@@ -64,8 +67,8 @@ public class ReplicateToDatastoreActionTest {
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder()
           .withDatastoreAndCloudSql()
-          .withOfyTestEntities(Lock.class, TestObject.class)
-          .withJpaUnitTestEntities(Lock.class, TestObject.class)
+          .withOfyTestEntities(AllocationToken.class, Lock.class, TestObject.class)
+          .withJpaUnitTestEntities(AllocationToken.class, Lock.class, TestObject.class)
           .withClock(fakeClock)
           .build();
 
@@ -97,9 +100,7 @@ public class ReplicateToDatastoreActionTest {
 
   @RetryingTest(4)
   void testReplication() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     TestObject foo = TestObject.create("foo");
     TestObject bar = TestObject.create("bar");
@@ -126,9 +127,7 @@ public class ReplicateToDatastoreActionTest {
 
   @RetryingTest(4)
   void testReplayFromLastTxn() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     TestObject foo = TestObject.create("foo");
     TestObject bar = TestObject.create("bar");
@@ -152,9 +151,7 @@ public class ReplicateToDatastoreActionTest {
 
   @RetryingTest(4)
   void testUnintentionalConcurrency() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     TestObject foo = TestObject.create("foo");
     TestObject bar = TestObject.create("bar");
@@ -189,11 +186,24 @@ public class ReplicateToDatastoreActionTest {
             Level.WARNING, "Ignoring transaction 1, which appears to have already been applied.");
   }
 
+  @RetryingTest(4)
+  void testCreateAutoTimestamp() {
+    // Verify that fields populated by the DB (e.g. CreateAutoTimestamp) correctly get populated in
+    // both databases.
+    assumeTrue(ReplayExtension.replayTestsEnabled());
+
+    AllocationToken allocationToken =
+        new AllocationToken.Builder().setToken("abc123").setTokenType(TokenType.SINGLE_USE).build();
+    insertInDb(allocationToken);
+    runAndVerifySuccess();
+
+    assertThat(ofyTm().transact(() -> ofyTm().loadByEntity(allocationToken)))
+        .isEqualTo(jpaTm().transact(() -> jpaTm().loadByEntity(allocationToken)));
+  }
+
   @RetryingTest(4)
   void testMissingTransactions() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     // Write a transaction (should have a transaction id of 1).
     TestObject foo = TestObject.create("foo");
@@ -212,9 +222,7 @@ public class ReplicateToDatastoreActionTest {
 
   @Test
   void testMissingTransactions_fullTask() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     // Write a transaction (should have a transaction id of 1).
     TestObject foo = TestObject.create("foo");
@@ -234,9 +242,7 @@ public class ReplicateToDatastoreActionTest {
 
   @Test
   void testBeforeDatastoreSaveCallback() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     TestObject testObject = TestObject.create("foo");
     insertInDb(testObject);
@@ -247,9 +253,7 @@ public class ReplicateToDatastoreActionTest {
 
   @Test
   void testNotInMigrationState_doesNothing() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     // set a schedule that backtracks the current status to DATASTORE_PRIMARY
     DateTime now = fakeClock.nowUtc();
@@ -287,9 +291,7 @@ public class ReplicateToDatastoreActionTest {
 
   @Test
   void testFailure_cannotAcquireLock() {
-    if (!ReplayExtension.replayTestsEnabled()) {
-      return;
-    }
+    assumeTrue(ReplayExtension.replayTestsEnabled());
 
     RequestStatusChecker requestStatusChecker = mock(RequestStatusChecker.class);
     when(requestStatusChecker.getLogId()).thenReturn("logId");

core/src/test/java/google/registry/model/tmch/TmchCrlTest.java

@@ -15,39 +15,26 @@
 package google.registry.model.tmch;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.testing.DatabaseHelper.loadByEntity;
 
 import google.registry.model.EntityTestCase;
-import google.registry.testing.DualDatabaseTest;
-import google.registry.testing.TestOfyAndSql;
 import java.util.Optional;
+import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link TmchCrl}. */
-@DualDatabaseTest
 public class TmchCrlTest extends EntityTestCase {
 
   TmchCrlTest() {
     super(JpaEntityCoverageCheck.ENABLED);
   }
 
-  @TestOfyAndSql
+  @Test
   void testSuccess() {
     assertThat(TmchCrl.get()).isEqualTo(Optional.empty());
     TmchCrl.set("lolcat", "https://lol.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("lolcat");
   }
 
-  @TestOfyAndSql
-  void testDualWrite() {
-    TmchCrl expected = new TmchCrl();
-    expected.crl = "lolcat";
-    expected.url = "https://lol.cat";
-    expected.updated = fakeClock.nowUtc();
-    TmchCrl.set("lolcat", "https://lol.cat");
-    assertThat(loadByEntity(new TmchCrl())).isEqualTo(expected);
-  }
-
-  @TestOfyAndSql
+  @Test
   void testMultipleWrites() {
     TmchCrl.set("first", "https://first.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("first");

core/src/test/java/google/registry/persistence/converter/CreateAutoTimestampConverterTest.java

@@ -1,82 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package google.registry.persistence.converter;
-
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.testing.DatabaseHelper.insertInDb;
-
-import google.registry.model.CreateAutoTimestamp;
-import google.registry.model.ImmutableObject;
-import google.registry.model.replay.EntityTest.EntityForTesting;
-import google.registry.persistence.transaction.JpaTestExtensions;
-import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension;
-import google.registry.testing.FakeClock;
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import org.joda.time.DateTime;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-/** Unit tests for {@link CreateAutoTimestampConverter}. */
-public class CreateAutoTimestampConverterTest {
-
-  private final FakeClock fakeClock = new FakeClock();
-
-  @RegisterExtension
-  public final JpaUnitTestExtension jpaExtension =
-      new JpaTestExtensions.Builder()
-          .withClock(fakeClock)
-          .withEntityClass(TestEntity.class)
-          .buildUnitTestExtension();
-
-  @Test
-  void testTypeConversion() {
-    CreateAutoTimestamp ts = CreateAutoTimestamp.create(DateTime.parse("2019-09-9T11:39:00Z"));
-    TestEntity ent = new TestEntity("myinst", ts);
-
-    insertInDb(ent);
-    TestEntity result =
-        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "myinst"));
-    assertThat(result).isEqualTo(new TestEntity("myinst", ts));
-  }
-
-  @Test
-  void testAutoInitialization() {
-    CreateAutoTimestamp ts = CreateAutoTimestamp.create(null);
-    TestEntity ent = new TestEntity("autoinit", ts);
-
-    insertInDb(ent);
-
-    TestEntity result =
-        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "autoinit"));
-    assertThat(result.cat.getTimestamp()).isEqualTo(fakeClock.nowUtc());
-  }
-
-  @Entity(name = "TestEntity") // Override entity name to avoid the nested class reference.
-  @EntityForTesting
-  public static class TestEntity extends ImmutableObject {
-
-    @Id String name;
-
-    CreateAutoTimestamp cat;
-
-    public TestEntity() {}
-
-    TestEntity(String name, CreateAutoTimestamp cat) {
-      this.name = name;
-      this.cat = cat;
-    }
-  }
-}

core/src/test/java/google/registry/rde/RdeStagingActionCloudSqlTest.java

@@ -79,6 +79,7 @@ public class RdeStagingActionCloudSqlTest extends BeamActionTestBase {
     action.watermarks = ImmutableSet.of();
     action.revision = Optional.empty();
     action.dataflow = dataflow;
+    action.machineType = "machine-type";
   }
 
   @TestSqlOnly
@@ -156,7 +157,7 @@ public class RdeStagingActionCloudSqlTest extends BeamActionTestBase {
   }
 
   @TestSqlOnly
-  void testRun_afterTransactionCooldown_runsMapReduce() throws Exception {
+  void testRun_afterTransactionCooldown_runsPipeline() throws Exception {
     createTldWithEscrowEnabled("lol");
     clock.setTo(DateTime.parse("2000-01-01T00:05:00Z"));
     action.transactionCooldown = Duration.standardMinutes(5);
@@ -241,18 +242,19 @@ public class RdeStagingActionCloudSqlTest extends BeamActionTestBase {
   }
 
   @TestSqlOnly
-  void testManualRun_validParameters_runsMapReduce() throws Exception {
+  void testManualRun_validParameters_runsPipeline() throws Exception {
     createTldWithEscrowEnabled("lol");
     clock.setTo(DateTime.parse("2000-01-01TZ"));
     action.manual = true;
     action.directory = Optional.of("test/");
     action.modeStrings = ImmutableSet.of("full");
     action.tlds = ImmutableSet.of("lol");
-    action.watermarks = ImmutableSet.of(DateTime.parse("2001-01-01TZ"));
+    action.watermarks =
+        ImmutableSet.of(DateTime.parse("1999-12-31TZ"), DateTime.parse("2001-01-01TZ"));
     action.run();
     assertThat(response.getStatus()).isEqualTo(200);
-    assertThat(response.getPayload()).contains("Launched RDE pipeline: jobid");
-    verify(templates, times(1))
+    assertThat(response.getPayload()).contains("Launched RDE pipeline: jobid, jobid1");
+    verify(templates, times(2))
         .launch(eq("projectId"), eq("jobRegion"), any(LaunchFlexTemplateRequest.class));
   }
 

core/src/test/java/google/registry/testing/CloudTasksHelper.java

@@ -188,7 +188,12 @@ public class CloudTasksHelper implements Serializable {
 
     String taskName;
     String service;
-    HttpMethod method;
+    // App Engine TaskOption methods default to "POST".  This isn't obvious from the code, so we
+    // default it to POST here so that we don't accidentally create an entry with a GET method when
+    // converting to CloudTaskUtils, which requires that the method be specified explicitly.
+    // Should we ever actually want to do a GET, we'll likewise have to set this explicitly for the
+    // tests.
+    HttpMethod method = HttpMethod.POST;
     String url;
     Multimap<String, String> headers = ArrayListMultimap.create();
     Multimap<String, String> params = ArrayListMultimap.create();

core/src/test/resources/google/registry/export/crosstld_kinds.txt

@@ -3,4 +3,3 @@ Registrar
 RegistrarContact
 Registry
 ServerSecret
-TmchCrl

core/src/test/resources/google/registry/flows/domain/domain_update_change_server_statuses.xml

@@ -0,0 +1,23 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <update>
+      <domain:update
+       xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>example.tld</domain:name>
+        <domain:add>
+          <domain:status s="serverHold" lang="en">
+            Payment overdue.
+          </domain:status>
+          <domain:status s="serverRenewProhibited"/>
+          <domain:status s="clientUpdateProhibited"/>
+        </domain:add>
+        <domain:rem>
+          <domain:status s="clientRenewProhibited"/>
+          <domain:status s="serverTransferProhibited"/>
+        </domain:rem>
+        <domain:chg />
+      </domain:update>
+    </update>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>