Add more options to the generate escrow deposit command (#1453)

This adds two new options:

1) An option to run RDE in lenient mode.
2) An option to run RDE with the new Beam pipeline regardless of the datastore setting.

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1453)
<!-- Reviewable:end -->

build.gradle

@@ -46,7 +46,7 @@ plugins {
   id 'com.diffplug.gradle.spotless' version '3.25.0'
 
   id 'jacoco'
-  id 'com.dorongold.task-tree' version '1.5'
+  id 'com.dorongold.task-tree' version '2.1.0'
 }
 
 node {

core/build.gradle

@@ -703,6 +703,12 @@ createToolTask(
     'google.registry.tools.DevTool',
     sourceSets.nonprod)
 
+createToolTask(
+    'initSqlPipeline', 'google.registry.beam.initsql.InitSqlPipeline')
+
+createToolTask(
+    'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
+
 
 createToolTask(
     'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
@@ -711,30 +717,6 @@ createToolTask(
     'createSyntheticHistoryEntries',
     'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
-project.tasks.create('initSqlPipeline', JavaExec) {
-  main = 'google.registry.beam.initsql.InitSqlPipeline'
-
-  doFirst {
-    getToolArgsList().ifPresent {
-      args it
-    }
-
-    def isDirectRunner =
-        args.contains('DirectRunner') || args.contains('--runner=DirectRunner')
-    // The dependency containing DirectRunner is intentionally excluded from the
-    // production binary, so that it won't be chosen by mistake: we definitely do
-    // not want to use it for the real jobs, yet DirectRunner is the default if
-    // the user forgets to override it.
-    // DirectRunner is required for tests and is already on testRuntimeClasspath.
-    // For simplicity, we add testRuntimeClasspath to this task's classpath instead
-    // of defining a new configuration just for the DirectRunner dependency.
-    classpath =
-        isDirectRunner
-            ? sourceSets.main.runtimeClasspath.plus(sourceSets.test.runtimeClasspath)
-            : sourceSets.main.runtimeClasspath
-  }
-}
-
 // Caller must provide projectId, GCP region, runner, and the kinds to delete
 // (comma-separated kind names or '*' for all). E.g.:
 // nom_build :core:bulkDeleteDatastore --args="--project=domain-registry-crash \

core/src/main/java/google/registry/backup/BackupUtils.java

@@ -45,7 +45,7 @@ public class BackupUtils {
    * OutputStream} in delimited protocol buffer format.
    */
   static void serializeEntity(ImmutableObject entity, OutputStream stream) throws IOException {
-    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnly().toEntity(entity))
+    EntityTranslator.convertToPb(auditedOfy().saveIgnoringReadOnlyWithoutBackup().toEntity(entity))
         .writeDelimitedTo(stream);
   }
 

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -74,7 +74,7 @@ public final class CommitLogCheckpointAction implements Runnable {
                 return;
               }
               auditedOfy()
-                  .saveIgnoringReadOnly()
+                  .saveIgnoringReadOnlyWithoutBackup()
                   .entities(
                       checkpoint, CommitLogCheckpointRoot.create(checkpoint.getCheckpointTime()));
               // Enqueue a diff task between previous and current checkpoints.

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -264,7 +264,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
             .ifPresent(
                 sqlEntity -> {
                   sqlEntity.beforeSqlSaveOnReplay();
-                  jpaTm().putIgnoringReadOnly(sqlEntity);
+                  jpaTm().putIgnoringReadOnlyWithoutBackup(sqlEntity);
                 });
       } else {
         // this should never happen, but we shouldn't fail on it
@@ -297,7 +297,7 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
           && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
           && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
         ReplaySpecializer.beforeSqlDelete(entityVKey);
-        jpaTm().deleteIgnoringReadOnly(entityVKey);
+        jpaTm().deleteIgnoringReadOnlyWithoutBackup(entityVKey);
       }
     } catch (Throwable t) {
       logger.atSevere().log("Error when deleting key %s.", entityVKey);

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -24,10 +24,8 @@ import com.google.appengine.api.taskqueue.TransientFailureException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.EppResource;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.RegistryLock;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostResource;
@@ -84,8 +82,7 @@ public final class AsyncTaskEnqueuer {
   }
 
   /** Enqueues a task to asynchronously re-save an entity at some point in the future. */
-  public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, DateTime whenToResave) {
+  public void enqueueAsyncResave(VKey<?> entityToResave, DateTime now, DateTime whenToResave) {
     enqueueAsyncResave(entityToResave, now, ImmutableSortedSet.of(whenToResave));
   }
 
@@ -96,10 +93,9 @@ public final class AsyncTaskEnqueuer {
    * itself to run at the next time if there are remaining re-saves scheduled.
    */
   public void enqueueAsyncResave(
-      ImmutableObject entityToResave, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
+      VKey<?> entityKey, DateTime now, ImmutableSortedSet<DateTime> whenToResave) {
     DateTime firstResave = whenToResave.first();
     checkArgument(isBeforeOrAt(now, firstResave), "Can't enqueue a resave to run in the past");
-    Key<ImmutableObject> entityKey = Key.create(entityToResave);
     Duration etaDuration = new Duration(now, firstResave);
     if (etaDuration.isLongerThan(MAX_ASYNC_ETA)) {
       logger.atInfo().log(
@@ -114,7 +110,7 @@ public final class AsyncTaskEnqueuer {
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())
-            .param(PARAM_RESOURCE_KEY, entityKey.getString())
+            .param(PARAM_RESOURCE_KEY, entityKey.getOfyKey().getString())
             .param(PARAM_REQUESTED_TIME, now.toString());
     if (whenToResave.size() > 1) {
       task.param(PARAM_RESAVE_TIMES, Joiner.on(',').join(whenToResave.tailSet(firstResave, false)));
@@ -129,14 +125,13 @@ public final class AsyncTaskEnqueuer {
       String requestingRegistrarId,
       Trid trid,
       boolean isSuperuser) {
-    Key<EppResource> resourceKey = Key.create(resourceToDelete);
     logger.atInfo().log(
         "Enqueuing async deletion of %s on behalf of registrar %s.",
-        resourceKey, requestingRegistrarId);
+        resourceToDelete.getRepoId(), requestingRegistrarId);
     TaskOptions task =
         TaskOptions.Builder.withMethod(Method.PULL)
             .countdownMillis(asyncDeleteDelay.getMillis())
-            .param(PARAM_RESOURCE_KEY, resourceKey.getString())
+            .param(PARAM_RESOURCE_KEY, resourceToDelete.createVKey().getOfyKey().getString())
             .param(PARAM_REQUESTING_CLIENT_ID, requestingRegistrarId)
             .param(PARAM_SERVER_TRANSACTION_ID, trid.getServerTransactionId())
             .param(PARAM_IS_SUPERUSER, Boolean.toString(isSuperuser))

core/src/main/java/google/registry/batch/BatchModule.java

@@ -79,6 +79,7 @@ public class BatchModule {
 
   @Provides
   @Parameter(PARAM_RESOURCE_KEY)
+  // TODO(b/207363014): figure out if this needs to be modified for vkey string replacement
   static Key<ImmutableObject> provideResourceKey(HttpServletRequest req) {
     return Key.create(extractRequiredParameter(req, PARAM_RESOURCE_KEY));
   }

core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java

@@ -523,18 +523,19 @@ public class DeleteContactsAndHostsAction implements Runnable {
     static DeletionRequest createFromTask(TaskHandle task, DateTime now)
         throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<EppResource> resourceKey =
-          Key.create(
+      VKey<EppResource> resourceKey =
+          VKey.create(
               checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(), "Resource to delete doesn't exist");
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
+              "Resource to delete doesn't exist");
       checkState(
           resource instanceof ContactResource || resource instanceof HostResource,
           "Cannot delete a %s via this action",
           resource.getClass().getSimpleName());
       return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
-          .setKey(resourceKey)
+          .setKey(resourceKey.getOfyKey())
           .setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
           .setRequestingClientId(
               checkNotNull(

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -353,8 +353,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
       VKey<HostResource> hostKey =
-          VKey.fromWebsafeKey(
-              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+          VKey.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
           tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
               .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -76,13 +76,15 @@ public class ResaveEntityAction implements Runnable {
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
     tm().transact(
             () -> {
+              // TODO(/207363014): figure out if this should modified for vkey string replacement
               ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
               tm().put(
                       (entity instanceof EppResource)
                           ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
                           : entity);
               if (!resaveTimes.isEmpty()) {
-                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+                asyncTaskEnqueuer.enqueueAsyncResave(
+                    VKey.from(resourceKey), requestedTime, resaveTimes);
               }
             });
     response.setPayload("Entity re-saved.");

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -175,7 +175,7 @@ public final class RegistryJpaIO {
      * JpaTransactionManager#setDatabaseSnapshot}.
      */
     // TODO(b/193662898): vendor-independent support for richer transaction semantics.
-    public Read<R, T> withSnapshot(String snapshotId) {
+    public Read<R, T> withSnapshot(@Nullable String snapshotId) {
       return toBuilder().snapshotId(snapshotId).build();
     }
 

core/src/main/java/google/registry/beam/common/RegistryPipelineWorkerInitializer.java

@@ -63,7 +63,8 @@ public class RegistryPipelineWorkerInitializer implements JvmInitializer {
     TransactionManagerFactory.setJpaTmOnBeamWorker(transactionManagerLazy::get);
     // Masquerade all threads as App Engine threads so we can create Ofy keys in the pipeline. Also
     // loads all ofy entities.
-    new AppEngineEnvironment("Beam").setEnvironmentForAllThreads();
+    new AppEngineEnvironment("s~" + registryPipelineComponent.getProjectId())
+        .setEnvironmentForAllThreads();
     // Set the system property so that we can call IdService.allocateId() without access to
     // datastore.
     SystemPropertySetter.PRODUCTION_IMPL.setProperty(PROPERTY, "true");

core/src/main/java/google/registry/beam/comparedb/DatastoreSnapshots.java

@@ -0,0 +1,169 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.initsql.Transforms.createTagForKind;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.backup.VersionedEntity;
+import google.registry.beam.initsql.Transforms;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.model.tld.Registry;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.apache.beam.sdk.values.TupleTagList;
+import org.joda.time.DateTime;
+
+/** Utilities for loading Datastore snapshots. */
+public final class DatastoreSnapshots {
+
+  private DatastoreSnapshots() {}
+
+  /**
+   * Datastore kinds eligible for validation. This set must be consistent with {@link
+   * SqlSnapshots#ALL_SQL_ENTITIES}.
+   */
+  @VisibleForTesting
+  static final ImmutableSet<Class<?>> ALL_DATASTORE_KINDS =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          HistoryEntry.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class);
+
+  /**
+   * Returns the Datastore snapshot right before {@code commitLogToTime} for the user specified
+   * {@code kinds}. The resulting snapshot has all changes that happened before {@code
+   * commitLogToTime}, and none at or after {@code commitLogToTime}.
+   *
+   * <p>If {@code HistoryEntry} is included in {@code kinds}, the result will contain {@code
+   * PCollections} for the child entities, {@code DomainHistory}, {@code ContactHistory}, and {@code
+   * HostHistory}.
+   */
+  static PCollectionTuple loadDatastoreSnapshotByKind(
+      Pipeline pipeline,
+      String exportDir,
+      String commitLogDir,
+      DateTime commitLogFromTime,
+      DateTime commitLogToTime,
+      Set<Class<?>> kinds) {
+    PCollectionTuple snapshot =
+        pipeline.apply(
+            "Load Datastore snapshot.",
+            Transforms.loadDatastoreSnapshot(
+                exportDir,
+                commitLogDir,
+                commitLogFromTime,
+                commitLogToTime,
+                kinds.stream().map(Key::getKind).collect(ImmutableSet.toImmutableSet())));
+
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<?> kind : kinds) {
+      PCollection<VersionedEntity> perKindSnapshot =
+          snapshot.get(createTagForKind(Key.getKind(kind)));
+      if (SqlEntity.class.isAssignableFrom(kind)) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag((Class<? extends SqlEntity>) kind),
+                datastoreEntityToPojo(perKindSnapshot, kind.getSimpleName()));
+        continue;
+      }
+      Verify.verify(kind == HistoryEntry.class, "Unexpected Non-SqlEntity class: %s", kind);
+      PCollectionTuple historyEntriesByType = splitHistoryEntry(perKindSnapshot);
+      for (Map.Entry<TupleTag<?>, PCollection<?>> entry :
+          historyEntriesByType.getAll().entrySet()) {
+        perTypeSnapshots = perTypeSnapshots.and(entry.getKey().getId(), entry.getValue());
+      }
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Splits a {@link PCollection} of {@link HistoryEntry HistoryEntries} into three collections of
+   * its child entities by type.
+   */
+  static PCollectionTuple splitHistoryEntry(PCollection<VersionedEntity> historyEntries) {
+    return historyEntries.apply(
+        "Split HistoryEntry by Resource Type",
+        ParDo.of(
+                new DoFn<VersionedEntity, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element VersionedEntity historyEntry, MultiOutputReceiver out) {
+                    Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(historyEntry))
+                        .ifPresent(
+                            sqlEntity ->
+                                out.get(createSqlEntityTupleTag(sqlEntity.getClass()))
+                                    .output(sqlEntity));
+                  }
+                })
+            .withOutputTags(
+                createSqlEntityTupleTag(DomainHistory.class),
+                TupleTagList.of(createSqlEntityTupleTag(ContactHistory.class))
+                    .and(createSqlEntityTupleTag(HostHistory.class))));
+  }
+
+  /**
+   * Transforms a {@link PCollection} of {@link VersionedEntity VersionedEntities} to Ofy Java
+   * objects.
+   */
+  static PCollection<SqlEntity> datastoreEntityToPojo(
+      PCollection<VersionedEntity> entities, String desc) {
+    return entities.apply(
+        "Datastore Entity to Pojo " + desc,
+        ParDo.of(
+            new DoFn<VersionedEntity, SqlEntity>() {
+              @ProcessElement
+              public void processElement(
+                  @Element VersionedEntity entity, OutputReceiver<SqlEntity> out) {
+                Optional.ofNullable(Transforms.convertVersionedEntityToSqlEntity(entity))
+                    .ifPresent(out::output);
+              }
+            }));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -0,0 +1,145 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import com.google.auto.value.AutoValue;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import dagger.Component;
+import google.registry.config.CloudTasksUtilsModule;
+import google.registry.config.CredentialModule;
+import google.registry.config.RegistryConfig;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.util.Clock;
+import google.registry.util.UtilsModule;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.NoSuchElementException;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import org.joda.time.DateTime;
+import org.joda.time.Instant;
+import org.joda.time.Interval;
+
+/** Finds the necessary information for loading the most recent Datastore snapshot. */
+public class LatestDatastoreSnapshotFinder {
+  private final String projectId;
+  private final GcsUtils gcsUtils;
+  private final Clock clock;
+
+  @Inject
+  LatestDatastoreSnapshotFinder(
+      @Config("projectId") String projectId, GcsUtils gcsUtils, Clock clock) {
+    this.projectId = projectId;
+    this.gcsUtils = gcsUtils;
+    this.clock = clock;
+  }
+
+  /**
+   * Finds information of the most recent Datastore snapshot, including the GCS folder of the
+   * exported data files and the start and stop times of the export. The folder of the CommitLogs is
+   * also included in the return.
+   */
+  public DatastoreSnapshotInfo getSnapshotInfo() {
+    String bucketName = RegistryConfig.getDatastoreBackupsBucket().substring("gs://".length());
+    /**
+     * Find the bucket-relative path to the overall metadata file of the last Datastore export.
+     * Since Datastore export is saved daily, we may need to look back to yesterday. If found, the
+     * return value is like
+     * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
+     */
+    Optional<String> metaFilePathOptional = findMostRecentExportMetadataFile(bucketName, 2);
+    if (!metaFilePathOptional.isPresent()) {
+      throw new NoSuchElementException("No exports found over the past 2 days.");
+    }
+    String metaFilePath = metaFilePathOptional.get();
+    String metaFileFolder = metaFilePath.substring(0, metaFilePath.indexOf('/'));
+    Instant exportStartTime = Instant.parse(metaFileFolder.replace('_', '.') + 'Z');
+    BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath));
+    Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+    return DatastoreSnapshotInfo.create(
+        String.format("gs://%s/%s", bucketName, metaFileFolder),
+        getCommitLogDir(),
+        new Interval(exportStartTime, exportEndTime));
+  }
+
+  public String getCommitLogDir() {
+    return "gs://" + projectId + "-commits";
+  }
+
+  /**
+   * Finds the bucket-relative path of the overall export metadata file, in the given bucket,
+   * searching back up to {@code lookBackDays} days, including today.
+   *
+   * <p>The overall export metadata file is the last file created during a Datastore export. All
+   * data has been exported by the creation time of this file. The name of this file, like that of
+   * all files in the same export, begins with the timestamp when the export starts.
+   *
+   * <p>An example return value: {@code
+   * 2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata}.
+   */
+  private Optional<String> findMostRecentExportMetadataFile(String bucketName, int lookBackDays) {
+    DateTime today = clock.nowUtc();
+    for (int day = 0; day < lookBackDays; day++) {
+      String dateString = today.minusDays(day).toString("yyyy-MM-dd");
+      try {
+        Optional<String> metaFilePath =
+            gcsUtils.listFolderObjects(bucketName, dateString).stream()
+                .filter(s -> s.endsWith("overall_export_metadata"))
+                .map(s -> dateString + s)
+                .sorted(Comparator.<String>naturalOrder().reversed())
+                .findFirst();
+        if (metaFilePath.isPresent()) {
+          return metaFilePath;
+        }
+      } catch (IOException ioe) {
+        throw new RuntimeException(ioe);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /** Holds information about a Datastore snapshot. */
+  @AutoValue
+  abstract static class DatastoreSnapshotInfo {
+    abstract String exportDir();
+
+    abstract String commitLogDir();
+
+    abstract Interval exportInterval();
+
+    static DatastoreSnapshotInfo create(
+        String exportDir, String commitLogDir, Interval exportOperationInterval) {
+      return new AutoValue_LatestDatastoreSnapshotFinder_DatastoreSnapshotInfo(
+          exportDir, commitLogDir, exportOperationInterval);
+    }
+  }
+
+  @Singleton
+  @Component(
+      modules = {
+        CredentialModule.class,
+        ConfigModule.class,
+        CloudTasksUtilsModule.class,
+        UtilsModule.class
+      })
+  interface LatestDatastoreSnapshotFinderFinderComponent {
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotInfoFinder();
+  }
+}

core/src/main/java/google/registry/beam/comparedb/SqlSnapshots.java

@@ -0,0 +1,384 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static google.registry.beam.comparedb.ValidateSqlUtils.createSqlEntityTupleTag;
+import static google.registry.beam.comparedb.ValidateSqlUtils.getMedianIdForHistoryTable;
+
+import com.google.common.base.Verify;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSetMultimap;
+import com.google.common.collect.Streams;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.bulkquery.BulkQueryEntities;
+import google.registry.model.bulkquery.DomainBaseLite;
+import google.registry.model.bulkquery.DomainHistoryHost;
+import google.registry.model.bulkquery.DomainHistoryLite;
+import google.registry.model.bulkquery.DomainHost;
+import google.registry.model.common.Cursor;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.GracePeriod.GracePeriodHistory;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.domain.secdns.DomainDsDataHistory;
+import google.registry.model.domain.token.AllocationToken;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.DomainTransactionRecord;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.SerializableFunction;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TypeDescriptor;
+import org.apache.beam.sdk.values.TypeDescriptors;
+
+/**
+ * Utilities for loading SQL snapshots.
+ *
+ * <p>For {@link DomainBase} and {@link DomainHistory}, this class assumes the presence of the
+ * {@link google.registry.persistence.PersistenceModule.JpaTransactionManagerType#BULK_QUERY
+ * bulk-query-capable JpaTransactionManager}, and takes advantage of it for higher throughput.
+ *
+ * <p>For now this class is meant for use during the database migration period only. Therefore, it
+ * contains optimizations specifically for the production database at the current size, e.g.,
+ * parallel queries for select tables.
+ */
+public final class SqlSnapshots {
+
+  private SqlSnapshots() {}
+
+  /**
+   * SQL entity types that are eligible for validation. This set must be consistent with {@link
+   * DatastoreSnapshots#ALL_DATASTORE_KINDS}.
+   */
+  static final ImmutableSet<Class<? extends SqlEntity>> ALL_SQL_ENTITIES =
+      ImmutableSet.of(
+          Registry.class,
+          Cursor.class,
+          Registrar.class,
+          ContactResource.class,
+          RegistrarContact.class,
+          HostResource.class,
+          AllocationToken.class,
+          BillingEvent.Recurring.class,
+          BillingEvent.OneTime.class,
+          BillingEvent.Cancellation.class,
+          PollMessage.class,
+          DomainBase.class,
+          ContactHistory.class,
+          HostHistory.class,
+          DomainHistory.class);
+
+  /**
+   * Loads a SQL snapshot for the given {@code sqlEntityTypes}.
+   *
+   * <p>If {@code snapshotId} is present, all queries use the specified database snapshot,
+   * guaranteeing a consistent result.
+   */
+  public static PCollectionTuple loadCloudSqlSnapshotByType(
+      Pipeline pipeline,
+      ImmutableSet<Class<? extends SqlEntity>> sqlEntityTypes,
+      Optional<String> snapshotId) {
+    PCollectionTuple perTypeSnapshots = PCollectionTuple.empty(pipeline);
+    for (Class<? extends SqlEntity> clazz : sqlEntityTypes) {
+      if (clazz == DomainBase.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainBase.class),
+                loadAndAssembleDomainBase(pipeline, snapshotId));
+        continue;
+      }
+      if (clazz == DomainHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(DomainHistory.class),
+                loadAndAssembleDomainHistory(pipeline, snapshotId));
+        continue;
+      }
+      if (clazz == ContactHistory.class) {
+        perTypeSnapshots =
+            perTypeSnapshots.and(
+                createSqlEntityTupleTag(ContactHistory.class),
+                loadContactHistory(pipeline, snapshotId));
+        continue;
+      }
+      perTypeSnapshots =
+          perTypeSnapshots.and(
+              createSqlEntityTupleTag(clazz),
+              pipeline.apply(
+                  "SQL Load " + clazz.getSimpleName(),
+                  RegistryJpaIO.read(
+                          () -> CriteriaQueryBuilder.create(clazz).build(), SqlEntity.class::cast)
+                      .withSnapshot(snapshotId.orElse(null))));
+    }
+    return perTypeSnapshots;
+  }
+
+  /**
+   * Bulk-loads parts of {@link DomainBase} and assembles them in the pipeline.
+   *
+   * @see BulkQueryEntities
+   */
+  public static PCollection<SqlEntity> loadAndAssembleDomainBase(
+      Pipeline pipeline, Optional<String> snapshotId) {
+    PCollection<KV<String, Serializable>> baseObjects =
+        readAllAndAssignKey(pipeline, DomainBaseLite.class, DomainBaseLite::getRepoId, snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(pipeline, GracePeriod.class, GracePeriod::getDomainRepoId, snapshotId);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DelegationSignerData.class,
+            DelegationSignerData::getDomainRepoId,
+            snapshotId);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(pipeline, DomainHost.class, DomainHost::getDomainRepoId, snapshotId);
+
+    return PCollectionList.of(
+            ImmutableList.of(baseObjects, gracePeriods, delegationSigners, domainHosts))
+        .apply("SQL Merge DomainBase parts", Flatten.pCollections())
+        .apply("Group by Domain Parts by RepoId", GroupByKey.create())
+        .apply(
+            "Assemble DomainBase",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainBaseLite> baseObjects =
+                        partsByType.getAllOf(DomainBaseLite.class);
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainBaseLite object per repoId: " + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainBase(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(GracePeriod.class),
+                            partsByType.getAllOf(DelegationSignerData.class),
+                            partsByType.getAllOf(DomainHost.class).stream()
+                                .map(DomainHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet())));
+                  }
+                }));
+  }
+
+  /**
+   * Loads all {@link ContactHistory} entities from the database.
+   *
+   * <p>This method uses two queries to load data in parallel. This is a performance optimization
+   * specifically for the production database.
+   */
+  static PCollection<SqlEntity> loadContactHistory(Pipeline pipeline, Optional<String> snapshotId) {
+    long medianId =
+        getMedianIdForHistoryTable("ContactHistory")
+            .orElseThrow(
+                () -> new IllegalStateException("Not a valid database: no ContactHistory."));
+    PCollection<SqlEntity> part1 =
+        pipeline.apply(
+            "SQL Load ContactHistory first half",
+            RegistryJpaIO.read(
+                    String.format("select c from ContactHistory c where id <= %s", medianId),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    PCollection<SqlEntity> part2 =
+        pipeline.apply(
+            "SQL Load ContactHistory second half",
+            RegistryJpaIO.read(
+                    String.format("select c from ContactHistory c where id > %s", medianId),
+                    false,
+                    SqlEntity.class::cast)
+                .withSnapshot(snapshotId.orElse(null)));
+    return PCollectionList.of(part1)
+        .and(part2)
+        .apply("Combine ContactHistory parts", Flatten.pCollections());
+  }
+
+  /**
+   * Bulk-loads all parts of {@link DomainHistory} and assembles them in the pipeline.
+   *
+   * <p>This method uses two queries to load {@link DomainBaseLite} in parallel. This is a
+   * performance optimization specifically for the production database.
+   *
+   * @see BulkQueryEntities
+   */
+  static PCollection<SqlEntity> loadAndAssembleDomainHistory(
+      Pipeline pipeline, Optional<String> snapshotId) {
+    long medianId =
+        getMedianIdForHistoryTable("DomainHistory")
+            .orElseThrow(
+                () -> new IllegalStateException("Not a valid database: no DomainHistory."));
+    PCollection<KV<String, Serializable>> baseObjectsPart1 =
+        queryAndAssignKey(
+            pipeline,
+            "first half",
+            String.format("select c from DomainHistory c where id <= %s", medianId),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> baseObjectsPart2 =
+        queryAndAssignKey(
+            pipeline,
+            "second half",
+            String.format("select c from DomainHistory c where id > %s", medianId),
+            DomainHistoryLite.class,
+            compose(DomainHistoryLite::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> gracePeriods =
+        readAllAndAssignKey(
+            pipeline,
+            GracePeriodHistory.class,
+            compose(GracePeriodHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> delegationSigners =
+        readAllAndAssignKey(
+            pipeline,
+            DomainDsDataHistory.class,
+            compose(DomainDsDataHistory::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> domainHosts =
+        readAllAndAssignKey(
+            pipeline,
+            DomainHistoryHost.class,
+            compose(DomainHistoryHost::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+    PCollection<KV<String, Serializable>> transactionRecords =
+        readAllAndAssignKey(
+            pipeline,
+            DomainTransactionRecord.class,
+            compose(DomainTransactionRecord::getDomainHistoryId, DomainHistoryId::toString),
+            snapshotId);
+
+    return PCollectionList.of(
+            ImmutableList.of(
+                baseObjectsPart1,
+                baseObjectsPart2,
+                gracePeriods,
+                delegationSigners,
+                domainHosts,
+                transactionRecords))
+        .apply("Merge DomainHistory parts", Flatten.pCollections())
+        .apply("Group by DomainHistory Parts by DomainHistoryId string", GroupByKey.create())
+        .apply(
+            "Assemble DomainHistory",
+            ParDo.of(
+                new DoFn<KV<String, Iterable<Serializable>>, SqlEntity>() {
+                  @ProcessElement
+                  public void processElement(
+                      @Element KV<String, Iterable<Serializable>> kv,
+                      OutputReceiver<SqlEntity> outputReceiver) {
+                    TypedClassifier partsByType = new TypedClassifier(kv.getValue());
+                    ImmutableSet<DomainHistoryLite> baseObjects =
+                        partsByType.getAllOf(DomainHistoryLite.class);
+                    Verify.verify(
+                        baseObjects.size() == 1,
+                        "Expecting one DomainHistoryLite object per domainHistoryId: "
+                            + kv.getKey());
+                    outputReceiver.output(
+                        BulkQueryEntities.assembleDomainHistory(
+                            baseObjects.iterator().next(),
+                            partsByType.getAllOf(DomainDsDataHistory.class),
+                            partsByType.getAllOf(DomainHistoryHost.class).stream()
+                                .map(DomainHistoryHost::getHostVKey)
+                                .collect(ImmutableSet.toImmutableSet()),
+                            partsByType.getAllOf(GracePeriodHistory.class),
+                            partsByType.getAllOf(DomainTransactionRecord.class)));
+                  }
+                }));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> readAllAndAssignKey(
+      Pipeline pipeline,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName(),
+            RegistryJpaIO.read(() -> CriteriaQueryBuilder.create(type).build())
+                .withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName(),
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  static <R, T> PCollection<KV<String, Serializable>> queryAndAssignKey(
+      Pipeline pipeline,
+      String diffrentiator,
+      String jplQuery,
+      Class<R> type,
+      SerializableFunction<R, String> keyFunction,
+      Optional<String> snapshotId) {
+    return pipeline
+        .apply(
+            "SQL Load " + type.getSimpleName() + " " + diffrentiator,
+            RegistryJpaIO.read(jplQuery, false, type::cast).withSnapshot(snapshotId.orElse(null)))
+        .apply(
+            "Assign Key to " + type.getSimpleName() + " " + diffrentiator,
+            MapElements.into(
+                    TypeDescriptors.kvs(
+                        TypeDescriptors.strings(), TypeDescriptor.of(Serializable.class)))
+                .via(obj -> KV.of(keyFunction.apply(obj), (Serializable) obj)));
+  }
+
+  // TODO(b/205988530): don't use beam serializablefunction, make one that extends Java's Function.
+  private static <R, I, T> SerializableFunction<R, T> compose(
+      SerializableFunction<R, I> f1, SerializableFunction<I, T> f2) {
+    return r -> f2.apply(f1.apply(r));
+  }
+
+  /** Container that receives mixed-typed data and groups them by {@link Class}. */
+  static class TypedClassifier {
+    private final ImmutableSetMultimap<Class<?>, Object> classifiedEntities;
+
+    TypedClassifier(Iterable<Serializable> inputs) {
+      this.classifiedEntities =
+          Streams.stream(inputs)
+              .collect(ImmutableSetMultimap.toImmutableSetMultimap(Object::getClass, x -> x));
+    }
+
+    <T> ImmutableSet<T> getAllOf(Class<T> clazz) {
+      return classifiedEntities.get(clazz).stream()
+          .map(clazz::cast)
+          .collect(ImmutableSet.toImmutableSet());
+    }
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -0,0 +1,159 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static org.apache.beam.sdk.values.TypeDescriptors.strings;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import google.registry.beam.common.DatabaseSnapshot;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.replay.SqlReplayCheckpoint;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import java.io.Serializable;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.PipelineResult.State;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.Flatten;
+import org.apache.beam.sdk.transforms.GroupByKey;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.WithKeys;
+import org.apache.beam.sdk.values.PCollectionList;
+import org.apache.beam.sdk.values.PCollectionTuple;
+import org.apache.beam.sdk.values.TupleTag;
+import org.joda.time.DateTime;
+
+/**
+ * Validates the asynchronous data replication process from Datastore (primary storage) to Cloud SQL
+ * (secondary storage).
+ */
+public class ValidateSqlPipeline {
+
+  /** Specifies the extra CommitLogs to load before the start of a Database export. */
+  private static final int COMMIT_LOG_MARGIN_MINUTES = 10;
+
+  private final ValidateSqlPipelineOptions options;
+  private final DatastoreSnapshotInfo mostRecentExport;
+
+  public ValidateSqlPipeline(
+      ValidateSqlPipelineOptions options, DatastoreSnapshotInfo mostRecentExport) {
+    this.options = options;
+    this.mostRecentExport = mostRecentExport;
+  }
+
+  void run() {
+    run(Pipeline.create(options));
+  }
+
+  @VisibleForTesting
+  void run(Pipeline pipeline) {
+    // TODO(weiminyu): Acquire the commit log replay lock when the lock release bug is fixed.
+    DateTime latestCommitLogTime =
+        TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+    Preconditions.checkState(
+        latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
+        "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
+    try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
+      setupPipeline(pipeline, Optional.of(databaseSnapshot.getSnapshotId()), latestCommitLogTime);
+      State state = pipeline.run().waitUntilFinish();
+      if (!State.DONE.equals(state)) {
+        throw new IllegalStateException("Unexpected pipeline state: " + state);
+      }
+    }
+  }
+
+  void setupPipeline(
+      Pipeline pipeline, Optional<String> sqlSnapshotId, DateTime latestCommitLogTime) {
+    pipeline
+        .getCoderRegistry()
+        .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
+
+    PCollectionTuple datastoreSnapshot =
+        DatastoreSnapshots.loadDatastoreSnapshotByKind(
+            pipeline,
+            mostRecentExport.exportDir(),
+            mostRecentExport.commitLogDir(),
+            mostRecentExport.exportInterval().getStart().minusMinutes(COMMIT_LOG_MARGIN_MINUTES),
+            // Increase by 1ms since we want to include commitLogs latestCommitLogTime but
+            // this parameter is exclusive.
+            latestCommitLogTime.plusMillis(1),
+            DatastoreSnapshots.ALL_DATASTORE_KINDS);
+
+    PCollectionTuple cloudSqlSnapshot =
+        SqlSnapshots.loadCloudSqlSnapshotByType(
+            pipeline, SqlSnapshots.ALL_SQL_ENTITIES, sqlSnapshotId);
+
+    verify(
+        datastoreSnapshot.getAll().keySet().equals(cloudSqlSnapshot.getAll().keySet()),
+        "Expecting the same set of types in both snapshots.");
+
+    for (Class<? extends SqlEntity> clazz : SqlSnapshots.ALL_SQL_ENTITIES) {
+      TupleTag<SqlEntity> tag = ValidateSqlUtils.createSqlEntityTupleTag(clazz);
+      verify(
+          datastoreSnapshot.has(tag), "Missing %s in Datastore snapshot.", clazz.getSimpleName());
+      verify(cloudSqlSnapshot.has(tag), "Missing %s in Cloud SQL snapshot.", clazz.getSimpleName());
+      PCollectionList.of(datastoreSnapshot.get(tag))
+          .and(cloudSqlSnapshot.get(tag))
+          .apply("Combine from both snapshots: " + clazz.getSimpleName(), Flatten.pCollections())
+          .apply(
+              "Assign primary key to merged " + clazz.getSimpleName(),
+              WithKeys.of(ValidateSqlPipeline::getPrimaryKeyString).withKeyType(strings()))
+          .apply("Group by primary key " + clazz.getSimpleName(), GroupByKey.create())
+          .apply("Compare " + clazz.getSimpleName(), ParDo.of(new CompareSqlEntity()));
+    }
+  }
+
+  private static String getPrimaryKeyString(SqlEntity sqlEntity) {
+    // SqlEntity.getPrimaryKeyString only works with entities registered with Hibernate.
+    // We are using the BulkQueryJpaTransactionManager, which does not recognize DomainBase and
+    // DomainHistory. See BulkQueryEntities.java for more information.
+    if (sqlEntity instanceof DomainBase) {
+      return "DomainBase_" + ((DomainBase) sqlEntity).getRepoId();
+    }
+    if (sqlEntity instanceof DomainHistory) {
+      return "DomainHistory_" + ((DomainHistory) sqlEntity).getDomainHistoryId().toString();
+    }
+    return sqlEntity.getPrimaryKeyString();
+  }
+
+  public static void main(String[] args) {
+    ValidateSqlPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(ValidateSqlPipelineOptions.class);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    DatastoreSnapshotInfo mostRecentExport =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder()
+            .getSnapshotInfo();
+
+    new ValidateSqlPipeline(options, mostRecentExport).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipelineOptions.java

@@ -0,0 +1,20 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+
+/** BEAM pipeline options for {@link ValidateSqlPipeline}. */
+public interface ValidateSqlPipelineOptions extends RegistryPipelineOptions {}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -0,0 +1,270 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.contact.ContactBase;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.domain.DomainContent;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.eppcommon.AuthInfo;
+import google.registry.model.host.HostHistory;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.replay.SqlEntity;
+import google.registry.model.reporting.HistoryEntry;
+import java.lang.reflect.Field;
+import java.math.BigInteger;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import org.apache.beam.sdk.metrics.Counter;
+import org.apache.beam.sdk.metrics.Metrics;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.KV;
+import org.apache.beam.sdk.values.TupleTag;
+
+/** Helpers for use by {@link ValidateSqlPipeline}. */
+final class ValidateSqlUtils {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private ValidateSqlUtils() {}
+
+  /**
+   * Query template for finding the median value of the {@code history_revision_id} column in one of
+   * the History tables.
+   *
+   * <p>The {@link ValidateSqlPipeline} uses this query to parallelize the query to some of the
+   * history tables. Although the {@code repo_id} column is the leading column in the primary keys
+   * of these tables, in practice and with production data, division by {@code history_revision_id}
+   * works slightly faster for unknown reasons.
+   */
+  private static final String MEDIAN_ID_QUERY_TEMPLATE =
+      "SELECT history_revision_id FROM (                                                        "
+          + "  SELECT"
+          + "    ROW_NUMBER() OVER (ORDER BY history_revision_id ASC) AS rownumber,"
+          + "    history_revision_id"
+          + "  FROM \"%TABLE%\""
+          + ") AS foo\n"
+          + "WHERE rownumber in (select count(*) / 2 + 1 from \"%TABLE%\")";
+
+  static Optional<Long> getMedianIdForHistoryTable(String tableName) {
+    Preconditions.checkArgument(
+        tableName.endsWith("History"), "Table must be one of the History tables.");
+    String sqlText = MEDIAN_ID_QUERY_TEMPLATE.replace("%TABLE%", tableName);
+    List results =
+        jpaTm()
+            .transact(() -> jpaTm().getEntityManager().createNativeQuery(sqlText).getResultList());
+    verify(results.size() < 2, "MidPoint query should have at most one result.");
+    if (results.isEmpty()) {
+      return Optional.empty();
+    }
+    return Optional.of(((BigInteger) results.get(0)).longValue());
+  }
+
+  static TupleTag<SqlEntity> createSqlEntityTupleTag(Class<? extends SqlEntity> actualType) {
+    return new TupleTag<SqlEntity>(actualType.getSimpleName()) {};
+  }
+
+  static class CompareSqlEntity extends DoFn<KV<String, Iterable<SqlEntity>>, Void> {
+    private final HashMap<String, Counter> totalCounters = new HashMap<>();
+    private final HashMap<String, Counter> missingCounters = new HashMap<>();
+    private final HashMap<String, Counter> unequalCounters = new HashMap<>();
+    private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
+
+    private volatile boolean logPrinted = false;
+
+    private String getCounterKey(Class<?> clazz) {
+      return PollMessage.class.isAssignableFrom(clazz) ? "PollMessage" : clazz.getSimpleName();
+    }
+
+    private synchronized void ensureCounterExists(String counterKey) {
+      if (totalCounters.containsKey(counterKey)) {
+        return;
+      }
+      totalCounters.put(counterKey, Metrics.counter("CompareDB", "Total Compared: " + counterKey));
+      missingCounters.put(
+          counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
+      unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
+      badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
+    }
+
+    /**
+     * A rudimentary debugging helper that prints the first pair of unequal entities in each worker.
+     * This will be removed when we start exporting such entities to GCS.
+     */
+    void logDiff(String key, Object entry0, Object entry1) {
+      if (logPrinted) {
+        return;
+      }
+      logPrinted = true;
+      Map<String, Object> fields0 = ((ImmutableObject) entry0).toDiffableFieldMap();
+      Map<String, Object> fields1 = ((ImmutableObject) entry1).toDiffableFieldMap();
+      StringBuilder sb = new StringBuilder();
+      fields0.forEach(
+          (field, value) -> {
+            if (fields1.containsKey(field)) {
+              if (!Objects.equals(value, fields1.get(field))) {
+                sb.append(field + " not match: " + value + " -> " + fields1.get(field) + "\n");
+              }
+            } else {
+              sb.append(field + "Not found in entity 2\n");
+            }
+          });
+      fields1.forEach(
+          (field, value) -> {
+            if (!fields0.containsKey(field)) {
+              sb.append(field + "Not found in entity 1\n");
+            }
+          });
+      logger.atWarning().log(key + "  " + sb.toString());
+    }
+
+    @ProcessElement
+    public void processElement(@Element KV<String, Iterable<SqlEntity>> kv) {
+      ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
+
+      verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
+      verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
+
+      String counterKey = getCounterKey(entities.get(0).getClass());
+      ensureCounterExists(counterKey);
+      totalCounters.get(counterKey).inc();
+
+      if (entities.size() == 1) {
+        missingCounters.get(counterKey).inc();
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          logger.atWarning().log("Unexpected single entity: %s", kv.getKey());
+        }
+        return;
+      }
+      SqlEntity entity0;
+      SqlEntity entity1;
+
+      try {
+        entity0 = normalizeEntity(entities.get(0));
+        entity1 = normalizeEntity(entities.get(1));
+      } catch (Exception e) {
+        // Temporary debugging help. See logDiff() above.
+        if (!logPrinted) {
+          logPrinted = true;
+          badEntityCounters.get(counterKey).inc();
+        }
+        return;
+      }
+
+      if (!Objects.equals(entity0, entity1)) {
+        unequalCounters.get(counterKey).inc();
+        logDiff(kv.getKey(), entities.get(0), entities.get(1));
+      }
+    }
+  }
+
+  static SqlEntity normalizeEntity(SqlEntity sqlEntity) {
+    if (sqlEntity instanceof EppResource) {
+      return normalizeEppResource(sqlEntity);
+    }
+    if (sqlEntity instanceof HistoryEntry) {
+      return (SqlEntity) normalizeHistoryEntry((HistoryEntry) sqlEntity);
+    }
+    return sqlEntity;
+  }
+
+  /**
+   * Normalizes an {@link EppResource} instance for comparison.
+   *
+   * <p>This method may modify the input object using reflection instead of making a copy with
+   * {@code eppResource.asBuilder().build()}, because when {@code eppResource} is a {@link
+   * google.registry.model.domain.DomainBase}, the {@code build} method accesses the Database, which
+   * we want to avoid.
+   */
+  static SqlEntity normalizeEppResource(SqlEntity eppResource) {
+    try {
+      Field authField =
+          eppResource instanceof DomainContent
+              ? DomainContent.class.getDeclaredField("authInfo")
+              : eppResource instanceof ContactBase
+                  ? ContactBase.class.getDeclaredField("authInfo")
+                  : null;
+      if (authField != null) {
+        authField.setAccessible(true);
+        AuthInfo authInfo = (AuthInfo) authField.get(eppResource);
+        // When AuthInfo is missing, the authInfo field is null if the object is loaded from
+        // Datastore, or a PasswordAuth with null properties if loaded from SQL. In the second case
+        // we set the authInfo field to null.
+        if (authInfo != null
+            && authInfo.getPw() != null
+            && authInfo.getPw().getRepoId() == null
+            && authInfo.getPw().getValue() == null) {
+          authField.set(eppResource, null);
+        }
+      }
+
+      Field field = EppResource.class.getDeclaredField("revisions");
+      field.setAccessible(true);
+      field.set(eppResource, null);
+      return eppResource;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Normalizes a {@link HistoryEntry} for comparison.
+   *
+   * <p>This method modifies the input using reflection because relevant builder methods performs
+   * unwanted checks and changes.
+   */
+  static HistoryEntry normalizeHistoryEntry(HistoryEntry historyEntry) {
+    // History objects from Datastore do not have details of their EppResource objects
+    // (domainContent, contactBase, hostBase).
+    try {
+      if (historyEntry instanceof DomainHistory) {
+        Field domainContent = DomainHistory.class.getDeclaredField("domainContent");
+        domainContent.setAccessible(true);
+        domainContent.set(historyEntry, null);
+        Field domainTransactionRecords =
+            HistoryEntry.class.getDeclaredField("domainTransactionRecords");
+        domainTransactionRecords.setAccessible(true);
+        Set<?> domainTransactionRecordsValue = (Set<?>) domainTransactionRecords.get(historyEntry);
+        if (domainTransactionRecordsValue != null && domainTransactionRecordsValue.isEmpty()) {
+          domainTransactionRecords.set(historyEntry, null);
+        }
+      } else if (historyEntry instanceof ContactHistory) {
+        Field contactBase = ContactHistory.class.getDeclaredField("contactBase");
+        contactBase.setAccessible(true);
+        contactBase.set(historyEntry, null);
+      } else if (historyEntry instanceof HostHistory) {
+        Field hostBase = HostHistory.class.getDeclaredField("hostBase");
+        hostBase.setAccessible(true);
+        hostBase.set(historyEntry, null);
+      }
+      return historyEntry;
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+}

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -101,8 +101,9 @@ public final class Transforms {
   }
 
   /**
-   * Composite {@link PTransform transform} that loads the Datastore snapshot at {@code
-   * commitLogToTime} for caller specified {@code kinds}.
+   * Composite {@link PTransform transform} that loads the Datastore snapshot right before {@code
+   * commitLogToTime} for caller specified {@code kinds}. The resulting snapshot has all changes
+   * that happened before {@code commitLogToTime}, and none at or after {@code commitLogToTime}.
    *
    * <p>Caller must provide the location of a Datastore export that started AFTER {@code
    * commitLogFromTime} and completed BEFORE {@code commitLogToTime}, as well as the root directory
@@ -363,7 +364,7 @@ public final class Transforms {
    *     to make Optional work with BEAM)
    */
   @Nullable
-  public static Object convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
+  public static SqlEntity convertVersionedEntityToSqlEntity(VersionedEntity dsEntity) {
     return dsEntity
         .getEntity()
         .filter(Transforms::isMigratable)

core/src/main/java/google/registry/cron/TldFanoutAction.java

@@ -30,7 +30,6 @@ import static google.registry.cron.CronModule.RUN_IN_EMPTY_PARAM;
 import static google.registry.model.tld.Registries.getTldsOfType;
 import static google.registry.model.tld.Registry.TldType.REAL;
 import static google.registry.model.tld.Registry.TldType.TEST;
-import static java.util.concurrent.TimeUnit.SECONDS;
 
 import com.google.cloud.tasks.v2.Task;
 import com.google.common.collect.ArrayListMultimap;
@@ -39,7 +38,6 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
-import com.google.protobuf.Timestamp;
 import google.registry.request.Action;
 import google.registry.request.Action.Service;
 import google.registry.request.Parameter;
@@ -49,9 +47,7 @@ import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.util.Clock;
 import google.registry.util.CloudTasksUtils;
-import java.time.Instant;
 import java.util.Optional;
-import java.util.Random;
 import java.util.stream.Stream;
 import javax.inject.Inject;
 
@@ -100,8 +96,6 @@ public final class TldFanoutAction implements Runnable {
           EXCLUDE_PARAM,
           JITTER_SECONDS_PARAM);
 
-  private static final Random random = new Random();
-
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   @Inject Clock clock;
@@ -165,22 +159,7 @@ public final class TldFanoutAction implements Runnable {
       params = ArrayListMultimap.create(params);
       params.put(RequestParameters.PARAM_TLD, tld);
     }
-    Instant scheduleTime =
-        Instant.ofEpochMilli(
-            clock
-                .nowUtc()
-                .plusMillis(
-                    jitterSeconds
-                        .map(seconds -> random.nextInt((int) SECONDS.toMillis(seconds)))
-                        .orElse(0))
-                .getMillis());
-    return Task.newBuilder(
-            CloudTasksUtils.createPostTask(endpoint, Service.BACKEND.toString(), params))
-        .setScheduleTime(
-            Timestamp.newBuilder()
-                .setSeconds(scheduleTime.getEpochSecond())
-                .setNanos(scheduleTime.getNano())
-                .build())
-        .build();
+    return CloudTasksUtils.createPostTask(
+        endpoint, Service.BACKEND.toString(), params, clock, jitterSeconds);
   }
 }

core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml

@@ -208,6 +208,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/readDnsQueue?jitterSeconds=45]]></url>
     <description>

core/src/main/java/google/registry/env/crash/default/WEB-INF/cron.xml

@@ -211,6 +211,15 @@
   </cron>
   -->
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <!--
     The next two wipeout jobs are required when crash has production data.
    -->

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -120,6 +120,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/commitLogCheckpoint]]></url>
     <description>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -247,6 +247,15 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/cron/replicateToDatastore]]></url>
+    <description>
+      Replays recent transactions from SQL to the Datastore secondary backend.
+    </description>
+    <schedule>every 3 minutes</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
     <description>

core/src/main/java/google/registry/flows/EppController.java

@@ -143,9 +143,6 @@ public final class EppController {
   /** Creates a response indicating an EPP failure. */
   @VisibleForTesting
   static EppOutput getErrorResponse(Result result, Trid trid) {
-    return EppOutput.create(new EppResponse.Builder()
-        .setResult(result)
-        .setTrid(trid)
-        .build());
+    return EppOutput.create(new EppResponse.Builder().setResult(result).setTrid(trid).build());
   }
 }

core/src/main/java/google/registry/flows/EppException.java

@@ -25,10 +25,12 @@ import google.registry.model.eppinput.EppInput.InnerCommand;
 import google.registry.model.eppinput.EppInput.ResourceCommandWrapper;
 import google.registry.model.eppoutput.Result;
 import google.registry.model.eppoutput.Result.Code;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import java.lang.annotation.Documented;
 import java.lang.annotation.Inherited;
 import java.lang.annotation.Retention;
 import java.lang.annotation.Target;
+import javax.annotation.Nullable;
 
 /** Exception used to propagate all failures containing one or more EPP responses. */
 public abstract class EppException extends Exception {
@@ -37,7 +39,12 @@ public abstract class EppException extends Exception {
 
   /** Create an EppException with a custom message. */
   private EppException(String message) {
-    super(message);
+    this(message, null);
+  }
+
+  /** Create an EppException with a custom message and cause. */
+  private EppException(String message, @Nullable Throwable cause) {
+    super(message, cause);
     Code code = getClass().getAnnotation(EppResultCode.class).value();
     Preconditions.checkState(!code.isSuccess());
     this.result = Result.create(code, message);
@@ -255,4 +262,12 @@ public abstract class EppException extends Exception {
       super("Specified protocol version is not implemented");
     }
   }
+
+  /** Registry is currently undergoing maintenance and is in read-only mode. */
+  @EppResultCode(Code.COMMAND_FAILED)
+  public static class ReadOnlyModeEppException extends EppException {
+    ReadOnlyModeEppException(ReadOnlyModeException cause) {
+      super("Registry is currently undergoing maintenance and is in read-only mode", cause);
+    }
+  }
 }

core/src/main/java/google/registry/flows/FlowRunner.java

@@ -19,6 +19,7 @@ import static google.registry.xml.XmlTransformer.prettyPrint;
 
 import com.google.common.base.Strings;
 import com.google.common.flogger.FluentLogger;
+import google.registry.flows.EppException.ReadOnlyModeEppException;
 import google.registry.flows.FlowModule.DryRun;
 import google.registry.flows.FlowModule.InputXml;
 import google.registry.flows.FlowModule.RegistrarId;
@@ -28,6 +29,7 @@ import google.registry.flows.session.LoginFlow;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.eppoutput.EppOutput;
 import google.registry.monitoring.whitebox.EppMetric;
+import google.registry.persistence.transaction.TransactionManagerFactory.ReadOnlyModeException;
 import javax.inject.Inject;
 import javax.inject.Provider;
 
@@ -97,6 +99,8 @@ public class FlowRunner {
       return e.output;
     } catch (EppRuntimeException e) {
       throw e.getCause();
+    } catch (ReadOnlyModeException e) {
+      throw new ReadOnlyModeEppException(e);
     }
   }
 

core/src/main/java/google/registry/flows/contact/ContactCheckFlow.java

@@ -42,6 +42,7 @@ import javax.inject.Inject;
  * <p>This flows can check the existence of multiple contacts simultaneously.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  */
 @ReportingSpec(ActivityReportField.CONTACT_CHECK)
 public final class ContactCheckFlow implements Flow {

core/src/main/java/google/registry/flows/contact/ContactCreateFlow.java

@@ -50,6 +50,8 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that creates a new contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link ContactFlowUtils.BadInternationalizedPostalInfoException}

core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java

@@ -60,6 +60,8 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/contact/ContactInfoFlow.java

@@ -46,6 +46,7 @@ import org.joda.time.DateTime;
  * ever been transferred. Any registrar can see any contact's information, but the authInfo is only
  * visible to the registrar that owns the contact or to a registrar that already supplied it.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  */

core/src/main/java/google/registry/flows/contact/ContactTransferApproveFlow.java

@@ -54,6 +54,8 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * explicitly approve the transfer request, which then becomes effective immediately.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}

core/src/main/java/google/registry/flows/contact/ContactTransferCancelFlow.java

@@ -54,6 +54,8 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the gaining client to
  * withdraw the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactTransferQueryFlow.java

@@ -40,11 +40,12 @@ import javax.inject.Inject;
  *
  * <p>The "gaining" registrar requests a transfer from the "losing" (aka current) registrar. The
  * losing registrar has a "transfer" time period to respond (by default five days) after which the
- * transfer is automatically approved. This flow can be used by the gaining or losing registrars
- * (or anyone with the correct authId) to see the status of a transfer, which may still be pending
- * or may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
+ * transfer is automatically approved. This flow can be used by the gaining or losing registrars (or
+ * anyone with the correct authId) to see the status of a transfer, which may still be pending or
+ * may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
  * period expiring.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NoTransferHistoryToQueryException}

core/src/main/java/google/registry/flows/contact/ContactTransferRejectFlow.java

@@ -53,6 +53,8 @@ import org.joda.time.DateTime;
  * transfer is automatically approved. Within that window, this flow allows the losing client to
  * reject the transfer request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/contact/ContactTransferRequestFlow.java

@@ -63,6 +63,8 @@ import org.joda.time.Duration;
  * by the losing registrar or rejected, and the gaining registrar can also cancel the transfer
  * request.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.AlreadyPendingTransferException}

core/src/main/java/google/registry/flows/contact/ContactUpdateFlow.java

@@ -55,6 +55,8 @@ import org.joda.time.DateTime;
 /**
  * An EPP flow that updates a contact.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainCheckFlow.java

@@ -85,6 +85,7 @@ import org.joda.time.DateTime;
  * <p>This flow also supports the EPP fee extension and can return pricing information.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link DomainFlowUtils.BadDomainNameCharacterException}
  * @error {@link DomainFlowUtils.BadDomainNamePartsCountException}

core/src/main/java/google/registry/flows/domain/DomainClaimsCheckFlow.java

@@ -57,13 +57,14 @@ import org.joda.time.DateTime;
  * An EPP flow that checks whether domain labels are trademarked.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link DomainFlowUtils.BadCommandForRegistryPhaseException}
  * @error {@link DomainFlowUtils.ClaimsPeriodEndedException}
  * @error {@link DomainFlowUtils.NotAuthorizedForTldException}
  * @error {@link DomainFlowUtils.TldDoesNotExistException}
  * @error {@link DomainClaimsCheckNotAllowedWithAllocationTokens}
  */
-@ReportingSpec(ActivityReportField.DOMAIN_CHECK)  // Claims check is a special domain check.
+@ReportingSpec(ActivityReportField.DOMAIN_CHECK) // Claims check is a special domain check.
 public final class DomainClaimsCheckFlow implements Flow {
 
   @Inject ExtensionManager extensionManager;

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -133,11 +133,13 @@ import org.joda.time.Duration;
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.AlreadyRedeemedAllocationTokenException}
  * @error {@link
  *     google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException}
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
  * @error {@link google.registry.flows.ExtensionManager.UndeclaredServiceExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link DomainCreateFlow.AnchorTenantCreatePeriodException}
  * @error {@link DomainCreateFlow.MustHaveSignedMarksInCurrentPhaseException}

core/src/main/java/google/registry/flows/domain/DomainDeleteFlow.java

@@ -103,7 +103,9 @@ import org.joda.time.Duration;
 /**
  * An EPP flow that deletes a domain.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.OnlyToolCanPassMetadataException}
@@ -187,7 +189,7 @@ public final class DomainDeleteFlow implements TransactionalFlow {
     } else {
       DateTime redemptionTime = now.plus(redemptionGracePeriodLength);
       asyncTaskEnqueuer.enqueueAsyncResave(
-          existingDomain, now, ImmutableSortedSet.of(redemptionTime, deletionTime));
+          existingDomain.createVKey(), now, ImmutableSortedSet.of(redemptionTime, deletionTime));
       builder
           .setDeletionTime(deletionTime)
           .setStatusValues(ImmutableSet.of(StatusValue.PENDING_DELETE))

core/src/main/java/google/registry/flows/domain/DomainInfoFlow.java

@@ -63,6 +63,7 @@ import org.joda.time.DateTime;
  * domain, will get a rich result with all of the domain's fields. All other requests will be
  * answered with a minimal result containing only basic information about the domain.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -94,6 +94,8 @@ import org.joda.time.Duration;
  * longer than 10 years unless it comes in at the exact millisecond that the domain would have
  * expired.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -93,7 +93,9 @@ import org.joda.time.DateTime;
  * restored to a single year expiration starting at the restore time, regardless of what the
  * original expiration time was.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
@@ -110,7 +112,7 @@ import org.joda.time.DateTime;
  * @error {@link DomainRestoreRequestFlow.RestoreCommandIncludesChangesException}
  */
 @ReportingSpec(ActivityReportField.DOMAIN_RGP_RESTORE_REQUEST)
-public final class DomainRestoreRequestFlow implements TransactionalFlow  {
+public final class DomainRestoreRequestFlow implements TransactionalFlow {
 
   @Inject ResourceCommand resourceCommand;
   @Inject ExtensionManager extensionManager;

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -78,6 +78,8 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted and replaced with new ones with the correct approval time.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -65,6 +65,8 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NotPendingTransferException}

core/src/main/java/google/registry/flows/domain/DomainTransferQueryFlow.java

@@ -44,11 +44,12 @@ import org.joda.time.DateTime;
  *
  * <p>The "gaining" registrar requests a transfer from the "losing" (aka current) registrar. The
  * losing registrar has a "transfer" time period to respond (by default five days) after which the
- * transfer is automatically approved. This flow can be used by the gaining or losing registrars
- * (or anyone with the correct authId) to see the status of a transfer, which may still be pending
- * or may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
+ * transfer is automatically approved. This flow can be used by the gaining or losing registrars (or
+ * anyone with the correct authId) to see the status of a transfer, which may still be pending or
+ * may have been approved, rejected, cancelled or implicitly approved by virtue of the transfer
  * period expiring.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.exceptions.NoTransferHistoryToQueryException}

core/src/main/java/google/registry/flows/domain/DomainTransferRejectFlow.java

@@ -67,6 +67,8 @@ import org.joda.time.DateTime;
  * timestamps such that they only would become active when the transfer period passed. In this flow,
  * those speculative objects are deleted.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -93,6 +93,8 @@ import org.joda.time.DateTime;
  * rejection or cancellation of the request, they will be deleted (and in the approval case,
  * replaced with new ones with the correct approval time).
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.FlowUtils.UnknownCurrencyEppException}
  * @error {@link google.registry.flows.ResourceFlowUtils.BadAuthInfoForResourceException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
@@ -102,7 +104,8 @@ import org.joda.time.DateTime;
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}
  * @error {@link google.registry.flows.exceptions.TransferPeriodMustBeOneYearException}
  * @error {@link InvalidTransferPeriodValueException}
- * @error {@link google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
+ * @error {@link
+ *     google.registry.flows.exceptions.TransferPeriodZeroAndFeeTransferExtensionException}
  * @error {@link DomainFlowUtils.BadPeriodUnitException}
  * @error {@link DomainFlowUtils.CurrencyUnitMismatchException}
  * @error {@link DomainFlowUtils.CurrencyValueScaleException}
@@ -235,7 +238,7 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
             .build();
     DomainHistory domainHistory = buildDomainHistory(newDomain, registry, now, period);
 
-    asyncTaskEnqueuer.enqueueAsyncResave(newDomain, now, automaticTransferTime);
+    asyncTaskEnqueuer.enqueueAsyncResave(newDomain.createVKey(), now, automaticTransferTime);
     tm().putAll(
             new ImmutableSet.Builder<>()
                 .add(newDomain, domainHistory, requestPollMessage)

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -97,7 +97,9 @@ import org.joda.time.DateTime;
  * superuser. As such, adding or removing these statuses incurs a billing event. There will be only
  * one charge per update, even if several such statuses are updated at once.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.EppException.UnimplementedExtensionException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/flows/host/HostCheckFlow.java

@@ -42,6 +42,7 @@ import javax.inject.Inject;
  * <p>This flows can check the existence of multiple hosts simultaneously.
  *
  * @error {@link google.registry.flows.exceptions.TooManyResourceChecksException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  */
 @ReportingSpec(ActivityReportField.HOST_CHECK)
 public final class HostCheckFlow implements Flow {

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -65,7 +65,9 @@ import org.joda.time.DateTime;
  * hosts cannot have any. This flow allows creating a host name, and if necessary enqueues tasks to
  * update DNS.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
  * @error {@link google.registry.flows.FlowUtils.IpAddressVersionMismatchException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link ResourceAlreadyExistsForThisClientException}
  * @error {@link ResourceCreateContentionException}
  * @error {@link HostFlowUtils.HostNameTooLongException}

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -58,6 +58,8 @@ import org.joda.time.DateTime;
  * references to the host before the deletion is allowed to proceed. A poll message will be written
  * with the success or failure message when the process is complete.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}
  * @error {@link google.registry.flows.exceptions.ResourceStatusProhibitsOperationException}

core/src/main/java/google/registry/flows/host/HostInfoFlow.java

@@ -44,6 +44,7 @@ import org.joda.time.DateTime;
  * <p>The returned information included IP addresses, if any, and details of the host's most recent
  * transfer if it has ever been transferred. Any registrar can see the information for any host.
  *
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link HostFlowUtils.HostNameNotLowerCaseException}
  * @error {@link HostFlowUtils.HostNameNotNormalizedException}

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -73,11 +73,13 @@ import org.joda.time.DateTime;
  * hosts. Internal hosts must have at least one IP address associated with them, whereas external
  * hosts cannot have any.
  *
- * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When
- * a host is renamed from internal to external all IP addresses must be simultaneously removed, and
+ * <p>This flow allows changing a host name, and adding or removing IP addresses to hosts. When a
+ * host is renamed from internal to external all IP addresses must be simultaneously removed, and
  * when it is renamed from external to internal at least one must be added. If the host is renamed
  * or IP addresses are added, tasks are enqueued to update DNS accordingly.
  *
+ * @error {@link google.registry.flows.EppException.ReadOnlyModeEppException}
+ * @error {@link google.registry.flows.FlowUtils.NotLoggedInException}
  * @error {@link google.registry.flows.ResourceFlowUtils.AddRemoveSameValueException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException}
  * @error {@link google.registry.flows.ResourceFlowUtils.ResourceNotOwnedException}

core/src/main/java/google/registry/model/EppResource.java

@@ -228,10 +228,8 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
   /** Used when replaying from SQL to DS to populate the Datastore indexes. */
   protected void saveIndexesToDatastore() {
-    ofyTm()
-        .putAll(
-            ForeignKeyIndex.create(this, getDeletionTime()),
-            EppResourceIndex.create(Key.create(this)));
+    ofyTm().putIgnoringReadOnlyWithBackup(ForeignKeyIndex.create(this, getDeletionTime()));
+    ofyTm().putIgnoringReadOnlyWithBackup(EppResourceIndex.create(Key.create(this)));
   }
 
   /** EppResources that are loaded via foreign keys should implement this marker interface. */

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -156,6 +156,10 @@ public abstract class ImmutableObject implements Cloneable {
    *   }
    * }
    * </pre>
+   *
+   * <p>This method makes use of {@link #toStringHelper}, which embeds {@link
+   * System#identityHashCode} in the output string. Subclasses that require deterministic string
+   * representations <em>across</em> JVM instances should override this method.
    */
   @Override
   public String toString() {

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -53,6 +53,7 @@ import google.registry.persistence.BillingVKey.BillingEventVKey;
 import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
+import google.registry.persistence.converter.JodaMoneyType;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -66,6 +67,8 @@ import javax.persistence.Enumerated;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.PostLoad;
 import javax.persistence.Transient;
+import org.hibernate.annotations.Columns;
+import org.hibernate.annotations.Type;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 
@@ -312,10 +315,8 @@ public abstract class BillingEvent extends ImmutableObject
   public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
-    @AttributeOverrides({
-      @AttributeOverride(name = "money.amount", column = @Column(name = "cost_amount")),
-      @AttributeOverride(name = "money.currency", column = @Column(name = "cost_currency"))
-    })
+    @Type(type = JodaMoneyType.TYPE_NAME)
+    @Columns(columns = {@Column(name = "cost_amount"), @Column(name = "cost_currency")})
     Money cost;
 
     /** When the cost should be billed. */

core/src/main/java/google/registry/model/bulkquery/DomainHistoryLite.java

@@ -107,6 +107,10 @@ public class DomainHistoryLite extends HistoryEntry implements SqlOnlyEntity {
     return VKey.create(DomainBase.class, getDomainRepoId());
   }
 
+  public DomainHistoryId getDomainHistoryId() {
+    return new DomainHistoryId(getDomainRepoId(), getId());
+  }
+
   @PostLoad
   void postLoad() {
     if (domainContent == null) {

core/src/main/java/google/registry/model/common/ClassPathManager.java

@@ -0,0 +1,64 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.common;
+
+import static com.google.common.base.Functions.identity;
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.model.EntityClasses.ALL_CLASSES;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.googlecode.objectify.annotation.EntitySubclass;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/** A helper to manage class name and class path mapping. */
+public class ClassPathManager {
+  /**
+   * Class registry allowing us to restore the original class object from the unqualified class
+   * name, which is all the datastore key gives us. Note that entities annotated
+   * with @EntitySubclass are removed because they share the same kind of the key with their parent
+   * class.
+   */
+  public static final Map<String, Class<?>> CLASS_REGISTRY =
+      ALL_CLASSES.stream()
+          .filter(clazz -> !clazz.isAnnotationPresent(EntitySubclass.class))
+          .collect(Collectors.toMap(com.googlecode.objectify.Key::getKind, identity()));
+
+  /**
+   * Class name registry allowing us to obtain the class name the unqualified class, which is all
+   * the datastore key gives us. Note that entities annotated with @EntitySubclass are removed
+   * because they share the same kind of the key with their parent class.
+   */
+  public static final Map<Class<?>, String> CLASS_NAME_REGISTRY =
+      ALL_CLASSES.stream()
+          .filter(clazz -> !clazz.isAnnotationPresent(EntitySubclass.class))
+          .collect(Collectors.toMap(identity(), com.googlecode.objectify.Key::getKind));
+
+  @VisibleForTesting
+  public static void addTestEntityClass(Class<?> clazz) {
+    CLASS_REGISTRY.put(clazz.getSimpleName(), clazz);
+    CLASS_NAME_REGISTRY.put(clazz, clazz.getSimpleName());
+  }
+
+  public static <T> Class<T> getClass(String className) {
+    checkArgument(CLASS_REGISTRY.containsKey(className), "Class not found in class registry");
+    return (Class<T>) CLASS_REGISTRY.get(className);
+  }
+
+  public static <T> String getClassName(Class<T> clazz) {
+    checkArgument(CLASS_NAME_REGISTRY.containsKey(clazz), "Class not found in class name registry");
+    return CLASS_NAME_REGISTRY.get(clazz);
+  }
+}

core/src/main/java/google/registry/model/common/Cursor.java

@@ -19,7 +19,9 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
+import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -294,5 +296,22 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity, Un
       this.type = type;
       this.scope = scope;
     }
+
+    /**
+     * A deterministic string representation of a {@link CursorId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of("scope", scope, "type", type)
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 }

core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java

@@ -204,7 +204,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
             MigrationState.DATASTORE_ONLY,
             "migrationTransitionMap must start with DATASTORE_ONLY");
     validateTransitionAtCurrentTime(transitions);
-    jpaTm().putIgnoringReadOnly(new DatabaseMigrationStateSchedule(transitions));
+    jpaTm().putIgnoringReadOnlyWithoutBackup(new DatabaseMigrationStateSchedule(transitions));
     CACHE.invalidateAll();
   }
 

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -16,6 +16,8 @@ package google.registry.model.contact;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
@@ -200,6 +202,24 @@ public class ContactHistory extends HistoryEntry implements SqlEntity, UnsafeSer
     private void setId(long id) {
       this.id = id;
     }
+
+    /**
+     * A deterministic string representation of a {@link ContactHistoryId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "contactRepoId", getContactRepoId(), "id", getId())
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   @Override

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.domain;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+
 import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ForeignKeyedEppResource;
@@ -168,6 +170,10 @@ public class DomainBase extends DomainContent
   @Override
   public void beforeSqlSaveOnReplay() {
     fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+    dsData =
+        dsData.stream()
+            .filter(datum -> datum.getDigest() != null && datum.getDigest().length > 0)
+            .collect(toImmutableSet());
   }
 
   @Override

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -86,6 +86,7 @@ import javax.persistence.Embedded;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.PostLoad;
 import javax.persistence.Transient;
+import org.hibernate.collection.internal.PersistentSet;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -508,16 +509,37 @@ public class DomainContent extends EppResource
     this.nsHosts = forceEmptyToNull(nsHosts);
   }
 
+  // Note: for the two methods below, how we wish to treat the Hibernate setters depends on the
+  // current state of the object and what's passed in. The key principle is that we wish to maintain
+  // the link between parent and child objects, meaning that we should keep around whichever of the
+  // two sets (the parameter vs the class variable and clear/populate that as appropriate.
+  //
+  // If the class variable is a PersistentSet and we overwrite it here, Hibernate will throw
+  // an exception "A collection with cascade=”all-delete-orphan” was no longer referenced by the
+  // owning entity instance". See https://stackoverflow.com/questions/5587482 for more details.
+
   // Hibernate needs this in order to populate gracePeriods but no one else should ever use it
   @SuppressWarnings("UnusedMethod")
   private void setInternalGracePeriods(Set<GracePeriod> gracePeriods) {
-    this.gracePeriods = gracePeriods;
+    if (this.gracePeriods instanceof PersistentSet) {
+      Set<GracePeriod> nonNullGracePeriods = nullToEmpty(gracePeriods);
+      this.gracePeriods.retainAll(nonNullGracePeriods);
+      this.gracePeriods.addAll(nonNullGracePeriods);
+    } else {
+      this.gracePeriods = gracePeriods;
+    }
   }
 
   // Hibernate needs this in order to populate dsData but no one else should ever use it
   @SuppressWarnings("UnusedMethod")
   private void setInternalDelegationSignerData(Set<DelegationSignerData> dsData) {
-    this.dsData = dsData;
+    if (this.dsData instanceof PersistentSet) {
+      Set<DelegationSignerData> nonNullDsData = nullToEmpty(dsData);
+      this.dsData.retainAll(nonNullDsData);
+      this.dsData.addAll(nonNullDsData);
+    } else {
+      this.dsData = dsData;
+    }
   }
 
   public final String getCurrentSponsorRegistrarId() {

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -18,7 +18,9 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
@@ -194,7 +196,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   @Access(AccessType.PROPERTY)
   @OneToMany(
       cascade = {CascadeType.ALL},
-      fetch = FetchType.EAGER)
+      fetch = FetchType.EAGER,
+      orphanRemoval = true)
   @JoinColumn(name = "historyRevisionId", referencedColumnName = "historyRevisionId")
   @JoinColumn(name = "domainRepoId", referencedColumnName = "domainRepoId")
   @SuppressWarnings("unused")
@@ -206,7 +209,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   @SuppressWarnings("unused")
   private void setInternalDomainTransactionRecords(
       Set<DomainTransactionRecord> domainTransactionRecords) {
-    this.domainTransactionRecords = domainTransactionRecords;
+    super.setDomainTransactionRecords(domainTransactionRecords);
   }
 
   @Id
@@ -315,6 +318,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
       domainHistory.nsHosts = nullToEmptyImmutableCopy(domainHistory.domainContent.nsHosts);
       domainHistory.dsDataHistories =
           nullToEmptyImmutableCopy(domainHistory.domainContent.getDsData()).stream()
+              .filter(dsData -> dsData.getDigest() != null && dsData.getDigest().length > 0)
               .map(dsData -> DomainDsDataHistory.createFrom(domainHistory.id, dsData))
               .collect(toImmutableSet());
       domainHistory.gracePeriodHistories =
@@ -382,6 +386,24 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     private void setId(long id) {
       this.id = id;
     }
+
+    /**
+     * A deterministic string representation of a {@link DomainHistoryId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "domainRepoId", getDomainRepoId(), "id", getId())
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   @Override

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -16,6 +16,8 @@ package google.registry.model.host;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSortedMap;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.EppResource;
@@ -203,6 +205,24 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial
     private void setId(long id) {
       this.id = id;
     }
+
+    /**
+     * A deterministic string representation of a {@link HostHistoryId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "hostRepoId", getHostRepoId(), "id", getId())
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   @Override

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -64,6 +64,7 @@ public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEn
     EppResourceIndex instance = instantiate(EppResourceIndex.class);
     instance.reference = resourceKey;
     instance.kind = resourceKey.getKind();
+    // TODO(b/207368050): figure out if this value has ever been used other than test cases
     instance.id = resourceKey.getString(); // creates a web-safe key string
     instance.bucket = bucket;
     return instance;

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -82,6 +82,7 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
       com.google.appengine.api.datastore.Entity rawEntity) {
     CommitLogMutation instance = new CommitLogMutation();
     instance.parent = checkNotNull(parent);
+    // TODO(b/207516684): figure out if this should be converted to a vkey string via stringify()
     // Creates a web-safe key string.
     instance.entityKey = KeyFactory.keyToString(rawEntity.getKey());
     instance.entityProtoBytes = convertToPb(rawEntity).toByteArray();
@@ -91,6 +92,8 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
   /** Returns the key of a mutation based on the {@code entityKey} of the entity it stores. */
   public static
       Key<CommitLogMutation> createKey(Key<CommitLogManifest> parent, Key<?> entityKey) {
+    // TODO(b/207516684): figure out if the return type needs to be VKey and
+    //  if the string used to create a key should remain the same
     return Key.create(parent, CommitLogMutation.class, entityKey.getString());
   }
 }

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -356,13 +356,25 @@ public class DatastoreTransactionManager implements TransactionManager {
   }
 
   @Override
-  public void putIgnoringReadOnly(Object entity) {
-    syncIfTransactionless(getOfy().saveIgnoringReadOnly().entities(toDatastoreEntity(entity)));
+  public void putIgnoringReadOnlyWithoutBackup(Object entity) {
+    syncIfTransactionless(
+        getOfy().saveIgnoringReadOnlyWithoutBackup().entities(toDatastoreEntity(entity)));
   }
 
   @Override
-  public void deleteIgnoringReadOnly(VKey<?> key) {
-    syncIfTransactionless(getOfy().deleteIgnoringReadOnly().key(key.getOfyKey()));
+  public void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key) {
+    syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithoutBackup().key(key.getOfyKey()));
+  }
+
+  /** Performs the write ignoring read-only restrictions and also writes commit logs. */
+  public void putIgnoringReadOnlyWithBackup(Object entity) {
+    syncIfTransactionless(
+        getOfy().saveIgnoringReadOnlyWithBackup().entities(toDatastoreEntity(entity)));
+  }
+
+  /** Performs the delete ignoring read-only restrictions and also writes commit logs. */
+  public void deleteIgnoringReadOnlyWithBackup(VKey<?> key) {
+    syncIfTransactionless(getOfy().deleteIgnoringReadOnlyWithBackup().key(key.getOfyKey()));
   }
 
   /**

core/src/main/java/google/registry/model/ofy/Ofy.java

@@ -133,15 +133,7 @@ public class Ofy {
    */
   public Deleter delete() {
     assertNotReadOnlyMode();
-    return new AugmentedDeleter() {
-      @Override
-      protected void handleDeletion(Iterable<Key<?>> keys) {
-        assertInTransaction();
-        checkState(Streams.stream(keys).allMatch(Objects::nonNull), "Can't delete a null key.");
-        checkProhibitedAnnotations(keys, NotBackedUp.class, VirtualEntity.class);
-        TRANSACTION_INFO.get().putDeletes(keys);
-      }
-    };
+    return deleteIgnoringReadOnlyWithBackup();
   }
 
   /**
@@ -151,7 +143,7 @@ public class Ofy {
    */
   public Deleter deleteWithoutBackup() {
     assertNotReadOnlyMode();
-    return deleteIgnoringReadOnly();
+    return deleteIgnoringReadOnlyWithoutBackup();
   }
 
   /**
@@ -162,6 +154,41 @@ public class Ofy {
    */
   public Saver save() {
     assertNotReadOnlyMode();
+    return saveIgnoringReadOnlyWithBackup();
+  }
+
+  /**
+   * Save, without any augmentations except to check that we're not saving any virtual entities.
+   *
+   * <p>No backups get written.
+   */
+  public Saver saveWithoutBackup() {
+    assertNotReadOnlyMode();
+    return saveIgnoringReadOnlyWithoutBackup();
+  }
+
+  /** Save, ignoring any backups or any read-only settings. */
+  public Saver saveIgnoringReadOnlyWithoutBackup() {
+    return new AugmentedSaver() {
+      @Override
+      protected void handleSave(Iterable<?> entities) {
+        checkProhibitedAnnotations(entities, VirtualEntity.class);
+      }
+    };
+  }
+
+  /** Delete, ignoring any backups or any read-only settings. */
+  public Deleter deleteIgnoringReadOnlyWithoutBackup() {
+    return new AugmentedDeleter() {
+      @Override
+      protected void handleDeletion(Iterable<Key<?>> keys) {
+        checkProhibitedAnnotations(keys, VirtualEntity.class);
+      }
+    };
+  }
+
+  /** Save, ignoring any read-only settings (but still write commit logs). */
+  public Saver saveIgnoringReadOnlyWithBackup() {
     return new AugmentedSaver() {
       @Override
       protected void handleSave(Iterable<?> entities) {
@@ -175,32 +202,15 @@ public class Ofy {
     };
   }
 
-  /**
-   * Save, without any augmentations except to check that we're not saving any virtual entities.
-   *
-   * <p>No backups get written.
-   */
-  public Saver saveWithoutBackup() {
-    assertNotReadOnlyMode();
-    return saveIgnoringReadOnly();
-  }
-
-  /** Save, ignoring any backups or any read-only settings. */
-  public Saver saveIgnoringReadOnly() {
-    return new AugmentedSaver() {
-      @Override
-      protected void handleSave(Iterable<?> entities) {
-        checkProhibitedAnnotations(entities, VirtualEntity.class);
-      }
-    };
-  }
-
-  /** Delete, ignoring any backups or any read-only settings. */
-  public Deleter deleteIgnoringReadOnly() {
+  /** Delete, ignoring any read-only settings (but still write commit logs). */
+  public Deleter deleteIgnoringReadOnlyWithBackup() {
     return new AugmentedDeleter() {
       @Override
       protected void handleDeletion(Iterable<Key<?>> keys) {
-        checkProhibitedAnnotations(keys, VirtualEntity.class);
+        assertInTransaction();
+        checkState(Streams.stream(keys).allMatch(Objects::nonNull), "Can't delete a null key.");
+        checkProhibitedAnnotations(keys, NotBackedUp.class, VirtualEntity.class);
+        TRANSACTION_INFO.get().putDeletes(keys);
       }
     };
   }

core/src/main/java/google/registry/model/ofy/ReplayQueue.java

@@ -95,22 +95,21 @@ public class ReplayQueue {
       // Sort the changes into an order that will work for insertion into the database.
       jpaTm()
           .transact(
-              () -> {
-                changes.entrySet().stream()
-                    .sorted(ReplayQueue::compareByPriority)
-                    .forEach(
-                        entry -> {
-                          if (entry.getValue().equals(TransactionInfo.Delete.SENTINEL)) {
-                            VKey<?> vkey = VKey.from(entry.getKey());
-                            ReplaySpecializer.beforeSqlDelete(vkey);
-                            jpaTm().delete(vkey);
-                          } else {
-                            ((DatastoreEntity) entry.getValue())
-                                .toSqlEntity()
-                                .ifPresent(jpaTm()::put);
-                          }
-                        });
-              });
+              () ->
+                  changes.entrySet().stream()
+                      .sorted(ReplayQueue::compareByPriority)
+                      .forEach(
+                          entry -> {
+                            if (entry.getValue().equals(TransactionInfo.Delete.SENTINEL)) {
+                              VKey<?> vkey = VKey.from(entry.getKey());
+                              ReplaySpecializer.beforeSqlDelete(vkey);
+                              jpaTm().delete(vkey);
+                            } else {
+                              ((DatastoreEntity) entry.getValue())
+                                  .toSqlEntity()
+                                  .ifPresent(jpaTm()::put);
+                            }
+                          }));
     }
   }
 }

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -354,6 +354,10 @@ public abstract class PollMessage extends ImmutableObject
     @Column(name = "transfer_response_contact_id")
     String contactId;
 
+    @Ignore
+    @Column(name = "transfer_response_host_id")
+    String hostId;
+
     @Override
     public VKey<OneTime> createVKey() {
       return VKey.create(OneTime.class, getId(), Key.create(this));
@@ -393,6 +397,9 @@ public abstract class PollMessage extends ImmutableObject
       if (!isNullOrEmpty(contactPendingActionNotificationResponses)) {
         pendingActionNotificationResponse = contactPendingActionNotificationResponses.get(0);
       }
+      if (!isNullOrEmpty(hostPendingActionNotificationResponses)) {
+        pendingActionNotificationResponse = hostPendingActionNotificationResponses.get(0);
+      }
       if (!isNullOrEmpty(contactTransferResponses)) {
         contactId = contactTransferResponses.get(0).getContactId();
         transferResponse = contactTransferResponses.get(0);
@@ -433,6 +440,16 @@ public abstract class PollMessage extends ImmutableObject
                   pendingActionNotificationResponse.processedDate);
           pendingActionNotificationResponse = domainPendingResponse;
           domainPendingActionNotificationResponses = ImmutableList.of(domainPendingResponse);
+        } else if (hostId != null) {
+          HostPendingActionNotificationResponse hostPendingActionNotificationResponse =
+              HostPendingActionNotificationResponse.create(
+                  pendingActionNotificationResponse.nameOrId.value,
+                  pendingActionNotificationResponse.getActionResult(),
+                  pendingActionNotificationResponse.getTrid(),
+                  pendingActionNotificationResponse.processedDate);
+          pendingActionNotificationResponse = hostPendingActionNotificationResponse;
+          hostPendingActionNotificationResponses =
+              ImmutableList.of(hostPendingActionNotificationResponse);
         }
       }
       if (transferResponse != null) {
@@ -527,6 +544,7 @@ public abstract class PollMessage extends ImmutableObject
         } else if (instance.hostPendingActionNotificationResponses != null) {
           instance.pendingActionNotificationResponse =
               instance.hostPendingActionNotificationResponses.get(0);
+          instance.hostId = instance.hostPendingActionNotificationResponses.get(0).nameOrId.value;
         }
         // Set the generic transfer response field as appropriate
         if (instance.contactTransferResponses != null) {

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -32,7 +32,9 @@ import static java.util.stream.Collectors.joining;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Enums;
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
@@ -413,6 +415,24 @@ public class RegistrarContact extends ImmutableObject
       this.emailAddress = emailAddress;
       this.registrarId = registrarId;
     }
+
+    /**
+     * A deterministic string representation of a {@link RegistrarPocId}. See {@link
+     * ImmutableObject#toString} for more information.
+     */
+    @Override
+    public String toString() {
+      return String.format(
+                  "%s: {\n%s",
+                  getClass().getSimpleName(),
+                  Joiner.on('\n')
+                      .join(
+                          ImmutableSortedMap.<String, Object>of(
+                                  "emailAddress", emailAddress, "registrarId", registrarId)
+                              .entrySet()))
+              .replaceAll("\n", "\n    ")
+          + "\n}";
+    }
   }
 
   /** A builder for constructing a {@link RegistrarContact}, since it is immutable. */

core/src/main/java/google/registry/model/replay/ReplicateToDatastoreAction.java

@@ -148,7 +148,9 @@ public class ReplicateToDatastoreAction implements Runnable {
 
                 // Write the updated last transaction id to Datastore as part of this Datastore
                 // transaction.
-                auditedOfy().save().entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
+                auditedOfy()
+                    .saveIgnoringReadOnlyWithoutBackup()
+                    .entity(lastSqlTxn.cloneWithNewTransactionId(nextTxnId));
                 logger.atInfo().log(
                     "Finished applying single transaction Cloud SQL -> Cloud Datastore.");
               });

core/src/main/java/google/registry/model/replay/SqlReplayCheckpoint.java

@@ -41,6 +41,6 @@ public class SqlReplayCheckpoint extends CrossTldSingleton implements SqlOnlyEnt
     SqlReplayCheckpoint checkpoint = new SqlReplayCheckpoint();
     checkpoint.lastReplayTime = lastReplayTime;
     // this will overwrite the existing object due to the constant revisionId
-    jpaTm().putIgnoringReadOnly(checkpoint);
+    jpaTm().putIgnoringReadOnlyWithoutBackup(checkpoint);
   }
 }

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -16,6 +16,7 @@ package google.registry.model.reporting;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.googlecode.objectify.Key.getKind;
+import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -63,6 +64,7 @@ import javax.persistence.Enumerated;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
 import org.apache.commons.lang3.BooleanUtils;
+import org.hibernate.collection.internal.PersistentSet;
 import org.joda.time.DateTime;
 
 /**
@@ -317,9 +319,23 @@ public class HistoryEntry extends ImmutableObject
 
   /** This method exists solely to satisfy Hibernate. Use the {@link Builder} instead. */
   @SuppressWarnings("UnusedMethod")
-  private void setDomainTransactionRecords(Set<DomainTransactionRecord> domainTransactionRecords) {
-    this.domainTransactionRecords =
-        domainTransactionRecords == null ? null : ImmutableSet.copyOf(domainTransactionRecords);
+  protected void setDomainTransactionRecords(
+      Set<DomainTransactionRecord> domainTransactionRecords) {
+    // Note: how we wish to treat this Hibernate setter depends on the current state of the object
+    // and what's passed in. The key principle is that we wish to maintain the link between parent
+    // and child objects, meaning that we should keep around whichever of the two sets (the
+    // parameter vs the class variable and clear/populate that as appropriate.
+    //
+    // If the class variable is a PersistentSet and we overwrite it here, Hibernate will throw
+    // an exception "A collection with cascade=”all-delete-orphan” was no longer referenced by the
+    // owning entity instance". See https://stackoverflow.com/questions/5587482 for more details.
+    if (this.domainTransactionRecords instanceof PersistentSet) {
+      Set<DomainTransactionRecord> nonNullRecords = nullToEmpty(domainTransactionRecords);
+      this.domainTransactionRecords.retainAll(nonNullRecords);
+      this.domainTransactionRecords.addAll(nonNullRecords);
+    } else {
+      this.domainTransactionRecords = domainTransactionRecords;
+    }
   }
 
   /**

core/src/main/java/google/registry/model/server/Lock.java

@@ -293,7 +293,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
               create(resourceName, scope, requestStatusChecker.getLogId(), now, leaseLength);
           // Locks are not parented under an EntityGroupRoot (so as to avoid write
           // contention) and don't need to be backed up.
-          transactionManager.putIgnoringReadOnly(newLock);
+          transactionManager.putIgnoringReadOnlyWithoutBackup(newLock);
 
           return AcquireResult.create(now, lock, newLock, lockState);
         };
@@ -325,7 +325,7 @@ public class Lock extends ImmutableObject implements DatastoreAndSqlEntity, Seri
             // Use deleteIgnoringReadOnly() so that we don't create a commit log entry for deleting
             // the lock.
             logger.atInfo().log("Deleting lock: %s", lockId);
-            transactionManager.deleteIgnoringReadOnly(key);
+            transactionManager.deleteIgnoringReadOnlyWithoutBackup(key);
 
             lockMetrics.recordRelease(
                 resourceName,

core/src/main/java/google/registry/model/tld/Registry.java

@@ -64,6 +64,7 @@ import google.registry.model.replay.DatastoreAndSqlEntity;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.ReservedList;
 import google.registry.persistence.VKey;
+import google.registry.persistence.converter.JodaMoneyType;
 import google.registry.util.Idn;
 import java.util.Map;
 import java.util.Optional;
@@ -72,13 +73,13 @@ import java.util.concurrent.ExecutionException;
 import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import javax.annotation.Nullable;
-import javax.persistence.AttributeOverride;
-import javax.persistence.AttributeOverrides;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.persistence.PostLoad;
 import javax.persistence.Transient;
+import org.hibernate.annotations.Columns;
+import org.hibernate.annotations.Type;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -466,47 +467,39 @@ public class Registry extends ImmutableObject
   CurrencyUnit currency = DEFAULT_CURRENCY;
 
   /** The per-year billing cost for registering a new domain name. */
-  @AttributeOverrides({
-    @AttributeOverride(
-        name = "money.amount",
-        column = @Column(name = "create_billing_cost_amount")),
-    @AttributeOverride(
-        name = "money.currency",
-        column = @Column(name = "create_billing_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "create_billing_cost_amount"),
+        @Column(name = "create_billing_cost_currency")
+      })
   Money createBillingCost = DEFAULT_CREATE_BILLING_COST;
 
   /** The one-time billing cost for restoring a domain name from the redemption grace period. */
-  @AttributeOverrides({
-    @AttributeOverride(
-        name = "money.amount",
-        column = @Column(name = "restore_billing_cost_amount")),
-    @AttributeOverride(
-        name = "money.currency",
-        column = @Column(name = "restore_billing_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "restore_billing_cost_amount"),
+        @Column(name = "restore_billing_cost_currency")
+      })
   Money restoreBillingCost = DEFAULT_RESTORE_BILLING_COST;
 
   /** The one-time billing cost for changing the server status (i.e. lock). */
-  @AttributeOverrides({
-    @AttributeOverride(
-        name = "money.amount",
-        column = @Column(name = "server_status_change_billing_cost_amount")),
-    @AttributeOverride(
-        name = "money.currency",
-        column = @Column(name = "server_status_change_billing_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "server_status_change_billing_cost_amount"),
+        @Column(name = "server_status_change_billing_cost_currency")
+      })
   Money serverStatusChangeBillingCost = DEFAULT_SERVER_STATUS_CHANGE_BILLING_COST;
 
   /** The one-time billing cost for a registry lock/unlock action initiated by a registrar. */
-  @AttributeOverrides({
-      @AttributeOverride(
-          name = "money.amount",
-          column = @Column(name = "registry_lock_or_unlock_cost_amount")),
-      @AttributeOverride(
-          name = "money.currency",
-          column = @Column(name = "registry_lock_or_unlock_cost_currency"))
-  })
+  @Type(type = JodaMoneyType.TYPE_NAME)
+  @Columns(
+      columns = {
+        @Column(name = "registry_lock_or_unlock_cost_amount"),
+        @Column(name = "registry_lock_or_unlock_cost_currency")
+      })
   Money registryLockOrUnlockBillingCost = DEFAULT_REGISTRY_LOCK_OR_UNLOCK_BILLING_COST;
 
   /**

core/src/main/java/google/registry/model/translators/VKeyTranslatorFactory.java

@@ -14,18 +14,13 @@
 
 package google.registry.model.translators;
 
-import static com.google.common.base.Functions.identity;
 import static com.google.common.base.Preconditions.checkArgument;
-import static google.registry.model.EntityClasses.ALL_CLASSES;
 
 import com.google.appengine.api.datastore.Key;
-import com.google.common.annotations.VisibleForTesting;
-import com.googlecode.objectify.annotation.EntitySubclass;
+import google.registry.model.common.ClassPathManager;
 import google.registry.persistence.VKey;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.Map;
-import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 /**
@@ -36,15 +31,6 @@ import javax.annotation.Nullable;
  */
 public class VKeyTranslatorFactory extends AbstractSimpleTranslatorFactory<VKey, Key> {
 
-  // Class registry allowing us to restore the original class object from the unqualified class
-  // name, which is all the datastore key gives us.
-  // Note that entities annotated with @EntitySubclass are removed because they share the same
-  // kind of the key with their parent class.
-  private static final Map<String, Class<?>> CLASS_REGISTRY =
-      ALL_CLASSES.stream()
-          .filter(clazz -> !clazz.isAnnotationPresent(EntitySubclass.class))
-          .collect(Collectors.toMap(com.googlecode.objectify.Key::getKind, identity()));
-
   public VKeyTranslatorFactory() {
     super(VKey.class);
   }
@@ -67,7 +53,7 @@ public class VKeyTranslatorFactory extends AbstractSimpleTranslatorFactory<VKey,
     }
 
     // Try to create the VKey from its reference type.
-    Class<T> clazz = (Class<T>) CLASS_REGISTRY.get(key.getKind());
+    Class<T> clazz = ClassPathManager.getClass(key.getKind());
     checkArgument(clazz != null, "Unknown Key type: %s", key.getKind());
     try {
       Method createVKeyMethod =
@@ -92,11 +78,6 @@ public class VKeyTranslatorFactory extends AbstractSimpleTranslatorFactory<VKey,
     }
   }
 
-  @VisibleForTesting
-  public static void addTestEntityClass(Class<?> clazz) {
-    CLASS_REGISTRY.put(com.googlecode.objectify.Key.getKind(clazz), clazz);
-  }
-
   @Override
   public SimpleTranslator<VKey, Key> createTranslator() {
     return new SimpleTranslator<VKey, Key>() {

core/src/main/java/google/registry/persistence/NomulusPostgreSQLDialect.java

@@ -14,6 +14,7 @@
 package google.registry.persistence;
 
 import google.registry.persistence.converter.IntervalDescriptor;
+import google.registry.persistence.converter.JodaMoneyType;
 import google.registry.persistence.converter.StringCollectionDescriptor;
 import google.registry.persistence.converter.StringMapDescriptor;
 import java.sql.Types;
@@ -34,6 +35,7 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     registerColumnType(IntervalDescriptor.COLUMN_TYPE, IntervalDescriptor.COLUMN_NAME);
   }
 
+  @SuppressWarnings("deprecation") // See comments below on JodaMoneyType.
   @Override
   public void contributeTypes(
       TypeContributions typeContributions, ServiceRegistry serviceRegistry) {
@@ -44,5 +46,8 @@ public class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
     typeContributions.contributeSqlTypeDescriptor(StringMapDescriptor.getInstance());
     typeContributions.contributeJavaTypeDescriptor(IntervalDescriptor.getInstance());
     typeContributions.contributeSqlTypeDescriptor(IntervalDescriptor.getInstance());
+    // Below method (contributing CompositeUserType) is deprecated. Please see javadoc of
+    // JodaMoneyType for reasons.
+    typeContributions.contributeType(JodaMoneyType.INSTANCE, JodaMoneyType.TYPE_NAME);
   }
 }

core/src/main/java/google/registry/persistence/VKey.java

@@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableMap;
 import com.googlecode.objectify.Key;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.ImmutableObject;
+import google.registry.model.common.ClassPathManager;
 import google.registry.model.translators.VKeyTranslatorFactory;
 import google.registry.util.SerializeUtils;
 import java.io.Serializable;
@@ -137,9 +138,9 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    *
    * <p>Example of a Vkey string by fromWebsafeKey(): "agR0ZXN0chYLEgpEb21haW5CYXNlIgZST0lELTEM"
    *
-   * <p>Example of a vkey string by stringify(): "google.registry.testing.TestObject@sql:rO0ABX" +
-   * "QAA2Zvbw@ofy:agR0ZXN0cjELEg9FbnRpdHlHcm91cFJvb3QiCWNyb3NzLXRsZAwLEgpUZXN0T2JqZWN0IgNmb28M",
-   * where sql key and ofy key are values are encoded in Base64.
+   * <p>Example of a vkey string by stringify(): "kind:TestObject@sql:rO0ABXQAA2Zvbw" +
+   * "@ofy:agR0ZXN0cjELEg9FbnRpdHlHcm91cFJvb3QiCWNyb3NzLXRsZAwLEgpUZXN0T2JqZWN0IgNmb28M", where sql
+   * key and ofy key values are encoded in Base64.
    */
   public static <T> VKey<T> create(String keyString) throws Exception {
     if (!keyString.startsWith(CLASS_TYPE + KV_SEPARATOR)) {
@@ -149,7 +150,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
       ImmutableMap<String, String> kvs =
           ImmutableMap.copyOf(
               Splitter.on(DELIMITER).withKeyValueSeparator(KV_SEPARATOR).split(keyString));
-      Class classType = Class.forName(kvs.get(CLASS_TYPE));
+      Class classType = ClassPathManager.getClass(kvs.get(CLASS_TYPE));
 
       if (kvs.containsKey(SQL_LOOKUP_KEY) && kvs.containsKey(OFY_LOOKUP_KEY)) {
         return VKey.create(
@@ -290,7 +291,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
   /**
    * Constructs the string representation of a {@link VKey}.
    *
-   * <p>The string representation of a vkey contains its type, and sql key or ofy key, or both. Each
+   * <p>The string representation of a vkey contains its kind, and sql key or ofy key, or both. Each
    * of the keys is first serialized into a byte array then encoded via Base64 into a web safe
    * string.
    *
@@ -302,7 +303,7 @@ public class VKey<T> extends ImmutableObject implements Serializable {
    */
   public String stringify() {
     // class type is required to create a vkey
-    String key = CLASS_TYPE + KV_SEPARATOR + getKind().getName();
+    String key = CLASS_TYPE + KV_SEPARATOR + ClassPathManager.getClassName(getKind());
     if (maybeGetSqlKey().isPresent()) {
       key += DELIMITER + SQL_LOOKUP_KEY + KV_SEPARATOR + SerializeUtils.stringify(getSqlKey());
     }
@@ -311,4 +312,29 @@ public class VKey<T> extends ImmutableObject implements Serializable {
     }
     return key;
   }
+
+  /**
+   * Constructs the readable string representation of a {@link VKey}.
+   *
+   * <p>This readable string representation of a vkey contains its kind and its sql key or ofy key,
+   * or both.
+   */
+  @Override
+  public String toString() {
+    if (maybeGetOfyKey().isPresent() && maybeGetSqlKey().isPresent()) {
+      return String.format(
+          "VKey<%s>(%s:%s,%s:%s)",
+          getKind().getSimpleName(), SQL_LOOKUP_KEY, sqlKey, OFY_LOOKUP_KEY, ofyKeyToString());
+    } else if (maybeGetSqlKey().isPresent()) {
+      return String.format("VKey<%s>(%s:%s)", getKind().getSimpleName(), SQL_LOOKUP_KEY, sqlKey);
+    } else if (maybeGetOfyKey().isPresent()) {
+      return String.format("VKey<%s>(%s:%s)", ofyKey.getKind(), OFY_LOOKUP_KEY, ofyKeyToString());
+    } else {
+      throw new IllegalStateException("VKey should contain at least one form of key");
+    }
+  }
+
+  private String ofyKeyToString() {
+    return ofyKey.getName() == null ? String.valueOf(ofyKey.getId()) : ofyKey.getName();
+  }
 }

core/src/main/java/google/registry/persistence/converter/JodaMoneyType.java

@@ -0,0 +1,178 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import com.google.common.collect.ImmutableList;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Objects;
+import org.hibernate.HibernateException;
+import org.hibernate.engine.spi.SharedSessionContractImplementor;
+import org.hibernate.type.StandardBasicTypes;
+import org.hibernate.type.Type;
+import org.hibernate.usertype.CompositeUserType;
+import org.joda.money.CurrencyUnit;
+import org.joda.money.Money;
+
+/**
+ * Defines JPA mapping for {@link Money Joda Money type}.
+ *
+ * <p>{@code Money} is mapped to two table columns, a text {@code currency} column that stores the
+ * currency code, and a numeric {@code amount} column that stores the amount.
+ *
+ * <p>The main purpose of this class is to normalize the amount loaded from the database. To support
+ * all currency types, the scale of the numeric column is set to 2. As a result, the {@link
+ * BigDecimal} instances obtained from query ResultSets all have their scale at 2. However, some
+ * currency types, e.g., JPY requires that the scale be zero. This class strips trailing zeros from
+ * each loaded BigDecimal, then calls the appropriate factory method for Money, which will adjust
+ * the scale appropriately.
+ *
+ * <p>Although {@link CompositeUserType} is likely to suffer breaking change in Hibernate 6, it is
+ * the only option. The suggested alternatives such as Hibernate component or Java Embeddable do not
+ * work in this case. Hibernate component (our previous solution that is replaced by this class)
+ * does not allow manipulation of the loaded amount objects. Java Embeddable is not applicable since
+ * we do not own the Joda money classes.
+ *
+ * <p>Usage:
+ *
+ * <pre>{@code
+ * '@'Type(type = JodaMoneyType.TYPE_NAME)
+ * '@'Columns(
+ *   columns = {
+ *     '@'Column(name = "cost_amount"),
+ *     '@'Column(name = "cost_currency")
+ *   }
+ * )
+ * Money cost;
+ * }</pre>
+ */
+public class JodaMoneyType implements CompositeUserType {
+
+  public static final JodaMoneyType INSTANCE = new JodaMoneyType();
+
+  /** The name of this type registered with JPA. See the example in class doc. */
+  public static final String TYPE_NAME = "JodaMoney";
+
+  // JPA property names that can be used in JPQL queries.
+  private static final ImmutableList<String> JPA_PROPERTY_NAMES =
+      ImmutableList.of("amount", "currency");
+  private static final ImmutableList<Type> PROPERTY_TYPES =
+      ImmutableList.of(StandardBasicTypes.BIG_DECIMAL, StandardBasicTypes.STRING);
+  private static final int AMOUNT_ID = JPA_PROPERTY_NAMES.indexOf("amount");
+  private static final int CURRENCY_ID = JPA_PROPERTY_NAMES.indexOf("currency");
+
+  @Override
+  public String[] getPropertyNames() {
+    return JPA_PROPERTY_NAMES.toArray(new String[0]);
+  }
+
+  @Override
+  public Type[] getPropertyTypes() {
+    return PROPERTY_TYPES.toArray(new Type[0]);
+  }
+
+  @Override
+  public Object getPropertyValue(Object component, int property) throws HibernateException {
+    if (property >= JPA_PROPERTY_NAMES.size()) {
+      throw new HibernateException("Property index too large: " + property);
+    }
+    Money money = (Money) component;
+    return property == AMOUNT_ID ? money.getAmount() : money.getCurrencyUnit().getCode();
+  }
+
+  @Override
+  public void setPropertyValue(Object component, int property, Object value)
+      throws HibernateException {
+    throw new HibernateException("Money is immutable");
+  }
+
+  @Override
+  public Class returnedClass() {
+    return Money.class;
+  }
+
+  @Override
+  public boolean equals(Object x, Object y) throws HibernateException {
+    return Objects.equals(x, y);
+  }
+
+  @Override
+  public int hashCode(Object x) throws HibernateException {
+    return Objects.hashCode(x);
+  }
+
+  @Override
+  public Object nullSafeGet(
+      ResultSet rs, String[] names, SharedSessionContractImplementor session, Object owner)
+      throws HibernateException, SQLException {
+    BigDecimal amount = StandardBasicTypes.BIG_DECIMAL.nullSafeGet(rs, names[AMOUNT_ID], session);
+    CurrencyUnit currencyUnit =
+        CurrencyUnit.of(StandardBasicTypes.STRING.nullSafeGet(rs, names[CURRENCY_ID], session));
+    if (amount != null && currencyUnit != null) {
+      return Money.of(currencyUnit, amount.stripTrailingZeros());
+    }
+    if (amount == null && currencyUnit == null) {
+      return null;
+    }
+    throw new HibernateException("Mismatching null state between currency and amount.");
+  }
+
+  @Override
+  public void nullSafeSet(
+      PreparedStatement st, Object value, int index, SharedSessionContractImplementor session)
+      throws HibernateException, SQLException {
+    BigDecimal amount = value == null ? null : ((Money) value).getAmount();
+    String currencyUnit = value == null ? null : ((Money) value).getCurrencyUnit().getCode();
+
+    if ((amount == null && currencyUnit != null) || (amount != null && currencyUnit == null)) {
+      throw new HibernateException("Mismatching null state between currency and amount.");
+    }
+    StandardBasicTypes.BIG_DECIMAL.nullSafeSet(st, amount, index, session);
+    StandardBasicTypes.STRING.nullSafeSet(st, currencyUnit, index + 1, session);
+  }
+
+  @Override
+  public Object deepCopy(Object value) throws HibernateException {
+    return value;
+  }
+
+  @Override
+  public boolean isMutable() {
+    return false;
+  }
+
+  @Override
+  public Serializable disassemble(Object value, SharedSessionContractImplementor session)
+      throws HibernateException {
+    return ((Money) value);
+  }
+
+  @Override
+  public Object assemble(
+      Serializable cached, SharedSessionContractImplementor session, Object owner)
+      throws HibernateException {
+    return cached;
+  }
+
+  @Override
+  public Object replace(
+      Object original, Object target, SharedSessionContractImplementor session, Object owner)
+      throws HibernateException {
+    return original;
+  }
+}

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -637,7 +637,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void putIgnoringReadOnly(Object entity) {
+  public void putIgnoringReadOnlyWithoutBackup(Object entity) {
     checkArgumentNotNull(entity);
     if (isEntityOfIgnoredClass(entity)) {
       return;
@@ -652,7 +652,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   }
 
   @Override
-  public void deleteIgnoringReadOnly(VKey<?> key) {
+  public void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key) {
     checkArgumentNotNull(key, "key must be specified");
     assertInTransaction();
     if (IGNORED_ENTITY_CLASSES.contains(key.getKind())) {

core/src/main/java/google/registry/persistence/transaction/Transaction.java

@@ -242,7 +242,7 @@ public class Transaction extends ImmutableObject implements Buildable {
       if (entity instanceof DatastoreEntity) {
         ((DatastoreEntity) entity).beforeDatastoreSaveOnReplay();
       }
-      ofyTm().putIgnoringReadOnly(entity);
+      ofyTm().putIgnoringReadOnlyWithBackup(entity);
     }
 
     @Override
@@ -280,7 +280,7 @@ public class Transaction extends ImmutableObject implements Buildable {
 
     @Override
     public void writeToDatastore() {
-      ofyTm().deleteIgnoringReadOnly(key);
+      ofyTm().deleteIgnoringReadOnlyWithBackup(key);
     }
 
     @Override

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -312,9 +312,9 @@ public interface TransactionManager {
   /** Returns true if the transaction manager is DatastoreTransactionManager, false otherwise. */
   boolean isOfy();
 
-  /** Performs the given write ignoring any read-only restrictions, for use only in replay. */
-  void putIgnoringReadOnly(Object entity);
+  /** Performs the write ignoring any read-only restrictions or backup, for use only in replay. */
+  void putIgnoringReadOnlyWithoutBackup(Object entity);
 
-  /** Performs the given delete ignoring any read-only restrictions, for use only in replay. */
-  void deleteIgnoringReadOnly(VKey<?> key);
+  /** Performs the delete ignoring any read-only restrictions or backup, for use only in replay. */
+  void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key);
 }

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -157,7 +157,7 @@ public class TransactionManagerFactory {
 
   /** Registry is currently undergoing maintenance and is in read-only mode. */
   public static class ReadOnlyModeException extends IllegalStateException {
-    ReadOnlyModeException() {
+    public ReadOnlyModeException() {
       super("Registry is currently undergoing maintenance and is in read-only mode");
     }
   }

core/src/main/java/google/registry/rde/RdeModule.java

@@ -45,6 +45,7 @@ public abstract class RdeModule {
   public static final String PARAM_WATERMARK = "watermark";
   public static final String PARAM_WATERMARKS = "watermarks";
   public static final String PARAM_MANUAL = "manual";
+  public static final String PARAM_BEAM = "beam";
   public static final String PARAM_DIRECTORY = "directory";
   public static final String PARAM_MODE = "mode";
   public static final String PARAM_REVISION = "revision";
@@ -72,6 +73,12 @@ public abstract class RdeModule {
     return extractBooleanParameter(req, PARAM_MANUAL);
   }
 
+  @Provides
+  @Parameter(PARAM_BEAM)
+  static boolean provideBeam(HttpServletRequest req) {
+    return extractBooleanParameter(req, PARAM_BEAM);
+  }
+
   @Provides
   @Parameter(PARAM_DIRECTORY)
   static Optional<String> provideDirectory(HttpServletRequest req) {

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -264,6 +264,11 @@ public final class RdeStagingAction implements Runnable {
   @Inject @Config("beamStagingBucketUrl") String stagingBucketUrl;
   @Inject @Config("rdeBucket") String rdeBucket;
   @Inject @Parameter(RdeModule.PARAM_MANUAL) boolean manual;
+
+  @Inject
+  @Parameter(RdeModule.PARAM_BEAM)
+  boolean beam;
+
   @Inject @Parameter(RdeModule.PARAM_DIRECTORY) Optional<String> directory;
   @Inject @Parameter(RdeModule.PARAM_MODE) ImmutableSet<String> modeStrings;
   @Inject @Parameter(RequestParameters.PARAM_TLDS) ImmutableSet<String> tlds;
@@ -289,7 +294,7 @@ public final class RdeStagingAction implements Runnable {
       logger.atInfo().log("Pending deposit: %s", pending);
     }
     ValidationMode validationMode = lenient ? LENIENT : STRICT;
-    if (tm().isOfy()) {
+    if (tm().isOfy() && !beam) {
       RdeStagingMapper mapper = new RdeStagingMapper(validationMode, pendings);
       RdeStagingReducer reducer = reducerFactory.create(validationMode, gcsUtils);
       mrRunner
@@ -382,6 +387,9 @@ public final class RdeStagingAction implements Runnable {
     if (revision.isPresent()) {
       throw new BadRequestException("Revision parameter not allowed in standard operation");
     }
+    if (beam) {
+      throw new BadRequestException("Beam parameter not allowed in standard operation");
+    }
 
     return ImmutableSetMultimap.copyOf(
         Multimaps.filterValues(

core/src/main/java/google/registry/request/RequestMetrics.java

@@ -67,10 +67,16 @@ class RequestMetrics {
   private static String truncatePath(String path) {
     // We want to bucket RDAP requests by type to use less metric space,
     // e.g. "/rdap/domains" rather than "/rdap/domains/foo.tld"
+
     if (path.startsWith("/rdap")) {
       List<String> splitPath = Splitter.on("/").omitEmptyStrings().splitToList(path);
       return Streams.stream(Iterables.limit(splitPath, 2))
           .collect(Collectors.joining("/", "/", "/"));
+      // Similarly, we put all web WHOIS requests under the same path because otherwise its
+      // cardinality is unbound, and it is possible to generate a huge amount of metrics with all
+      // the different paths.
+    } else if (path.startsWith("/whois")) {
+      return "/whois";
     }
     return path;
   }

core/src/main/java/google/registry/tools/EnqueuePollMessageCommand.java

@@ -0,0 +1,126 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.model.EppResourceUtils.loadByForeignKey;
+import static google.registry.model.reporting.HistoryEntry.Type.SYNTHETIC;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.CollectionUtils.isNullOrEmpty;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Streams;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.registrar.Registrar;
+import google.registry.model.reporting.HistoryEntry;
+import java.util.List;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Tool to enqueue a poll message for a registrar.
+ *
+ * <p>The poll message in question must correspond to an existing domain owing to schema
+ * limitations, but does not necessarily need to correspond to the owner of that domain if an
+ * alternative <code>clientId</code> is provided.
+ *
+ * <p>If general broadcast messages are being sent to registrars (e.g. a notice of an upcoming
+ * maintenance window), then it is recommended to use a well-known registry-owned domain for the
+ * enqueueing of these poll messages, such as the nic domain.
+ */
+@Parameters(separators = " =", commandDescription = "Enqueue a poll message for a domain")
+class EnqueuePollMessageCommand extends MutatingCommand {
+
+  @Parameter(
+      names = {"-m", "--message"},
+      description = "The poll message to enqueue",
+      required = true)
+  String message;
+
+  @Parameter(
+      names = {"-d", "--domain"},
+      description = "The domain name to enqueue the poll message for",
+      required = true)
+  String domainName;
+
+  @Parameter(
+      names = {"-c", "--clients"},
+      description =
+          "Comma-delimited list of the client identifier(s) of the registrar(s) to send the poll"
+              + " message to, if not the owning registrar of the domain")
+  List<String> clientIds;
+
+  @Parameter(
+      names = {"-a", "--all"},
+      description = "Whether to send the message to all real registrars",
+      arity = 1)
+  boolean sendToAll;
+
+  @Inject
+  @Config("registryAdminClientId")
+  String registryAdminClientId;
+
+  @Override
+  protected final void init() {
+    checkArgument(
+        !sendToAll || isNullOrEmpty(clientIds), "Cannot specify both --all and --clients");
+    tm().transact(
+            () -> {
+              Optional<DomainBase> domainOpt =
+                  loadByForeignKey(DomainBase.class, domainName, tm().getTransactionTime());
+              checkArgument(
+                  domainOpt.isPresent(), "Domain %s doesn't exist or isn't active", domainName);
+              DomainBase domain = domainOpt.get();
+              ImmutableList<String> registrarIds;
+              if (sendToAll) {
+                registrarIds =
+                    Streams.stream(Registrar.loadAllCached())
+                        .filter(r -> r.isLive() && r.getType() == Registrar.Type.REAL)
+                        .map(Registrar::getRegistrarId)
+                        .collect(ImmutableList.toImmutableList());
+              } else if (!isNullOrEmpty(clientIds)) {
+                registrarIds = ImmutableList.copyOf(clientIds);
+              } else {
+                registrarIds = ImmutableList.of(domain.getCurrentSponsorRegistrarId());
+              }
+              HistoryEntry historyEntry =
+                  new DomainHistory.Builder()
+                      .setDomain(domain)
+                      .setType(SYNTHETIC)
+                      .setBySuperuser(true)
+                      .setReason("Manual enqueueing of poll message: " + message)
+                      .setModificationTime(tm().getTransactionTime())
+                      .setRequestedByRegistrar(false)
+                      .setRegistrarId(registryAdminClientId)
+                      .build();
+              stageEntityChange(null, historyEntry);
+              for (String registrarId : registrarIds) {
+                stageEntityChange(
+                    null,
+                    new PollMessage.OneTime.Builder()
+                        .setRegistrarId(registrarId)
+                        .setParent(historyEntry)
+                        .setEventTime(tm().getTransactionTime())
+                        .setMsg(message)
+                        .build());
+              }
+            });
+  }
+}

core/src/main/java/google/registry/tools/GenerateEscrowDepositCommand.java

@@ -16,7 +16,9 @@ package google.registry.tools;
 
 import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static google.registry.model.tld.Registries.assertTldsExist;
+import static google.registry.rde.RdeModule.PARAM_BEAM;
 import static google.registry.rde.RdeModule.PARAM_DIRECTORY;
+import static google.registry.rde.RdeModule.PARAM_LENIENT;
 import static google.registry.rde.RdeModule.PARAM_MANUAL;
 import static google.registry.rde.RdeModule.PARAM_MODE;
 import static google.registry.rde.RdeModule.PARAM_REVISION;
@@ -65,6 +67,20 @@ final class GenerateEscrowDepositCommand implements CommandWithRemoteApi {
       description = "Mode of operation: FULL for RDE deposits, THIN for BRDA deposits.")
   private RdeMode mode = RdeMode.FULL;
 
+  @Parameter(
+      names = {"-l", "--lenient"},
+      description =
+          "Whether to run RDE in LENIENT mode, which omits validation of the generated "
+              + "XML deposit files.")
+  private boolean lenient = false;
+
+  @Parameter(
+      names = {"-b", "--beam"},
+      description =
+          "Whether to explicitly launch the beam pipeline instead of letting the action decide"
+              + " which one to run.")
+  private boolean beam = false;
+
   @Parameter(
       names = {"-r", "--revision"},
       description = "Revision number. Use >0 for resends.")
@@ -119,6 +135,8 @@ final class GenerateEscrowDepositCommand implements CommandWithRemoteApi {
             .param(PARAM_MANUAL, String.valueOf(true))
             .param(PARAM_MODE, mode.toString())
             .param(PARAM_DIRECTORY, outdir)
+            .param(PARAM_LENIENT, Boolean.toString(lenient))
+            .param(PARAM_BEAM, Boolean.toString(beam))
             .param(PARAM_TLDS, tlds.stream().collect(Collectors.joining(",")))
             .param(
                 PARAM_WATERMARKS,

core/src/main/java/google/registry/tools/GetEppResourceCommand.java

@@ -19,7 +19,6 @@ import static org.joda.time.DateTimeZone.UTC;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import java.util.Optional;
 import org.joda.time.DateTime;
@@ -56,7 +55,7 @@ abstract class GetEppResourceCommand implements CommandWithRemoteApi {
             ? String.format(
                 "%s\n\nWebsafe key: %s",
                 expand ? resource.get().toHydratedString() : resource.get(),
-                Key.create(resource.get()).getString())
+                resource.get().createVKey().getOfyKey().getString())
             : String.format("%s '%s' does not exist or is deleted\n", resourceType, uniqueId));
   }
 

core/src/main/java/google/registry/tools/GetResourceByKeyCommand.java

@@ -19,8 +19,8 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
+import google.registry.persistence.VKey;
 import java.util.List;
 
 /**
@@ -40,14 +40,14 @@ final class GetResourceByKeyCommand implements CommandWithRemoteApi {
   boolean expand;
 
   @Override
-  public void run() {
+  public void run() throws Exception {
     for (String keyString : mainParameters) {
       System.out.println("\n");
-      Key<EppResource> resourceKey =
-          checkNotNull(Key.create(keyString), "Could not parse key string: " + keyString);
+      VKey<EppResource> resourceKey =
+          checkNotNull(VKey.create(keyString), "Could not parse key string: " + keyString);
       EppResource resource =
           checkNotNull(
-              auditedOfy().load().key(resourceKey).now(),
+              auditedOfy().load().key(resourceKey.getOfyKey()).now(),
               "Could not load resource for key: " + resourceKey);
       System.out.println(expand ? resource.toHydratedString() : resource.toString());
     }

core/src/main/java/google/registry/tools/MutatingCommand.java

@@ -103,7 +103,8 @@ public abstract class MutatingCommand extends ConfirmingCommand implements Comma
      * workaround to handle cases when a SqlEntity instance does not have a primary key before being
      * persisted.
      */
-    private EntityChange(ImmutableObject oldEntity, ImmutableObject newEntity, VKey<?> vkey) {
+    private EntityChange(
+        @Nullable ImmutableObject oldEntity, @Nullable ImmutableObject newEntity, VKey<?> vkey) {
       type = ChangeType.get(oldEntity != null, newEntity != null);
       if (type == ChangeType.UPDATE) {
         checkArgument(
@@ -127,7 +128,7 @@ public abstract class MutatingCommand extends ConfirmingCommand implements Comma
     }
 
     /** Returns a human-readable ID string for the entity being changed. */
-    public String getEntityId() {
+    String getEntityId() {
       return String.format(
           "%s@%s",
           key.getOfyKey().getKind(),

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -62,6 +62,7 @@ public final class RegistryTool {
           .put("delete_reserved_list", DeleteReservedListCommand.class)
           .put("delete_tld", DeleteTldCommand.class)
           .put("encrypt_escrow_deposit", EncryptEscrowDepositCommand.class)
+          .put("enqueue_poll_message", EnqueuePollMessageCommand.class)
           .put("execute_epp", ExecuteEppCommand.class)
           .put("generate_allocation_tokens", GenerateAllocationTokensCommand.class)
           .put("generate_dns_report", GenerateDnsReportCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -111,6 +111,8 @@ interface RegistryToolComponent {
 
   void inject(EncryptEscrowDepositCommand command);
 
+  void inject(EnqueuePollMessageCommand command);
+
   void inject(GenerateAllocationTokensCommand command);
 
   void inject(GenerateDnsReportCommand command);

core/src/main/java/google/registry/tools/ResaveEntitiesCommand.java

@@ -19,8 +19,8 @@ import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
-import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
 import java.util.List;
 
 /**
@@ -37,15 +37,16 @@ public final class ResaveEntitiesCommand extends MutatingCommand {
   /** The number of resaves to do in a single transaction. */
   private static final int BATCH_SIZE = 10;
 
+  // TODO(b/207376744): figure out if there's a guide that shows how a websafe key should look like
   @Parameter(description = "Websafe keys", required = true)
   List<String> mainParameters;
 
   @Override
-  protected void init() {
+  protected void init() throws Exception {
     for (List<String> batch : partition(mainParameters, BATCH_SIZE)) {
       for (String websafeKey : batch) {
         ImmutableObject entity =
-            auditedOfy().load().key(Key.<ImmutableObject>create(websafeKey)).now();
+            (ImmutableObject) auditedOfy().load().key(VKey.create(websafeKey).getOfyKey()).now();
         stageEntityChange(entity, entity);
       }
       flushTransaction();

core/src/main/resources/META-INF/orm.xml

@@ -4,13 +4,6 @@
     xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/persistence/orm
           http://xmlns.jcp.org/xml/ns/persistence/orm_2_2.xsd"
     version="2.2">
-  <embeddable class="org.joda.money.Money" access="FIELD" />
-  <embeddable class="org.joda.money.BigMoney" access="FIELD">
-    <attributes>
-      <basic name="amount" access="FIELD"/>
-    </attributes>
-  </embeddable>
-
   <persistence-unit-metadata>
     <persistence-unit-defaults>
       <entity-listeners>

core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java

@@ -48,6 +48,7 @@ import com.google.common.truth.Truth8;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.googlecode.objectify.Key;
 import google.registry.gcs.GcsUtils;
+import google.registry.model.common.ClassPathManager;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
 import google.registry.model.contact.ContactResource;
@@ -67,7 +68,6 @@ import google.registry.model.replay.SqlReplayCheckpoint;
 import google.registry.model.server.Lock;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.PremiumList.PremiumEntry;
-import google.registry.model.translators.VKeyTranslatorFactory;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -130,7 +130,7 @@ public class ReplayCommitLogsToSqlActionTest {
 
   @BeforeAll
   static void beforeAll() {
-    VKeyTranslatorFactory.addTestEntityClass(TestObject.class);
+    ClassPathManager.addTestEntityClass(TestObject.class);
   }
 
   @BeforeEach
@@ -382,10 +382,10 @@ public class ReplayCommitLogsToSqlActionTest {
     //    even though the domain came first in the file
     // 2. that the allocation token delete occurred after the insertions
     InOrder inOrder = Mockito.inOrder(spy);
-    inOrder.verify(spy).putIgnoringReadOnly(any(ContactResource.class));
-    inOrder.verify(spy).putIgnoringReadOnly(any(DomainBase.class));
-    inOrder.verify(spy).deleteIgnoringReadOnly(toDelete.createVKey());
-    inOrder.verify(spy).putIgnoringReadOnly(any(SqlReplayCheckpoint.class));
+    inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(any(ContactResource.class));
+    inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(any(DomainBase.class));
+    inOrder.verify(spy).deleteIgnoringReadOnlyWithoutBackup(toDelete.createVKey());
+    inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(any(SqlReplayCheckpoint.class));
   }
 
   @Test
@@ -424,8 +424,8 @@ public class ReplayCommitLogsToSqlActionTest {
     // deletes have higher weight
     ArgumentCaptor<Object> putCaptor = ArgumentCaptor.forClass(Object.class);
     InOrder inOrder = Mockito.inOrder(spy);
-    inOrder.verify(spy).deleteIgnoringReadOnly(contact.createVKey());
-    inOrder.verify(spy).putIgnoringReadOnly(putCaptor.capture());
+    inOrder.verify(spy).deleteIgnoringReadOnlyWithoutBackup(contact.createVKey());
+    inOrder.verify(spy).putIgnoringReadOnlyWithoutBackup(putCaptor.capture());
     assertThat(putCaptor.getValue().getClass()).isEqualTo(ContactResource.class);
     assertThat(jpaTm().transact(() -> jpaTm().loadByKey(contact.createVKey()).getEmailAddress()))
         .isEqualTo("replay@example.tld");
@@ -467,9 +467,9 @@ public class ReplayCommitLogsToSqlActionTest {
             });
     runAndAssertSuccess(now.minusMinutes(1), 1, 1);
     // jpaTm()::putIgnoringReadOnly should only have been called with the checkpoint and the lock
-    verify(spy, times(2)).putIgnoringReadOnly(any(SqlReplayCheckpoint.class));
-    verify(spy).putIgnoringReadOnly(any(Lock.class));
-    verify(spy, times(3)).putIgnoringReadOnly(any());
+    verify(spy, times(2)).putIgnoringReadOnlyWithoutBackup(any(SqlReplayCheckpoint.class));
+    verify(spy).putIgnoringReadOnlyWithoutBackup(any(Lock.class));
+    verify(spy, times(3)).putIgnoringReadOnlyWithoutBackup(any());
   }
 
   @Test