Add a beforeSqlSave callback to ReplaySpecializer (#1062)

* Add a beforeSqlSave callback to ReplaySpecializer

When in the Datastore-primary and SQL-secondary stage, we will want to
save the EppResource-at-this-point-in-time field in the *History
objects so that later on we can examine the *History objects to see what
the resource looked like at that point in time.

Without this PR, the full object at that point in time would be lost
during the asynchronous replay since Datastore doesn't know about it.

In addition, we modify the HistoryEntry weight / priority so that
additions to it come after the additions to the resource off of which it
is based. As a result, we need to DEFER some foreign keys so that we can
write the billing / poll message objects before the history object that
they're referencing.

buildSrc/src/main/java/google/registry/gradle/plugin/ProjectData.java

@@ -91,7 +91,7 @@ abstract class ProjectData {
       /** The task was actually run and has finished successfully. */
       SUCCESS,
       /** The task was up-to-date and successful, and hence didn't need to run again. */
-      UP_TO_DATE;
+      UP_TO_DATE
     }
 
     abstract String uniqueName();

config/presubmits.py

@@ -196,7 +196,7 @@ PRESUBMITS = {
         #    - concatenation of literals: (\s*\+\s*"([^"]|\\")*")*
         # Line 3: , or the closing parenthesis, marking the end of the first
         #    parameter
-        r'.*\.create(Native)?Query\('
+        r'.*\.(query|createQuery|createNativeQuery)\('
         r'(?!(\s*([A-Z_]+|"([^"]|\\")*"(\s*\+\s*"([^"]|\\")*")*)'
         r'(,|\s*\))))',
         "java",
@@ -206,10 +206,12 @@ PRESUBMITS = {
          # using Criteria
          "ForeignKeyIndex.java",
          "HistoryEntryDao.java",
+         "JpaTransactionManager.java",
          "JpaTransactionManagerImpl.java",
          # CriteriaQueryBuilder is a false positive
          "CriteriaQueryBuilder.java",
          "RdapDomainSearchAction.java",
+         "RdapNameserverSearchAction.java",
          "RdapSearchActionBase.java",
          },
     ):

core/build.gradle

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import java.lang.reflect.Constructor
+import com.google.common.base.CaseFormat
 import java.util.Optional
 
 plugins {
@@ -423,7 +423,7 @@ task jaxbToJava {
       }
     }
     execInBash(
-        'find . -name *.java -exec sed -i /\\*\\ \\<p\\>\\$/d {} +',
+        "find . -name *.java -exec sed -i -e '/" + /\* <p>$/ + "/d' {} +",
         generatedDir)
   }
 }
@@ -751,7 +751,8 @@ project.tasks.create('initSqlPipeline', JavaExec) {
 // nom_build :core:bulkDeleteDatastore --args="--project=domain-registry-crash \
 //     --region=us-central1 --runner=DataflowRunner --kindsToDelete=*"
 createToolTask(
-    'bulkDeleteDatastore', 'google.registry.beam.datastore.BulkDeletePipeline')
+    'bulkDeleteDatastore',
+    'google.registry.beam.datastore.BulkDeleteDatastorePipeline')
 
 project.tasks.create('generateSqlSchema', JavaExec) {
   classpath = sourceSets.nonprod.runtimeClasspath
@@ -782,6 +783,57 @@ generateGoldenImages.finalizedBy(findGoldenImages)
 
 createUberJar('nomulus', 'nomulus', 'google.registry.tools.RegistryTool')
 
+// Build the Uber jar shared by all flex-template based BEAM pipelines.
+// This packages more code and dependency than necessary. However, without
+// restructuring the source tree it is difficult to generate leaner jars.
+createUberJar(
+    'beam_pipeline_common',
+    'beam_pipeline_common',
+    '')
+
+// Create beam staging task if environment is alpha or crash.
+// All other environments use formally released pipelines through CloudBuild.
+//
+// User should install gcloud and login to GCP before invoking this tasks.
+if (environment in ['alpha', 'crash']) {
+  def pipelines = [
+      [
+          mainClass: 'google.registry.beam.initsql.InitSqlPipeline',
+          metaData: 'google/registry/beam/init_sql_pipeline_metadata.json'
+      ],
+      [
+          mainClass: 'google.registry.beam.datastore.BulkDeleteDatastorePipeline',
+          metaData: 'google/registry/beam/bulk_delete_datastore_pipeline_metadata.json'
+      ],
+  ]
+  project.tasks.create("stage_beam_pipelines") {
+    doLast {
+      pipelines.each {
+        def mainClass = it['mainClass']
+        def metaData = it['metaData']
+        def pipelineName = CaseFormat.UPPER_CAMEL.to(
+                CaseFormat.LOWER_UNDERSCORE,
+                mainClass.substring(mainClass.lastIndexOf('.') + 1))
+        def imageName = "gcr.io/${gcpProject}/beam/${pipelineName}"
+        def metaDataBaseName = metaData.substring(metaData.lastIndexOf('/') + 1)
+        def uberJarName = tasks.beam_pipeline_common.outputs.files.asPath
+
+        def command = "\
+            gcloud dataflow flex-template build \
+            gs://${gcpProject}-deploy/live/beam/${metaDataBaseName} \
+            --image-gcr-path ${imageName}:live \
+            --sdk-language JAVA \
+            --flex-template-base-image JAVA11 \
+            --metadata-file ${projectDir}/src/main/resources/${metaData} \
+            --jar ${uberJarName} \
+            --env FLEX_TEMPLATE_JAVA_MAIN_CLASS=${mainClass} \
+            --project ${gcpProject}".toString()
+        rootProject.ext.execInBash(command, '/tmp')
+      }
+    }
+  }.dependsOn(tasks.beam_pipeline_common)
+}
+
 // A jar with classes and resources from main sourceSet, excluding internal
 // data. See comments on configurations.nomulus_test above for details.
 // TODO(weiminyu): release process should build this using the public repo to eliminate the need

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -154,7 +154,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
     Object ofyPojo = ofy().toPojo(entity);
     if (ofyPojo instanceof DatastoreEntity) {
       DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
-      datastoreEntity.toSqlEntity().ifPresent(jpaTm()::put);
+      datastoreEntity
+          .toSqlEntity()
+          .ifPresent(
+              sqlEntity -> {
+                ReplaySpecializer.beforeSqlSave(sqlEntity);
+                jpaTm().put(sqlEntity);
+              });
     } else {
       // this should never happen, but we shouldn't fail on it
       logger.atSevere().log(

core/src/main/java/google/registry/batch/AsyncTaskEnqueuer.java

@@ -57,8 +57,6 @@ public final class AsyncTaskEnqueuer {
   public static final String QUEUE_ASYNC_DELETE = "async-delete-pull";
   public static final String QUEUE_ASYNC_HOST_RENAME = "async-host-rename-pull";
 
-  public static final String PATH_RESAVE_ENTITY = "/_dr/task/resaveEntity";
-
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final Duration MAX_ASYNC_ETA = Duration.standardDays(30);
 
@@ -112,7 +110,7 @@ public final class AsyncTaskEnqueuer {
     logger.atInfo().log("Enqueuing async re-save of %s to run at %s.", entityKey, whenToResave);
     String backendHostname = appEngineServiceUtils.getServiceHostname("backend");
     TaskOptions task =
-        TaskOptions.Builder.withUrl(PATH_RESAVE_ENTITY)
+        TaskOptions.Builder.withUrl(ResaveEntityAction.PATH)
             .method(Method.POST)
             .header("Host", backendHostname)
             .countdownMillis(etaDuration.getMillis())

core/src/main/java/google/registry/batch/DeleteExpiredDomainsAction.java

@@ -37,13 +37,13 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.ProtocolDefinition;
 import google.registry.model.eppoutput.EppOutput;
 import google.registry.request.Action;
-import google.registry.request.Action.Method;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import google.registry.request.lock.LockHandler;
 import google.registry.util.Clock;
 import java.util.Optional;
 import java.util.concurrent.Callable;
+import java.util.logging.Level;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
@@ -59,12 +59,17 @@ import org.joda.time.Duration;
  * <p>Note that this action works by running a superuser EPP domain delete command, and as a side
  * effect of when domains are deleted (just past their expiration date), they are invariably in the
  * autorenew grace period when this happens.
+ *
+ * <p>Note also that the delete flow may fail in the uncommon case that a non-autorenewing domain
+ * has a subordinate host. It is not trivial to handle this case automatically (as said host may be
+ * in use by other domains), nor is it possible to take the correct action without exercising some
+ * human judgment. Accordingly, such deletes will fail with SEVERE-level log messages every day when
+ * this action runs, thus alerting us that human action is needed to correctly process the delete.
  */
 @Action(
     service = Action.Service.BACKEND,
     path = DeleteExpiredDomainsAction.PATH,
-    auth = Auth.AUTH_INTERNAL_OR_ADMIN,
-    method = Method.POST)
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public class DeleteExpiredDomainsAction implements Runnable {
 
   public static final String PATH = "/_dr/task/deleteExpiredDomains";
@@ -141,12 +146,23 @@ public class DeleteExpiredDomainsAction implements Runnable {
         String.join(
             ", ",
             domainsToDelete.stream().map(DomainBase::getDomainName).collect(toImmutableList())));
-    domainsToDelete.forEach(this::runDomainDeleteFlow);
-    logger.atInfo().log("Finished deleting domains.");
-    response.setPayload("Finished deleting domains.");
+    int successes = 0;
+    for (DomainBase domain : domainsToDelete) {
+      if (runDomainDeleteFlow(domain)) {
+        successes++;
+      }
+    }
+    int failures = domainsToDelete.size() - successes;
+    String msg =
+        String.format(
+            "Finished; %d domains were successfully deleted and %d errored out.",
+            successes, failures);
+    logger.at(failures == 0 ? Level.INFO : Level.SEVERE).log(msg);
+    response.setPayload(msg);
   }
 
-  private void runDomainDeleteFlow(DomainBase domain) {
+  /** Runs the actual domain delete flow and returns whether the deletion was successful. */
+  private boolean runDomainDeleteFlow(DomainBase domain) {
     logger.atInfo().log("Attempting to delete domain %s", domain.getDomainName());
     // Create a new transaction that the flow's execution will be enlisted in that loads the domain
     // transactionally. This way we can ensure that nothing else has modified the domain in question
@@ -185,10 +201,11 @@ public class DeleteExpiredDomainsAction implements Runnable {
       if (eppOutput.get().isSuccess()) {
         logger.atInfo().log("Successfully deleted domain %s", domain.getDomainName());
       } else {
-        logger.atWarning().log(
+        logger.atSevere().log(
             "Failed to delete domain %s; EPP response:\n\n%s",
             domain.getDomainName(), new String(marshalWithLenientRetry(eppOutput.get()), UTF_8));
       }
     }
+    return eppOutput.map(EppOutput::isSuccess).orElse(false);
   }
 }

core/src/main/java/google/registry/batch/ExpandRecurringBillingEventsAction.java

@@ -25,8 +25,6 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_AUTORENEW;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
-import static google.registry.schema.cursor.Cursor.GLOBAL;
-import static google.registry.schema.cursor.CursorDao.loadAndCompare;
 import static google.registry.util.CollectionUtils.union;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.earliestOf;
@@ -60,7 +58,6 @@ import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
-import google.registry.schema.cursor.CursorDao;
 import google.registry.util.Clock;
 import java.util.Optional;
 import java.util.Set;
@@ -95,7 +92,6 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
   @Override
   public void run() {
     Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
-    loadAndCompare(cursor, GLOBAL);
     DateTime executeTime = clock.nowUtc();
     DateTime persistedCursorTime = (cursor == null ? START_OF_TIME : cursor.getCursorTime());
     DateTime cursorTime = cursorTimeParam.orElse(persistedCursorTime);
@@ -332,7 +328,6 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
       tm().transact(
               () -> {
                 Cursor cursor = ofy().load().key(Cursor.createGlobalKey(RECURRING_BILLING)).now();
-                loadAndCompare(cursor, GLOBAL);
                 DateTime currentCursorTime =
                     (cursor == null ? START_OF_TIME : cursor.getCursorTime());
                 if (!currentCursorTime.equals(expectedPersistedCursorTime)) {
@@ -342,8 +337,7 @@ public class ExpandRecurringBillingEventsAction implements Runnable {
                   return;
                 }
                 if (!isDryRun) {
-                  CursorDao.saveCursor(
-                      Cursor.createGlobal(RECURRING_BILLING, executionTime), GLOBAL);
+                  tm().put(Cursor.createGlobal(RECURRING_BILLING, executionTime));
                 }
               });
     }

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -25,7 +25,7 @@ import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
 import static google.registry.model.EppResourceUtils.isActive;
 import static google.registry.model.EppResourceUtils.isDeleted;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.latestOf;
 import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.SECONDS;
@@ -44,7 +44,6 @@ import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.batch.AsyncTaskMetrics.OperationResult;
 import google.registry.dns.DnsQueue;
 import google.registry.mapreduce.MapreduceRunner;
@@ -64,6 +63,7 @@ import google.registry.util.SystemClock;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.logging.Level;
 import javax.annotation.Nullable;
@@ -123,7 +123,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     }
 
     ImmutableList.Builder<DnsRefreshRequest> requestsBuilder = new ImmutableList.Builder<>();
-    ImmutableList.Builder<Key<HostResource>> hostKeys = new ImmutableList.Builder<>();
+    ImmutableList.Builder<VKey<HostResource>> hostKeys = new ImmutableList.Builder<>();
     final List<DnsRefreshRequest> requestsToDelete = new ArrayList<>();
 
     for (TaskHandle task : tasks) {
@@ -204,10 +204,10 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
         emit(true, true);
         return;
       }
-      Key<HostResource> referencingHostKey = null;
+      VKey<HostResource> referencingHostKey = null;
       for (DnsRefreshRequest request : refreshRequests) {
         if (isActive(domain, request.lastUpdateTime())
-            && domain.getNameservers().contains(VKey.from(request.hostKey()))) {
+            && domain.getNameservers().contains(request.hostKey())) {
           referencingHostKey = request.hostKey();
           break;
         }
@@ -293,7 +293,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
 
     private static final long serialVersionUID = 1772812852271288622L;
 
-    abstract Key<HostResource> hostKey();
+    abstract VKey<HostResource> hostKey();
+
     abstract DateTime lastUpdateTime();
     abstract DateTime requestedTime();
     abstract boolean isRefreshNeeded();
@@ -301,7 +302,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
 
     @AutoValue.Builder
     abstract static class Builder {
-      abstract Builder setHostKey(Key<HostResource> hostKey);
+      abstract Builder setHostKey(VKey<HostResource> hostKey);
+
       abstract Builder setLastUpdateTime(DateTime lastUpdateTime);
       abstract Builder setRequestedTime(DateTime requestedTime);
       abstract Builder setIsRefreshNeeded(boolean isRefreshNeeded);
@@ -314,10 +316,12 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
      */
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<HostResource> hostKey =
-          Key.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+      VKey<HostResource> hostKey =
+          VKey.fromWebsafeKey(
+              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
-          checkNotNull(ofy().load().key(hostKey).now(), "Host to refresh doesn't exist");
+          tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
+              .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));
       boolean isHostDeleted =
           isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp()));
       if (isHostDeleted) {

core/src/main/java/google/registry/batch/RelockDomainAction.java

@@ -16,7 +16,6 @@ package google.registry.batch;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
@@ -33,6 +32,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.registry.RegistryLockDao;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.Response;
@@ -125,6 +125,7 @@ public class RelockDomainAction implements Runnable {
     response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
 
     // nb: DomainLockUtils relies on the JPA transaction being the outermost transaction
+    // if we have Datastore as the primary DB (if SQL is the primary DB, it's irrelevant)
     jpaTm().transact(() -> tm().transact(this::relockDomain));
   }
 
@@ -139,12 +140,8 @@ public class RelockDomainAction implements Runnable {
                       new IllegalArgumentException(
                           String.format("Unknown revision ID %d", oldUnlockRevisionId)));
       domain =
-          ofy()
-              .load()
-              .type(DomainBase.class)
-              .id(oldLock.getRepoId())
-              .now()
-              .cloneProjectedAtTime(jpaTm().getTransactionTime());
+          tm().loadByKey(VKey.create(DomainBase.class, oldLock.getRepoId()))
+              .cloneProjectedAtTime(tm().getTransactionTime());
     } catch (Throwable t) {
       handleTransientFailure(Optional.ofNullable(oldLock), t);
       return;

core/src/main/java/google/registry/batch/ResaveEntityAction.java

@@ -17,7 +17,6 @@ package google.registry.batch;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESAVE_TIMES;
 import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
@@ -26,6 +25,7 @@ import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
 import google.registry.request.Action;
 import google.registry.request.Action.Method;
 import google.registry.request.Parameter;
@@ -74,16 +74,17 @@ public class ResaveEntityAction implements Runnable {
   public void run() {
     logger.atInfo().log(
         "Re-saving entity %s which was enqueued at %s.", resourceKey, requestedTime);
-    tm().transact(() -> {
-      ImmutableObject entity = ofy().load().key(resourceKey).now();
-      ofy().save().entity(
-          (entity instanceof EppResource)
-              ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime()) : entity
-      );
-      if (!resaveTimes.isEmpty()) {
-        asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
-      }
-    });
+    tm().transact(
+            () -> {
+              ImmutableObject entity = tm().loadByKey(VKey.from(resourceKey));
+              tm().put(
+                      (entity instanceof EppResource)
+                          ? ((EppResource) entity).cloneProjectedAtTime(tm().getTransactionTime())
+                          : entity);
+              if (!resaveTimes.isEmpty()) {
+                asyncTaskEnqueuer.enqueueAsyncResave(entity, requestedTime, resaveTimes);
+              }
+            });
     response.setPayload("Entity re-saved.");
   }
 }

core/src/main/java/google/registry/batch/WipeOutCloudSqlAction.java

@@ -0,0 +1,98 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.persistence.PersistenceModule.SchemaManagerConnection;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.util.Retrier;
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.function.Supplier;
+import javax.inject.Inject;
+import org.flywaydb.core.api.FlywayException;
+
+/**
+ * Wipes out all Cloud SQL data in a Nomulus GCP environment.
+ *
+ * <p>This class is created for the QA environment, where migration testing with production data
+ * will happen. A regularly scheduled wipeout is a prerequisite to using production data there.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/wipeOutCloudSql",
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeOutCloudSqlAction implements Runnable {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  // As a short-lived class, hardcode allowed projects here instead of using config files.
+  private static final ImmutableSet<String> ALLOWED_PROJECTS =
+      ImmutableSet.of("domain-registry-qa");
+
+  private final String projectId;
+  private final Supplier<Connection> connectionSupplier;
+  private final Response response;
+  private final Retrier retrier;
+
+  @Inject
+  WipeOutCloudSqlAction(
+      @Config("projectId") String projectId,
+      @SchemaManagerConnection Supplier<Connection> connectionSupplier,
+      Response response,
+      Retrier retrier) {
+    this.projectId = projectId;
+    this.connectionSupplier = connectionSupplier;
+    this.response = response;
+    this.retrier = retrier;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+
+    if (!ALLOWED_PROJECTS.contains(projectId)) {
+      response.setStatus(SC_FORBIDDEN);
+      response.setPayload("Wipeout is not allowed in " + projectId);
+      return;
+    }
+
+    try {
+      retrier.callWithRetry(
+          () -> {
+            try (Connection conn = connectionSupplier.get();
+                Statement statement = conn.createStatement()) {
+              statement.execute("drop owned by schema_deployer;");
+            }
+            return null;
+          },
+          e -> !(e instanceof FlywayException));
+      response.setStatus(SC_OK);
+      response.setPayload("Wiped out Cloud SQL in " + projectId);
+    } catch (RuntimeException e) {
+      logger.atSevere().withCause(e).log("Failed to wipe out Cloud SQL data.");
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload("Failed to wipe out Cloud SQL in " + projectId);
+    }
+  }
+}

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Wipes out all Cloud Datastore data in a Nomulus GCP environment.
+ *
+ * <p>This class is created for the QA environment, where migration testing with production data
+ * will happen. A regularly scheduled wipeout is a prerequisite to using production data there.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/wipeOutDatastore",
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeoutDatastoreAction implements Runnable {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
+
+  // As a short-lived class, hardcode allowed projects here instead of using config files.
+  private static final ImmutableSet<String> ALLOWED_PROJECTS =
+      ImmutableSet.of("domain-registry-qa");
+
+  private final String projectId;
+  private final String jobRegion;
+  private final Response response;
+  private final Dataflow dataflow;
+  private final String stagingBucketUrl;
+
+  @Inject
+  WipeoutDatastoreAction(
+      @Config("projectId") String projectId,
+      @Config("defaultJobRegion") String jobRegion,
+      @Config("beamStagingBucketUrl") String stagingBucketUrl,
+      Response response,
+      Dataflow dataflow) {
+    this.projectId = projectId;
+    this.jobRegion = jobRegion;
+    this.stagingBucketUrl = stagingBucketUrl;
+    this.response = response;
+    this.dataflow = dataflow;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+
+    if (!ALLOWED_PROJECTS.contains(projectId)) {
+      response.setStatus(SC_FORBIDDEN);
+      response.setPayload("Wipeout is not allowed in " + projectId);
+      return;
+    }
+
+    try {
+      LaunchFlexTemplateParameter parameters =
+          new LaunchFlexTemplateParameter()
+              // Job name must be unique and in [-a-z0-9].
+              .setJobName(
+                  "bulk-delete-datastore-"
+                      + DateTime.now(DateTimeZone.UTC).toString("yyyy-MM-dd'T'HH-mm-ss'Z'"))
+              .setContainerSpecGcsPath(
+                  String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+      LaunchFlexTemplateResponse launchResponse =
+          dataflow
+              .projects()
+              .locations()
+              .flexTemplates()
+              .launch(
+                  projectId,
+                  jobRegion,
+                  new LaunchFlexTemplateRequest().setLaunchParameter(parameters))
+              .execute();
+      response.setStatus(SC_OK);
+      response.setPayload("Launched " + launchResponse.getJob().getName());
+    } catch (Exception e) {
+      String msg = String.format("Failed to launch %s.", PIPELINE_NAME);
+      logger.atSevere().withCause(e).log(msg);
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(msg);
+    }
+  }
+}

core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java

@@ -78,7 +78,7 @@ import org.apache.beam.sdk.values.TupleTagList;
  * types in the Datastore using the {@code --numOfKindsHint} argument. If the default value for this
  * parameter is too low, performance will suffer.
  */
-public class BulkDeletePipeline {
+public class BulkDeleteDatastorePipeline {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
   // This tool is not for use in our critical projects.
@@ -89,7 +89,7 @@ public class BulkDeletePipeline {
 
   private final Pipeline pipeline;
 
-  BulkDeletePipeline(BulkDeletePipelineOptions options) {
+  BulkDeleteDatastorePipeline(BulkDeletePipelineOptions options) {
     this.options = options;
     pipeline = Pipeline.create(options);
   }
@@ -303,7 +303,7 @@ public class BulkDeletePipeline {
   public static void main(String[] args) {
     BulkDeletePipelineOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(BulkDeletePipelineOptions.class);
-    BulkDeletePipeline pipeline = new BulkDeletePipeline(options);
+    BulkDeleteDatastorePipeline pipeline = new BulkDeleteDatastorePipeline(options);
     pipeline.run();
     System.exit(0);
   }

core/src/main/java/google/registry/beam/datastore/DatastoreV1.java

@@ -505,7 +505,7 @@ public class DatastoreV1 {
       }
 
       @StartBundle
-      public void startBundle(StartBundleContext c) throws Exception {
+      public void startBundle(StartBundleContext c) {
         datastore =
             datastoreFactory.getDatastore(
                 c.getPipelineOptions(), v1Options.getProjectId(), v1Options.getLocalhost());
@@ -548,7 +548,7 @@ public class DatastoreV1 {
       }
 
       @StartBundle
-      public void startBundle(StartBundleContext c) throws Exception {
+      public void startBundle(StartBundleContext c) {
         datastore =
             datastoreFactory.getDatastore(
                 c.getPipelineOptions(), options.getProjectId(), options.getLocalhost());
@@ -556,7 +556,7 @@ public class DatastoreV1 {
       }
 
       @ProcessElement
-      public void processElement(ProcessContext c) throws Exception {
+      public void processElement(ProcessContext c) {
         Query query = c.element();
 
         // If query has a user set limit, then do not split.
@@ -626,7 +626,7 @@ public class DatastoreV1 {
       }
 
       @StartBundle
-      public void startBundle(StartBundleContext c) throws Exception {
+      public void startBundle(StartBundleContext c) {
         datastore =
             datastoreFactory.getDatastore(
                 c.getPipelineOptions(), options.getProjectId(), options.getLocalhost());

core/src/main/java/google/registry/beam/initsql/BackupPaths.java

@@ -93,7 +93,7 @@ public final class BackupPaths {
     checkArgument(!isNullOrEmpty(exportDir), "Null or empty exportDir.");
     checkArgument(!isNullOrEmpty(kind), "Null or empty kind.");
     checkArgument(shard >= 0, "Negative shard %s not allowed.", shard);
-    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, Integer.toString(shard));
+    return String.format(EXPORT_PATTERN_TEMPLATE, exportDir, kind, shard);
   }
 
   /** Returns an {@link ImmutableList} of regex patterns that match all CommitLog files. */

core/src/main/java/google/registry/bigquery/BigqueryConnection.java

@@ -300,10 +300,9 @@ public class BigqueryConnection implements AutoCloseable {
    * Initializes the BigqueryConnection object by setting up the API client and creating the default
    * dataset if it doesn't exist.
    */
-  private BigqueryConnection initialize() throws Exception {
+  private void initialize() throws Exception {
     createDatasetIfNeeded(datasetId);
     createDatasetIfNeeded(TEMP_DATASET_NAME);
-    return this;
   }
 
   /**
@@ -378,13 +377,11 @@ public class BigqueryConnection implements AutoCloseable {
 
   /**
    * Starts an asynchronous load job to populate the specified destination table with the given
-   * source URIs and source format.  Returns a ListenableFuture that holds the same destination
-   * table object on success.
+   * source URIs and source format. Returns a ListenableFuture that holds the same destination table
+   * object on success.
    */
-  public ListenableFuture<DestinationTable> load(
-      DestinationTable dest,
-      SourceFormat sourceFormat,
-      Iterable<String> sourceUris) {
+  public ListenableFuture<DestinationTable> startLoad(
+      DestinationTable dest, SourceFormat sourceFormat, Iterable<String> sourceUris) {
     Job job = new Job()
         .setConfiguration(new JobConfiguration()
             .setLoad(new JobConfigurationLoad()
@@ -400,9 +397,7 @@ public class BigqueryConnection implements AutoCloseable {
    * of the specified query, or if the table is a view, to update the view to reflect that query.
    * Returns a ListenableFuture that holds the same destination table object on success.
    */
-  public ListenableFuture<DestinationTable> query(
-      String querySql,
-      DestinationTable dest) {
+  public ListenableFuture<DestinationTable> startQuery(String querySql, DestinationTable dest) {
     if (dest.type == TableType.VIEW) {
       // Use Futures.transform() rather than calling apply() directly so that any exceptions thrown
       // by calling updateTable will be propagated on the get() call, not from here.
@@ -562,20 +557,18 @@ public class BigqueryConnection implements AutoCloseable {
     // Tracking bug for query-to-GCS support is b/13777340.
     DestinationTable tempTable = buildTemporaryTable().build();
     return transformAsync(
-        query(querySql, tempTable),
+        startQuery(querySql, tempTable),
         tempTable1 -> extractTable(tempTable1, destinationUri, destinationFormat, printHeader),
         directExecutor());
   }
 
   /** @see #runJob(Job, AbstractInputStreamContent) */
-  public Job runJob(Job job) {
+  private Job runJob(Job job) {
     return runJob(job, null);
   }
 
-  /**
-   * Launch a job, wait for it to complete, but <i>do not</i> check for errors.
-   */
-  public Job runJob(Job job, @Nullable AbstractInputStreamContent data) {
+  /** Launch a job, wait for it to complete, but <i>do not</i> check for errors. */
+  private Job runJob(Job job, @Nullable AbstractInputStreamContent data) {
     return checkJob(waitForJob(launchJob(job, data)));
   }
 

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -403,12 +403,6 @@ public final class RegistryConfig {
       return config.cloudSql.jdbcUrl;
     }
 
-    @Provides
-    @Config("cloudSqlUsername")
-    public static String providesCloudSqlUsername(RegistryConfigSettings config) {
-      return config.cloudSql.username;
-    }
-
     @Provides
     @Config("cloudSqlInstanceConnectionName")
     public static String providesCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
@@ -652,6 +646,13 @@ public final class RegistryConfig {
       return config.beam.defaultJobZone;
     }
 
+    /** Returns the GCS bucket URL with all staged BEAM flex templates. */
+    @Provides
+    @Config("beamStagingBucketUrl")
+    public static String provideBeamStagingBucketUrl(RegistryConfigSettings config) {
+      return config.beam.stagingBucketUrl;
+    }
+
     /**
      * Returns the URL of the GCS location we store jar dependencies for beam pipelines.
      *
@@ -1335,12 +1336,6 @@ public final class RegistryConfig {
       return config.registryTool.clientSecret;
     }
 
-    @Provides
-    @Config("toolsCloudSqlUsername")
-    public static String providesToolsCloudSqlUsername(RegistryConfigSettings config) {
-      return config.registryTool.username;
-    }
-
     @Provides
     @Config("rdapTos")
     public static ImmutableList<String> provideRdapTos(RegistryConfigSettings config) {

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -123,6 +123,7 @@ public class RegistryConfigSettings {
   /** Configuration for Cloud SQL. */
   public static class CloudSql {
     public String jdbcUrl;
+    // TODO(05012021): remove username field after it is removed from all yaml files.
     public String username;
     public String instanceConnectionName;
     public boolean replicateTransactions;
@@ -133,6 +134,7 @@ public class RegistryConfigSettings {
   public static class Beam {
     public String defaultJobRegion;
     public String defaultJobZone;
+    public String stagingBucketUrl;
   }
 
   /** Configuration for Cloud DNS. */
@@ -220,6 +222,7 @@ public class RegistryConfigSettings {
   public static class RegistryTool {
     public String clientId;
     public String clientSecret;
+    // TODO(05012021): remove username field after it is removed from all yaml files.
     public String username;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -207,18 +207,13 @@ hibernate:
 
   # Connection pool configurations.
   hikariConnectionTimeout: 20000
-  # We occasionally received "Connection is not available, request timed out"
-  # exception when setting minimumIdle to 0 and it turned out it is a bug (See
-  # https://github.com/brettwooldridge/HikariCP/issues/1212) in HikariCP.
-  #
-  # We tried to use a fixed size pool but ran into an issue(See b/155383029),
-  # so we need further investigation to figure out the proper size of the pool.
-  #
-  # HikariCP also recommends not setting minimumIdle for maximum performance
-  # and responsiveness to spike demands (See
-  # https://github.com/brettwooldridge/HikariCP).
-  #
-  # TODO(b/154720215): Investigate the long term fix.
+  # Cloud SQL connections are a relatively scarce resource (maximum is 1000 as
+  # of March 2021). The minimumIdle should be a small value so that machines may
+  # release connections after a demand spike. The maximumPoolSize is set to 10
+  # because that is the maximum number of concurrent requests a Nomulus server
+  # instance can handle (as limited by AppEngine for basic/manual scaling). Note
+  # that BEAM pipelines are not subject to the maximumPoolSize value defined
+  # here. See PersistenceModule.java for more information.
   hikariMinimumIdle: 1
   hikariMaximumPoolSize: 10
   hikariIdleTimeout: 300000
@@ -230,8 +225,6 @@ cloudSql:
   # If jdbcUrl in this file is moved elsewhere, be sure to move this notice
   # with it until the change is applied.
   jdbcUrl: jdbc:postgresql://localhost
-  # Username for the database user.
-  username: username
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
   # Set this to true to replicate cloud SQL transactions to datastore in the
@@ -430,6 +423,7 @@ beam:
   # The default zone to run Apache Beam (Cloud Dataflow) jobs in.
   # TODO(weiminyu): consider dropping zone config. No obvious needs for this.
   defaultJobZone: us-east1-c
+  stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:
   # The name of the active keyring, either "KMS" or "Dummy".
@@ -451,7 +445,6 @@ registryTool:
   clientId: YOUR_CLIENT_ID
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
-  username: toolusername
 
 # Configuration options for checking SSL certificates.
 sslCertificateValidation:

core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml

@@ -142,6 +142,16 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
+    <description>
+      This job runs an action that deletes domains that are past their
+      autorenew end date.
+    </description>
+    <schedule>every day 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -379,6 +379,18 @@
     <url-pattern>/_dr/task/relockDomain</url-pattern>
   </servlet-mapping>
 
+  <!-- Action to wipeout Cloud SQL data -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutCloudSql</url-pattern>
+  </servlet-mapping>
+
+  <!-- Action to wipeout Cloud Datastore data -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

core/src/main/java/google/registry/env/crash/default/WEB-INF/cron.xml

@@ -190,4 +190,14 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
+    <description>
+      This job runs an action that deletes domains that are past their
+      autorenew end date.
+    </description>
+    <schedule>every day 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
 </cronentries>

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -183,6 +183,16 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
+    <description>
+      This job runs an action that deletes domains that are past their
+      autorenew end date.
+    </description>
+    <schedule>every day 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -72,4 +72,32 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
+    <description>
+      This job runs an action that deletes domains that are past their
+      autorenew end date.
+    </description>
+    <schedule>every day 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutCloudSql]]></url>
+    <description>
+      This job runs an action that deletes all data in Cloud SQL.
+    </description>
+    <schedule>every saturday 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutDatastore]]></url>
+    <description>
+      This job runs an action that deletes all data in Cloud Datastore.
+    </description>
+    <schedule>every saturday 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
 </cronentries>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -158,6 +158,16 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
+    <description>
+      This job runs an action that deletes domains that are past their
+      autorenew end date.
+    </description>
+    <schedule>every day 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
     <description>

core/src/main/java/google/registry/export/AnnotatedEntities.java

@@ -21,12 +21,13 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Ordering;
 import com.googlecode.objectify.Key;
 import google.registry.model.EntityClasses;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.annotations.VirtualEntity;
 
 /** Constants related to export code. */
-public final class ExportConstants {
+public final class AnnotatedEntities {
 
   /** Returns the names of kinds to include in Datastore backups. */
   public static ImmutableSet<String> getBackupKinds() {
@@ -49,4 +50,13 @@ public final class ExportConstants {
         .map(Key::getKind)
         .collect(toImmutableSortedSet(Ordering.natural()));
   }
+
+  /** Returns the names of kinds that are in the cross-TLD entity group. */
+  public static ImmutableSet<String> getCrossTldKinds() {
+    return EntityClasses.ALL_CLASSES.stream()
+        .filter(hasAnnotation(InCrossTld.class))
+        .filter(hasAnnotation(VirtualEntity.class).negate())
+        .map(Key::getKind)
+        .collect(toImmutableSortedSet(Ordering.natural()));
+  }
 }

core/src/main/java/google/registry/export/BackupDatastoreAction.java

@@ -66,12 +66,13 @@ public class BackupDatastoreAction implements Runnable {
     try {
       Operation backup =
           datastoreAdmin
-              .export(RegistryConfig.getDatastoreBackupsBucket(), ExportConstants.getBackupKinds())
+              .export(
+                  RegistryConfig.getDatastoreBackupsBucket(), AnnotatedEntities.getBackupKinds())
               .execute();
 
       String backupName = backup.getName();
       // Enqueue a poll task to monitor the backup and load REPORTING-related kinds into bigquery.
-      enqueuePollTask(backupName, ExportConstants.getReportingKinds());
+      enqueuePollTask(backupName, AnnotatedEntities.getReportingKinds());
       String message =
           String.format(
               "Datastore backup started with name: %s\nSaving to %s",

core/src/main/java/google/registry/export/SyncGroupMembersAction.java

@@ -16,7 +16,6 @@ package google.registry.export;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.util.CollectionUtils.nullToEmpty;
@@ -164,7 +163,7 @@ public final class SyncGroupMembersAction implements Runnable {
         registrarsToSave.add(result.getKey().asBuilder().setContactsRequireSyncing(false).build());
       }
     }
-    tm().transactNew(() -> ofy().save().entities(registrarsToSave.build()));
+    tm().transactNew(() -> tm().updateAll(registrarsToSave.build()));
     return errors;
   }
 

core/src/main/java/google/registry/export/sheet/SyncRegistrarsSheet.java

@@ -17,7 +17,6 @@ package google.registry.export.sheet;
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.common.Cursor.CursorType.SYNC_REGISTRAR_SHEET;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.registrar.RegistrarContact.Type.ABUSE;
 import static google.registry.model.registrar.RegistrarContact.Type.ADMIN;
 import static google.registry.model.registrar.RegistrarContact.Type.BILLING;
@@ -25,8 +24,8 @@ import static google.registry.model.registrar.RegistrarContact.Type.LEGAL;
 import static google.registry.model.registrar.RegistrarContact.Type.MARKETING;
 import static google.registry.model.registrar.RegistrarContact.Type.TECH;
 import static google.registry.model.registrar.RegistrarContact.Type.WHOIS;
-import static google.registry.schema.cursor.Cursor.GLOBAL;
-import static google.registry.schema.cursor.CursorDao.loadAndCompare;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.base.Joiner;
@@ -38,10 +37,10 @@ import google.registry.model.common.Cursor;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarAddress;
 import google.registry.model.registrar.RegistrarContact;
-import google.registry.schema.cursor.CursorDao;
 import google.registry.util.Clock;
 import google.registry.util.DateTimeUtils;
 import java.io.IOException;
+import java.util.Optional;
 import java.util.function.Predicate;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
@@ -63,9 +62,10 @@ class SyncRegistrarsSheet {
    * successfully completed, as measured by a cursor.
    */
   boolean wereRegistrarsModified() {
-    Cursor cursor = ofy().load().key(Cursor.createGlobalKey(SYNC_REGISTRAR_SHEET)).now();
-    loadAndCompare(cursor, GLOBAL);
-    DateTime lastUpdateTime = (cursor == null) ? START_OF_TIME : cursor.getCursorTime();
+    Optional<Cursor> cursor =
+        transactIfJpaTm(
+            () -> tm().loadByKeyIfPresent(Cursor.createGlobalVKey(SYNC_REGISTRAR_SHEET)));
+    DateTime lastUpdateTime = !cursor.isPresent() ? START_OF_TIME : cursor.get().getCursorTime();
     for (Registrar registrar : Registrar.loadAllCached()) {
       if (DateTimeUtils.isAtOrAfter(registrar.getLastUpdateTime(), lastUpdateTime)) {
         return true;
@@ -155,9 +155,7 @@ class SyncRegistrarsSheet {
                   return builder.build();
                 })
             .collect(toImmutableList()));
-    CursorDao.saveCursor(
-        Cursor.createGlobal(SYNC_REGISTRAR_SHEET, executionTime),
-        google.registry.schema.cursor.Cursor.GLOBAL);
+    tm().transact(() -> tm().put(Cursor.createGlobal(SYNC_REGISTRAR_SHEET, executionTime)));
   }
 
   private static String convertContacts(

core/src/main/java/google/registry/flows/FlowUtils.java

@@ -15,7 +15,7 @@
 package google.registry.flows;
 
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.xml.ValidationMode.LENIENT;
 import static google.registry.xml.ValidationMode.STRICT;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -51,8 +51,8 @@ public final class FlowUtils {
 
   /** Persists the saves and deletes in an {@link EntityChanges} to Datastore. */
   public static void persistEntityChanges(EntityChanges entityChanges) {
-    ofy().save().entities(entityChanges.getSaves());
-    ofy().delete().keys(entityChanges.getDeletes());
+    tm().putAll(entityChanges.getSaves());
+    tm().delete(entityChanges.getDeletes());
   }
 
   /**

core/src/main/java/google/registry/flows/custom/EntityChanges.java

@@ -16,8 +16,8 @@ package google.registry.flows.custom;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
 
 /** A wrapper class that encapsulates Datastore entities to both save and delete. */
 @AutoValue
@@ -25,7 +25,7 @@ public abstract class EntityChanges {
 
   public abstract ImmutableSet<ImmutableObject> getSaves();
 
-  public abstract ImmutableSet<Key<ImmutableObject>> getDeletes();
+  public abstract ImmutableSet<VKey<ImmutableObject>> getDeletes();
 
   public static Builder newBuilder() {
     // Default both entities to save and entities to delete to empty sets, so that the build()
@@ -48,11 +48,11 @@ public abstract class EntityChanges {
       return this;
     }
 
-    public abstract Builder setDeletes(ImmutableSet<Key<ImmutableObject>> entitiesToDelete);
+    public abstract Builder setDeletes(ImmutableSet<VKey<ImmutableObject>> entitiesToDelete);
 
-    public abstract ImmutableSet.Builder<Key<ImmutableObject>> deletesBuilder();
+    public abstract ImmutableSet.Builder<VKey<ImmutableObject>> deletesBuilder();
 
-    public Builder addDelete(Key<ImmutableObject> entityToDelete) {
+    public Builder addDelete(VKey<ImmutableObject> entityToDelete) {
       deletesBuilder().add(entityToDelete);
       return this;
     }

core/src/main/java/google/registry/flows/domain/DomainClaimsCheckFlow.java

@@ -45,7 +45,7 @@ import google.registry.model.eppinput.ResourceCommand;
 import google.registry.model.eppoutput.EppResponse;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.model.tmch.ClaimsListShard;
+import google.registry.model.tmch.ClaimsListDualDatabaseDao;
 import google.registry.util.Clock;
 import java.util.HashSet;
 import java.util.Optional;
@@ -104,7 +104,8 @@ public final class DomainClaimsCheckFlow implements Flow {
           verifyClaimsPeriodNotEnded(registry, now);
         }
       }
-      Optional<String> claimKey = ClaimsListShard.get().getClaimKey(parsedDomain.parts().get(0));
+      Optional<String> claimKey =
+          ClaimsListDualDatabaseDao.get().getClaimKey(parsedDomain.parts().get(0));
       launchChecksBuilder.add(
           LaunchCheck.create(
               LaunchCheckName.create(claimKey.isPresent(), domainName), claimKey.orElse(null)));

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -129,7 +129,7 @@ import google.registry.model.registry.label.ReservedList;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.model.tmch.ClaimsListShard;
+import google.registry.model.tmch.ClaimsListDualDatabaseDao;
 import google.registry.persistence.VKey;
 import google.registry.tldconfig.idn.IdnLabelValidator;
 import google.registry.util.Idn;
@@ -994,7 +994,7 @@ public class DomainFlowUtils {
       InternetDomainName domainName, boolean hasSignedMarks, boolean hasClaimsNotice)
       throws EppException {
     boolean isInClaimsList =
-        ClaimsListShard.get().getClaimKey(domainName.parts().get(0)).isPresent();
+        ClaimsListDualDatabaseDao.get().getClaimKey(domainName.parts().get(0)).isPresent();
     if (hasClaimsNotice && !isInClaimsList) {
       throw new UnexpectedClaimsNoticeException(domainName.toString());
     }
@@ -1093,8 +1093,7 @@ public class DomainFlowUtils {
           .list();
     } else {
       return jpaTm()
-          .getEntityManager()
-          .createQuery(
+          .query(
               "FROM DomainHistory WHERE modificationTime >= :beginning "
                   + "ORDER BY modificationTime ASC",
               DomainHistory.class)

core/src/main/java/google/registry/flows/domain/DomainInfoFlow.java

@@ -22,6 +22,7 @@ import static google.registry.flows.domain.DomainFlowUtils.handleFeeRequest;
 import static google.registry.flows.domain.DomainFlowUtils.loadForeignKeyedDesignatedContacts;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
@@ -100,9 +101,11 @@ public final class DomainInfoFlow implements Flow {
     verifyOptionalAuthInfo(authInfo, domain);
     flowCustomLogic.afterValidation(
         AfterValidationParameters.newBuilder().setDomain(domain).build());
-    // Prefetch all referenced resources. Calling values() blocks until loading is done.
-    tm().loadByKeys(domain.getNameservers());
-    tm().loadByKeys(domain.getReferencedContacts());
+    // In ofy, refetch all referenced resources.
+    if (tm().isOfy()) {
+      tm().loadByKeys(domain.getNameservers());
+      tm().loadByKeys(domain.getReferencedContacts());
+    }
     // Registrars can only see a few fields on unauthorized domains.
     // This is a policy decision that is left up to us by the rfcs.
     DomainInfoData.Builder infoBuilder =
@@ -110,14 +113,16 @@ public final class DomainInfoFlow implements Flow {
             .setFullyQualifiedDomainName(domain.getDomainName())
             .setRepoId(domain.getRepoId())
             .setCurrentSponsorClientId(domain.getCurrentSponsorClientId())
-            .setRegistrant(tm().loadByKey(domain.getRegistrant()).getContactId());
+            .setRegistrant(
+                transactIfJpaTm(() -> tm().loadByKey(domain.getRegistrant())).getContactId());
     // If authInfo is non-null, then the caller is authorized to see the full information since we
     // will have already verified the authInfo is valid.
     if (clientId.equals(domain.getCurrentSponsorClientId()) || authInfo.isPresent()) {
       HostsRequest hostsRequest = ((Info) resourceCommand).getHostsRequest();
       infoBuilder
           .setStatusValues(domain.getStatusValues())
-          .setContacts(loadForeignKeyedDesignatedContacts(domain.getContacts()))
+          .setContacts(
+              transactIfJpaTm(() -> loadForeignKeyedDesignatedContacts(domain.getContacts())))
           .setNameservers(hostsRequest.requestDelegated() ? domain.loadNameserverHostNames() : null)
           .setSubordinateHosts(
               hostsRequest.requestSubordinate() ? domain.getSubordinateHosts() : null)

core/src/main/java/google/registry/flows/domain/DomainRestoreRequestFlow.java

@@ -233,6 +233,9 @@ public final class DomainRestoreRequestFlow implements TransactionalFlow  {
         .setDeletePollMessage(null)
         .setAutorenewBillingEvent(autorenewEvent.createVKey())
         .setAutorenewPollMessage(autorenewPollMessage.createVKey())
+        // Clear the autorenew end time so if it had expired but is now explicitly being restored,
+        // it won't immediately be deleted again.
+        .setAutorenewEndTime(Optional.empty())
         .setLastEppUpdateTime(now)
         .setLastEppUpdateClientId(clientId)
         .build();

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -27,13 +27,13 @@ import static google.registry.flows.domain.DomainFlowUtils.updateAutorenewRecurr
 import static google.registry.flows.domain.DomainTransferUtils.createGainingTransferPollMessage;
 import static google.registry.flows.domain.DomainTransferUtils.createTransferResponse;
 import static google.registry.model.ResourceTransferUtils.approvePendingTransfer;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.TRANSFER_SUCCESSFUL;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.pricing.PricingEngineProxy.getDomainRenewCost;
 import static google.registry.util.CollectionUtils.union;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
@@ -132,6 +132,8 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
                     .setBillingTime(now.plus(Registry.get(tld).getTransferGracePeriodLength()))
                     .setParent(historyEntry)
                     .build());
+    ImmutableList.Builder<ImmutableObject> entitiesToSave = new ImmutableList.Builder<>();
+    entitiesToSave.add(historyEntry);
     // If we are within an autorenew grace period, cancel the autorenew billing event and don't
     // increase the registration time, since the transfer subsumes the autorenew's extra year.
     GracePeriod autorenewGrace =
@@ -143,7 +145,7 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
       // then the gaining registrar is not charged for the one year renewal and the losing registrar
       // still needs to be charged for the auto-renew.
       if (billingEvent.isPresent()) {
-        ofy().save().entity(
+        entitiesToSave.add(
             BillingEvent.Cancellation.forGracePeriod(autorenewGrace, historyEntry, targetId));
       }
     }
@@ -190,31 +192,26 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
             .setAutorenewPollMessage(gainingClientAutorenewPollMessage.createVKey())
             // Remove all the old grace periods and add a new one for the transfer.
             .setGracePeriods(
-                billingEvent.isPresent()
-                    ? ImmutableSet.of(
-                        GracePeriod.forBillingEvent(
-                            GracePeriodStatus.TRANSFER,
-                            existingDomain.getRepoId(),
-                            billingEvent.get()))
-                    : ImmutableSet.of())
+                billingEvent
+                    .map(
+                        oneTime ->
+                            ImmutableSet.of(
+                                GracePeriod.forBillingEvent(
+                                    GracePeriodStatus.TRANSFER,
+                                    existingDomain.getRepoId(),
+                                    oneTime)))
+                    .orElseGet(ImmutableSet::of))
             .setLastEppUpdateTime(now)
             .setLastEppUpdateClientId(clientId)
             .build();
     // Create a poll message for the gaining client.
-    PollMessage gainingClientPollMessage = createGainingTransferPollMessage(
-        targetId,
-        newDomain.getTransferData(),
-        newExpirationTime,
-        historyEntry);
-    ImmutableSet.Builder<ImmutableObject> entitiesToSave = new ImmutableSet.Builder<>();
-    entitiesToSave.add(
-        newDomain,
-        historyEntry,
-        autorenewEvent,
-        gainingClientPollMessage,
-        gainingClientAutorenewPollMessage);
+    PollMessage gainingClientPollMessage =
+        createGainingTransferPollMessage(
+            targetId, newDomain.getTransferData(), newExpirationTime, historyEntry);
     billingEvent.ifPresent(entitiesToSave::add);
-    ofy().save().entities(entitiesToSave.build());
+    entitiesToSave.add(
+        autorenewEvent, gainingClientPollMessage, gainingClientAutorenewPollMessage, newDomain);
+    tm().putAll(entitiesToSave.build());
     // Delete the billing event and poll messages that were written in case the transfer would have
     // been implicitly server approved.
     tm().delete(existingDomain.getTransferData().getServerApproveEntities());

core/src/main/java/google/registry/flows/domain/DomainTransferCancelFlow.java

@@ -25,7 +25,6 @@ import static google.registry.flows.domain.DomainFlowUtils.updateAutorenewRecurr
 import static google.registry.flows.domain.DomainTransferUtils.createLosingTransferPollMessage;
 import static google.registry.flows.domain.DomainTransferUtils.createTransferResponse;
 import static google.registry.model.ResourceTransferUtils.denyPendingTransfer;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.DomainTransactionRecord.TransactionReportField.TRANSFER_SUCCESSFUL;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
@@ -39,7 +38,6 @@ import google.registry.flows.FlowModule.Superuser;
 import google.registry.flows.FlowModule.TargetId;
 import google.registry.flows.TransactionalFlow;
 import google.registry.flows.annotations.ReportingSpec;
-import google.registry.model.ImmutableObject;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.metadata.MetadataExtension;
 import google.registry.model.eppcommon.AuthInfo;
@@ -100,11 +98,11 @@ public final class DomainTransferCancelFlow implements TransactionalFlow {
     HistoryEntry historyEntry = buildHistoryEntry(existingDomain, registry, now);
     DomainBase newDomain =
         denyPendingTransfer(existingDomain, TransferStatus.CLIENT_CANCELLED, now, clientId);
-    ofy().save().<ImmutableObject>entities(
-        newDomain,
-        historyEntry,
-        createLosingTransferPollMessage(
-            targetId, newDomain.getTransferData(), null, historyEntry));
+    tm().putAll(
+            newDomain,
+            historyEntry,
+            createLosingTransferPollMessage(
+                targetId, newDomain.getTransferData(), null, historyEntry));
     // Reopen the autorenew event and poll message that we closed for the implicit transfer. This
     // may recreate the autorenew poll message if it was deleted when the transfer request was made.
     updateAutorenewRecurrenceEndTime(existingDomain, END_OF_TIME);

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -31,7 +31,6 @@ import static google.registry.flows.domain.DomainTransferUtils.createPendingTran
 import static google.registry.flows.domain.DomainTransferUtils.createTransferResponse;
 import static google.registry.flows.domain.DomainTransferUtils.createTransferServerApproveEntities;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
@@ -227,12 +226,11 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
             .setLastEppUpdateClientId(gainingClientId)
             .build();
     asyncTaskEnqueuer.enqueueAsyncResave(newDomain, now, automaticTransferTime);
-    ofy().save()
-        .entities(new ImmutableSet.Builder<>()
-            .add(newDomain, historyEntry, requestPollMessage)
-            .addAll(serverApproveEntities)
-            .build())
-        .now();
+    tm().putAll(
+            new ImmutableSet.Builder<>()
+                .add(newDomain, historyEntry, requestPollMessage)
+                .addAll(serverApproveEntities)
+                .build());
     return responseBuilder
         .setResultFromCode(SUCCESS_WITH_ACTION_PENDING)
         .setResData(createResponse(period, existingDomain, newDomain, now))

core/src/main/java/google/registry/flows/poll/PollAckFlow.java

@@ -16,15 +16,13 @@ package google.registry.flows.poll;
 
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
 import static google.registry.flows.poll.PollFlowUtils.ackPollMessage;
-import static google.registry.flows.poll.PollFlowUtils.getPollMessagesQuery;
+import static google.registry.flows.poll.PollFlowUtils.getPollMessageCount;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_NO_MESSAGES;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.poll.PollMessageExternalKeyConverter.makePollMessageExternalId;
 import static google.registry.model.poll.PollMessageExternalKeyConverter.parsePollMessageExternalId;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.EppException.AuthorizationErrorException;
 import google.registry.flows.EppException.ObjectDoesNotExistException;
@@ -39,6 +37,8 @@ import google.registry.model.poll.MessageQueueInfo;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.poll.PollMessageExternalKeyConverter;
 import google.registry.model.poll.PollMessageExternalKeyConverter.PollMessageExternalKeyParseException;
+import google.registry.persistence.VKey;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -71,7 +71,7 @@ public class PollAckFlow implements TransactionalFlow {
       throw new MissingMessageIdException();
     }
 
-    Key<PollMessage> pollMessageKey;
+    VKey<PollMessage> pollMessageKey;
     // Try parsing the messageId, and throw an exception if it's invalid.
     try {
       pollMessageKey = parsePollMessageExternalId(messageId);
@@ -84,12 +84,13 @@ public class PollAckFlow implements TransactionalFlow {
     // Load the message to be acked. If a message is queued to be delivered in the future, we treat
     // it as if it doesn't exist yet. Same for if the message ID year isn't the same as the actual
     // poll message's event time (that means they're passing in an old already-acked ID).
-    PollMessage pollMessage = ofy().load().key(pollMessageKey).now();
-    if (pollMessage == null
-        || !isBeforeOrAt(pollMessage.getEventTime(), now)
-        || !makePollMessageExternalId(pollMessage).equals(messageId)) {
+    Optional<PollMessage> maybePollMessage = tm().loadByKeyIfPresent(pollMessageKey);
+    if (!maybePollMessage.isPresent()
+        || !isBeforeOrAt(maybePollMessage.get().getEventTime(), now)
+        || !makePollMessageExternalId(maybePollMessage.get()).equals(messageId)) {
       throw new MessageDoesNotExistException(messageId);
     }
+    PollMessage pollMessage = maybePollMessage.get();
 
     // Make sure this client is authorized to ack this message. It could be that the message is
     // supposed to go to a different registrar.
@@ -106,8 +107,11 @@ public class PollAckFlow implements TransactionalFlow {
     // acked, then we return a special status code indicating that. Note that the query will
     // include the message being acked.
 
-    int messageCount = tm().doTransactionless(() -> getPollMessagesQuery(clientId, now).count());
-    if (!includeAckedMessageInCount) {
+    int messageCount = tm().doTransactionless(() -> getPollMessageCount(clientId, now));
+    // Within the same transaction, Datastore will not reflect the updated count (potentially
+    // reduced by one thanks to the acked poll message). SQL will, however, so we shouldn't reduce
+    // the count in the SQL case.
+    if (!includeAckedMessageInCount && tm().isOfy()) {
       messageCount--;
     }
     if (messageCount <= 0) {

core/src/main/java/google/registry/flows/poll/PollFlowUtils.java

@@ -16,25 +16,56 @@ package google.registry.flows.poll;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.poll.PollMessage;
+import java.util.Optional;
 import org.joda.time.DateTime;
 
 /** Static utility functions for poll flows. */
 public final class PollFlowUtils {
 
-  private PollFlowUtils() {}
+  public static final String SQL_POLL_MESSAGE_QUERY =
+      "FROM PollMessage WHERE clientId = :registrarId AND eventTime <= :now ORDER BY eventTime ASC";
+  private static final String SQL_POLL_MESSAGE_COUNT_QUERY =
+      "SELECT COUNT(*) FROM PollMessage WHERE clientId = :registrarId AND eventTime <= :now";
 
-  /** Returns a query for poll messages for the logged in registrar which are not in the future. */
-  public static Query<PollMessage> getPollMessagesQuery(String clientId, DateTime now) {
-    return ofy().load()
-        .type(PollMessage.class)
-        .filter("clientId", clientId)
-        .filter("eventTime <=", now.toDate())
-        .order("eventTime");
+  /** Returns the number of poll messages for the given registrar that are not in the future. */
+  public static int getPollMessageCount(String registrarId, DateTime now) {
+    if (tm().isOfy()) {
+      return datastorePollMessageQuery(registrarId, now).count();
+    } else {
+      return jpaTm()
+          .transact(
+              () ->
+                  jpaTm()
+                      .query(SQL_POLL_MESSAGE_COUNT_QUERY, Long.class)
+                      .setParameter("registrarId", registrarId)
+                      .setParameter("now", now)
+                      .getSingleResult()
+                      .intValue());
+    }
+  }
+
+  /** Returns the first (by event time) poll message not in the future for this registrar. */
+  public static Optional<PollMessage> getFirstPollMessage(String registrarId, DateTime now) {
+    if (tm().isOfy()) {
+      return Optional.ofNullable(datastorePollMessageQuery(registrarId, now).first().now());
+    } else {
+      return jpaTm()
+          .transact(
+              () ->
+                  jpaTm()
+                      .query(SQL_POLL_MESSAGE_QUERY, PollMessage.class)
+                      .setParameter("registrarId", registrarId)
+                      .setParameter("now", now)
+                      .setMaxResults(1)
+                      .getResultStream()
+                      .findFirst());
+    }
   }
 
   /**
@@ -74,4 +105,16 @@ public final class PollFlowUtils {
     }
     return includeAckedMessageInCount;
   }
+
+  /** A Datastore query for poll messages from the given registrar that are not in the future. */
+  public static Query<PollMessage> datastorePollMessageQuery(String registrarId, DateTime now) {
+    return ofy()
+        .load()
+        .type(PollMessage.class)
+        .filter("clientId", registrarId)
+        .filter("eventTime <=", now.toDate())
+        .order("eventTime");
+  }
+
+  private PollFlowUtils() {}
 }

core/src/main/java/google/registry/flows/poll/PollRequestFlow.java

@@ -15,7 +15,8 @@
 package google.registry.flows.poll;
 
 import static google.registry.flows.FlowUtils.validateClientIsLoggedIn;
-import static google.registry.flows.poll.PollFlowUtils.getPollMessagesQuery;
+import static google.registry.flows.poll.PollFlowUtils.getFirstPollMessage;
+import static google.registry.flows.poll.PollFlowUtils.getPollMessageCount;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACK_MESSAGE;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_NO_MESSAGES;
 import static google.registry.model.poll.PollMessageExternalKeyConverter.makePollMessageExternalId;
@@ -31,6 +32,7 @@ import google.registry.model.poll.MessageQueueInfo;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.poll.PollMessageExternalKeyConverter;
 import google.registry.util.Clock;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.joda.time.DateTime;
 
@@ -63,18 +65,20 @@ public class PollRequestFlow implements Flow {
     }
     // Return the oldest message from the queue.
     DateTime now = clock.nowUtc();
-    PollMessage pollMessage = getPollMessagesQuery(clientId, now).first().now();
-    if (pollMessage == null) {
+    Optional<PollMessage> maybePollMessage = getFirstPollMessage(clientId, now);
+    if (!maybePollMessage.isPresent()) {
       return responseBuilder.setResultFromCode(SUCCESS_WITH_NO_MESSAGES).build();
     }
+    PollMessage pollMessage = maybePollMessage.get();
     return responseBuilder
         .setResultFromCode(SUCCESS_WITH_ACK_MESSAGE)
-        .setMessageQueueInfo(new MessageQueueInfo.Builder()
-            .setQueueDate(pollMessage.getEventTime())
-            .setMsg(pollMessage.getMsg())
-            .setQueueLength(getPollMessagesQuery(clientId, now).count())
-            .setMessageId(makePollMessageExternalId(pollMessage))
-            .build())
+        .setMessageQueueInfo(
+            new MessageQueueInfo.Builder()
+                .setQueueDate(pollMessage.getEventTime())
+                .setMsg(pollMessage.getMsg())
+                .setQueueLength(getPollMessageCount(clientId, now))
+                .setMessageId(makePollMessageExternalId(pollMessage))
+                .build())
         .setMultipleResData(pollMessage.getResponseData())
         .build();
   }

core/src/main/java/google/registry/keyring/api/InMemoryKeyring.java

@@ -39,8 +39,6 @@ public final class InMemoryKeyring implements Keyring {
   private final String marksdbLordnPassword;
   private final String marksdbSmdrlLoginAndPassword;
   private final String jsonCredential;
-  private final String cloudSqlPassword;
-  private final String toolsCloudSqlPassword;
 
   public InMemoryKeyring(
       PGPKeyPair rdeStagingKey,
@@ -83,8 +81,6 @@ public final class InMemoryKeyring implements Keyring {
     this.marksdbSmdrlLoginAndPassword =
         checkNotNull(marksdbSmdrlLoginAndPassword, "marksdbSmdrlLoginAndPassword");
     this.jsonCredential = checkNotNull(jsonCredential, "jsonCredential");
-    this.cloudSqlPassword = checkNotNull(cloudSqlPassword, "cloudSqlPassword");
-    this.toolsCloudSqlPassword = checkNotNull(toolsCloudSqlPassword, "toolsCloudSqlPassword");
   }
 
   @Override
@@ -157,16 +153,6 @@ public final class InMemoryKeyring implements Keyring {
     return jsonCredential;
   }
 
-  @Override
-  public String getCloudSqlPassword() {
-    return cloudSqlPassword;
-  }
-
-  @Override
-  public String getToolsCloudSqlPassword() {
-    return toolsCloudSqlPassword;
-  }
-
   /** Does nothing. */
   @Override
   public void close() {}

core/src/main/java/google/registry/keyring/api/KeyModule.java

@@ -36,18 +36,6 @@ public final class KeyModule {
     String value();
   }
 
-  @Provides
-  @Key("cloudSqlPassword")
-  static String providesCloudSqlPassword(Keyring keyring) {
-    return keyring.getCloudSqlPassword();
-  }
-
-  @Provides
-  @Key("toolsCloudSqlPassword")
-  static String providesToolsCloudSqlPassword(Keyring keyring) {
-    return keyring.getToolsCloudSqlPassword();
-  }
-
   @Provides
   @Key("brdaReceiverKey")
   static PGPPublicKey provideBrdaReceiverKey(Keyring keyring) {

core/src/main/java/google/registry/keyring/api/Keyring.java

@@ -28,12 +28,6 @@ import org.bouncycastle.openpgp.PGPPublicKey;
 @ThreadSafe
 public interface Keyring extends AutoCloseable {
 
-  /** Returns the password which is used by App Engine to connect to the Cloud SQL database. */
-  String getCloudSqlPassword();
-
-  /** Returns the password which is used by nomulus tool to connect to the Cloud SQL database. */
-  String getToolsCloudSqlPassword();
-
   /**
    * Returns the key which should be used to sign RDE deposits being uploaded to a third-party.
    *

core/src/main/java/google/registry/keyring/kms/KmsKeyring.java

@@ -19,6 +19,7 @@ import static com.google.common.base.CaseFormat.UPPER_UNDERSCORE;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
@@ -26,7 +27,10 @@ import google.registry.keyring.api.KeySerializer;
 import google.registry.keyring.api.Keyring;
 import google.registry.keyring.api.KeyringException;
 import google.registry.model.server.KmsSecret;
+import google.registry.model.server.KmsSecretRevision;
+import google.registry.model.server.KmsSecretRevisionSqlDao;
 import java.io.IOException;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.bouncycastle.openpgp.PGPException;
 import org.bouncycastle.openpgp.PGPKeyPair;
@@ -68,7 +72,6 @@ public class KmsKeyring implements Keyring {
 
   /** Key labels for string secrets. */
   enum StringKeyLabel {
-    CLOUD_SQL_PASSWORD_STRING,
     SAFE_BROWSING_API_KEY,
     ICANN_REPORTING_PASSWORD_STRING,
     JSON_CREDENTIAL_STRING,
@@ -76,8 +79,7 @@ public class KmsKeyring implements Keyring {
     MARKSDB_LORDN_PASSWORD_STRING,
     MARKSDB_SMDRL_LOGIN_STRING,
     RDE_SSH_CLIENT_PRIVATE_STRING,
-    RDE_SSH_CLIENT_PUBLIC_STRING,
-    TOOLS_CLOUD_SQL_PASSWORD_STRING;
+    RDE_SSH_CLIENT_PUBLIC_STRING;
 
     String getLabel() {
       return UPPER_UNDERSCORE.to(LOWER_HYPHEN, name());
@@ -91,16 +93,6 @@ public class KmsKeyring implements Keyring {
     this.kmsConnection = kmsConnection;
   }
 
-  @Override
-  public String getCloudSqlPassword() {
-    return getString(StringKeyLabel.CLOUD_SQL_PASSWORD_STRING);
-  }
-
-  @Override
-  public String getToolsCloudSqlPassword() {
-    return getString(StringKeyLabel.TOOLS_CLOUD_SQL_PASSWORD_STRING);
-  }
-
   @Override
   public PGPKeyPair getRdeSigningKey() {
     return getKeyPair(PrivateKeyLabel.RDE_SIGNING_PRIVATE);
@@ -201,13 +193,21 @@ public class KmsKeyring implements Keyring {
   }
 
   private byte[] getDecryptedData(String keyName) {
-    KmsSecret secret =
-        ofy().load().key(Key.create(getCrossTldKey(), KmsSecret.class, keyName)).now();
-    checkState(secret != null, "Requested secret '%s' does not exist.", keyName);
-    String encryptedData = ofy().load().key(secret.getLatestRevision()).now().getEncryptedValue();
+    String encryptedData;
+    if (tm().isOfy()) {
+      KmsSecret secret =
+          ofy().load().key(Key.create(getCrossTldKey(), KmsSecret.class, keyName)).now();
+      checkState(secret != null, "Requested secret '%s' does not exist.", keyName);
+      encryptedData = ofy().load().key(secret.getLatestRevision()).now().getEncryptedValue();
+    } else {
+      Optional<KmsSecretRevision> revision =
+          tm().transact(() -> KmsSecretRevisionSqlDao.getLatestRevision(keyName));
+      checkState(revision.isPresent(), "Requested secret '%s' does not exist.", keyName);
+      encryptedData = revision.get().getEncryptedValue();
+    }
 
     try {
-      return kmsConnection.decrypt(secret.getName(), encryptedData);
+      return kmsConnection.decrypt(keyName, encryptedData);
     } catch (Exception e) {
       throw new KeyringException(
           String.format("CloudKMS decrypt operation failed for secret %s", keyName), e);

core/src/main/java/google/registry/keyring/kms/KmsUpdater.java

@@ -24,7 +24,6 @@ import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.BRDA_SIGNING
 import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.RDE_RECEIVER_PUBLIC;
 import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.RDE_SIGNING_PUBLIC;
 import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.RDE_STAGING_PUBLIC;
-import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.CLOUD_SQL_PASSWORD_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.ICANN_REPORTING_PASSWORD_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.JSON_CREDENTIAL_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.MARKSDB_DNL_LOGIN_STRING;
@@ -33,8 +32,6 @@ import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.MARKSDB_SMDR
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.RDE_SSH_CLIENT_PRIVATE_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.RDE_SSH_CLIENT_PUBLIC_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.SAFE_BROWSING_API_KEY;
-import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.TOOLS_CLOUD_SQL_PASSWORD_STRING;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -72,10 +69,6 @@ public final class KmsUpdater {
     this.secretValues = new LinkedHashMap<>();
   }
 
-  public KmsUpdater setCloudSqlPassword(String password) {
-    return setString(password, CLOUD_SQL_PASSWORD_STRING);
-  }
-
   public KmsUpdater setRdeSigningKey(PGPKeyPair keyPair) throws IOException, PGPException {
     return setKeyPair(keyPair, RDE_SIGNING_PRIVATE, RDE_SIGNING_PUBLIC);
   }
@@ -108,10 +101,6 @@ public final class KmsUpdater {
     return setString(apiKey, SAFE_BROWSING_API_KEY);
   }
 
-  public KmsUpdater setToolsCloudSqlPassword(String password) {
-    return setString(password, TOOLS_CLOUD_SQL_PASSWORD_STRING);
-  }
-
   public KmsUpdater setIcannReportingPassword(String password) {
     return setString(password, ICANN_REPORTING_PASSWORD_STRING);
   }
@@ -194,8 +183,7 @@ public final class KmsUpdater {
    */
   private static void persistEncryptedValues(
       final ImmutableMap<String, EncryptResponse> encryptedValues) {
-    tm()
-        .transact(
+    tm().transact(
             () -> {
               for (Map.Entry<String, EncryptResponse> entry : encryptedValues.entrySet()) {
                 String secretName = entry.getKey();
@@ -207,7 +195,7 @@ public final class KmsUpdater {
                         .setKmsCryptoKeyVersionName(revisionData.cryptoKeyVersionName())
                         .setParent(secretName)
                         .build();
-                ofy().save().entities(secretRevision, KmsSecret.create(secretName, secretRevision));
+                tm().putAll(secretRevision, KmsSecret.create(secretName, secretRevision));
               }
             });
   }

core/src/main/java/google/registry/mapreduce/inputs/CommitLogManifestReader.java

@@ -58,16 +58,16 @@ class CommitLogManifestReader
 
   @Override
   public QueryResultIterator<Key<CommitLogManifest>> getQueryIterator(@Nullable Cursor cursor) {
-    return startQueryAt(query(), cursor).keys().iterator();
+    return startQueryAt(createBucketQuery(), cursor).keys().iterator();
   }
 
   @Override
   public int getTotal() {
-    return query().count();
+    return createBucketQuery().count();
   }
 
   /** Query for children of this bucket. */
-  Query<CommitLogManifest> query() {
+  Query<CommitLogManifest> createBucketQuery() {
     Query<CommitLogManifest> query = ofy().load().type(CommitLogManifest.class).ancestor(bucketKey);
     if (olderThan != null) {
       query = query.filterKey(

core/src/main/java/google/registry/model/BackupGroupRoot.java

@@ -16,6 +16,8 @@ package google.registry.model;
 
 import javax.persistence.Access;
 import javax.persistence.AccessType;
+import javax.persistence.AttributeOverride;
+import javax.persistence.Column;
 import javax.persistence.MappedSuperclass;
 import javax.xml.bind.annotation.XmlTransient;
 
@@ -40,6 +42,7 @@ public abstract class BackupGroupRoot extends ImmutableObject {
   // Prevents subclasses from unexpectedly accessing as property (e.g., HostResource), which would
   // require an unnecessary non-private setter method.
   @Access(AccessType.FIELD)
+  @AttributeOverride(name = "lastUpdateTime", column = @Column(name = "updateTimestamp"))
   UpdateAutoTimestamp updateTimestamp = UpdateAutoTimestamp.create(null);
 
   /** Get the {@link UpdateAutoTimestamp} for this entity. */

core/src/main/java/google/registry/model/DatabaseMigrationUtils.java

@@ -14,6 +14,8 @@
 
 package google.registry.model;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryEnvironment;
 import google.registry.model.common.DatabaseTransitionSchedule;
@@ -44,5 +46,10 @@ public class DatabaseMigrationUtils {
         .orElse(PrimaryDatabase.DATASTORE);
   }
 
+  public static boolean isDatastore(TransitionId transitionId) {
+    return tm().transactNew(() -> DatabaseMigrationUtils.getPrimaryDatabase(transitionId))
+        .equals(PrimaryDatabase.DATASTORE);
+  }
+
   private DatabaseMigrationUtils() {}
 }

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -194,30 +194,6 @@ public final class EppResourceUtils {
     return ForeignKeyIndex.load(clazz, uniqueIds, now).keySet();
   }
 
-  /**
-   * Loads resources that match some filter and that have {@link EppResource#deletionTime} that is
-   * not before "now".
-   *
-   * <p>This is an eventually consistent query.
-   *
-   * @param clazz the resource type to load
-   * @param now the logical time of the check
-   * @param filterDefinition the filter to apply when loading resources
-   * @param filterValue the acceptable value for the filter
-   */
-  public static <T extends EppResource> Iterable<T> queryNotDeleted(
-      Class<T> clazz, DateTime now, String filterDefinition, Object filterValue) {
-    return ofy()
-        .load()
-        .type(clazz)
-        .filter(filterDefinition, filterValue)
-        .filter("deletionTime >", now.toDate())
-        .list()
-        .stream()
-        .map(EppResourceUtils.transformAtTime(now))
-        .collect(toImmutableSet());
-  }
-
   /**
    * Returns a Function that transforms an EppResource to the given DateTime, suitable for use with
    * Iterables.transform() over a collection of EppResources.
@@ -418,8 +394,7 @@ public final class EppResourceUtils {
                 if (isContactKey) {
                   query =
                       jpaTm()
-                          .getEntityManager()
-                          .createQuery(CONTACT_LINKED_DOMAIN_QUERY, String.class)
+                          .query(CONTACT_LINKED_DOMAIN_QUERY, String.class)
                           .setParameter("fkRepoId", key)
                           .setParameter("now", now);
                 } else {

core/src/main/java/google/registry/model/OteAccountBuilder.java

@@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.registry.Registry.TldState.GENERAL_AVAILABILITY;
 import static google.registry.model.registry.Registry.TldState.START_DATE_SUNRISE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -41,10 +40,10 @@ import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.model.registry.label.PremiumList;
 import google.registry.model.registry.label.PremiumListDualDao;
+import google.registry.persistence.VKey;
 import google.registry.util.CidrAddressBlock;
 import java.util.Collection;
 import java.util.Optional;
-import java.util.Set;
 import java.util.function.Function;
 import java.util.regex.Pattern;
 import org.joda.money.CurrencyUnit;
@@ -225,15 +224,14 @@ public final class OteAccountBuilder {
   }
 
   /**
-   * Persists all the OT&E entities to datastore.
+   * Persists all the OT&E entities to the database.
    *
    * @return map from the new clientIds created to the new TLDs they have access to. Can be used to
    *     go over all the newly created Registrars / Registries / RegistrarContacts if any
    *     post-creation work is needed.
    */
   public ImmutableMap<String, String> buildAndPersist() {
-    // save all the entitiesl in a single transaction
-    tm().transact(this::saveAllEntities);
+    saveAllEntities();
     return clientIdToTld;
   }
 
@@ -246,30 +244,38 @@ public final class OteAccountBuilder {
 
   /** Saves all the OT&amp;E entities we created. */
   private void saveAllEntities() {
-    tm().assertInTransaction();
-
     // use ImmutableObject instead of Registry so that the Key generation doesn't break
-    ImmutableList<ImmutableObject> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
+    ImmutableList<Registry> registries = ImmutableList.of(sunriseTld, gaTld, eapTld);
     ImmutableList<RegistrarContact> contacts = contactsBuilder.build();
 
-    if (!replaceExisting) {
-      ImmutableList<Key<ImmutableObject>> keys =
-          Streams.concat(registries.stream(), registrars.stream(), contacts.stream())
-              .map(Key::create)
-              .collect(toImmutableList());
-      Set<Key<ImmutableObject>> existingKeys = ofy().load().keys(keys).keySet();
-      checkState(
-          existingKeys.isEmpty(),
-          "Found existing object(s) conflicting with OT&E objects: %s",
-          existingKeys);
-    }
-    // Save the Registries (TLDs) first
-    ofy().save().entities(registries).now();
-    // Now we can set the allowedTlds for the registrars
-    registrars = registrars.stream().map(this::addAllowedTld).collect(toImmutableList());
-    // and we can save the registrars and contacts!
-    ofy().save().entities(registrars);
-    ofy().save().entities(contacts);
+    tm().transact(
+            () -> {
+              if (!replaceExisting) {
+                ImmutableList<VKey<? extends ImmutableObject>> keys =
+                    Streams.concat(
+                            registries.stream()
+                                .map(registry -> Registry.createVKey(registry.getTldStr())),
+                            registrars.stream().map(Registrar::createVKey),
+                            contacts.stream().map(RegistrarContact::createVKey))
+                        .collect(toImmutableList());
+                ImmutableMap<VKey<? extends ImmutableObject>, ImmutableObject> existingObjects =
+                    tm().loadByKeysIfPresent(keys);
+                checkState(
+                    existingObjects.isEmpty(),
+                    "Found existing object(s) conflicting with OT&E objects: %s",
+                    existingObjects.keySet());
+              }
+              // Save the Registries (TLDs) first
+              tm().putAll(registries);
+            });
+    // Now we can set the allowedTlds for the registrars in a new transaction
+    tm().transact(
+            () -> {
+              registrars = registrars.stream().map(this::addAllowedTld).collect(toImmutableList());
+              // and we can save the registrars and contacts!
+              tm().putAll(registrars);
+              tm().putAll(contacts);
+            });
   }
 
   private Registrar addAllowedTld(Registrar registrar) {

core/src/main/java/google/registry/model/OteStats.java

@@ -17,7 +17,6 @@ package google.registry.model;
 import static com.google.common.base.Predicates.equalTo;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.eppcommon.EppXmlTransformer.unmarshal;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
 import static google.registry.util.DomainNameUtils.ACE_PREFIX;
 
@@ -28,7 +27,6 @@ import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Multiset;
 import com.googlecode.objectify.Key;
-import com.googlecode.objectify.cmd.Query;
 import google.registry.model.domain.DomainCommand;
 import google.registry.model.domain.fee.FeeCreateCommandExtension;
 import google.registry.model.domain.launch.LaunchCreateExtension;
@@ -39,6 +37,7 @@ import google.registry.model.eppinput.EppInput.ResourceCommandWrapper;
 import google.registry.model.host.HostCommand;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.HistoryEntry.Type;
+import google.registry.model.reporting.HistoryEntryDao;
 import google.registry.xml.XmlException;
 import java.util.Arrays;
 import java.util.EnumSet;
@@ -196,16 +195,10 @@ public class OteStats {
    * <p>Stops when it notices that all tests have passed.
    */
   private OteStats recordRegistrarHistory(String registrarName) {
-    ImmutableCollection<String> clientIds =
+    ImmutableCollection<String> registrarIds =
         OteAccountBuilder.createClientIdToTldMap(registrarName).keySet();
 
-    Query<HistoryEntry> query =
-        ofy()
-            .load()
-            .type(HistoryEntry.class)
-            .filter("clientId in", clientIds)
-            .order("modificationTime");
-    for (HistoryEntry historyEntry : query) {
+    for (HistoryEntry historyEntry : HistoryEntryDao.loadHistoryObjectsByRegistrars(registrarIds)) {
       try {
         record(historyEntry);
       } catch (XmlException e) {

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -14,11 +14,22 @@
 
 package google.registry.model;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.OnLoad;
 import google.registry.model.translators.UpdateAutoTimestampTranslatorFactory;
+import google.registry.util.DateTimeUtils;
+import java.time.ZonedDateTime;
 import java.util.Optional;
 import javax.annotation.Nullable;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.PostLoad;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -26,14 +37,45 @@ import org.joda.time.DateTime;
  *
  * @see UpdateAutoTimestampTranslatorFactory
  */
+@Embeddable
 public class UpdateAutoTimestamp extends ImmutableObject {
 
-  // When set to true, database converters/translators should do tha auto update.  When set to
+  // When set to true, database converters/translators should do the auto update.  When set to
   // false, auto update should be suspended (this exists to allow us to preserve the original value
   // during a replay).
   private static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
 
-  DateTime timestamp;
+  @Transient DateTime timestamp;
+
+  @Ignore
+  @Column(nullable = false)
+  ZonedDateTime lastUpdateTime;
+
+  // Unfortunately, we cannot use the @UpdateTimestamp annotation on "lastUpdateTime" in this class
+  // because Hibernate does not allow it to be used on @Embeddable classes, see
+  // https://hibernate.atlassian.net/browse/HHH-13235. This is a workaround.
+  @PrePersist
+  @PreUpdate
+  void setTimestamp() {
+    if (autoUpdateEnabled() || lastUpdateTime == null) {
+      timestamp = jpaTm().getTransactionTime();
+      lastUpdateTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @OnLoad
+  void onLoad() {
+    if (timestamp != null) {
+      lastUpdateTime = DateTimeUtils.toZonedDateTime(timestamp);
+    }
+  }
+
+  @PostLoad
+  void postLoad() {
+    if (lastUpdateTime != null) {
+      timestamp = DateTimeUtils.toJodaDateTime(lastUpdateTime);
+    }
+  }
 
   /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
   public DateTime getTimestamp() {
@@ -43,6 +85,7 @@ public class UpdateAutoTimestamp extends ImmutableObject {
   public static UpdateAutoTimestamp create(@Nullable DateTime timestamp) {
     UpdateAutoTimestamp instance = new UpdateAutoTimestamp();
     instance.timestamp = timestamp;
+    instance.lastUpdateTime = timestamp == null ? null : DateTimeUtils.toZonedDateTime(timestamp);
     return instance;
   }
 

core/src/main/java/google/registry/model/annotations/InCrossTld.java

@@ -0,0 +1,34 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.annotations;
+
+import com.googlecode.objectify.annotation.Entity;
+import google.registry.model.common.EntityGroupRoot;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation for an Objectify {@link Entity} to indicate that it is in the cross-TLD entity group.
+ *
+ * <p>This means that the entity's <code>@Parent</code> field has to have the value of {@link
+ * EntityGroupRoot#getCrossTldKey}.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+@Inherited
+public @interface InCrossTld {}

core/src/main/java/google/registry/model/common/CrossTldSingleton.java

@@ -20,11 +20,13 @@ import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import javax.persistence.MappedSuperclass;
 import javax.persistence.Transient;
 
 /** A singleton entity in Datastore. */
 @MappedSuperclass
+@InCrossTld
 public abstract class CrossTldSingleton extends ImmutableObject {
 
   public static final long SINGLETON_ID = 1; // There is always exactly one of these.

core/src/main/java/google/registry/model/common/Cursor.java

@@ -24,12 +24,24 @@ import com.google.common.base.Splitter;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Ignore;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.InCrossTld;
+import google.registry.model.common.Cursor.CursorId;
 import google.registry.model.registry.Registry;
-import google.registry.schema.replay.DatastoreOnlyEntity;
+import google.registry.persistence.VKey;
+import google.registry.schema.replay.DatastoreAndSqlEntity;
+import java.io.Serializable;
 import java.util.List;
+import javax.persistence.Column;
+import javax.persistence.EnumType;
+import javax.persistence.Enumerated;
+import javax.persistence.IdClass;
+import javax.persistence.PostLoad;
+import javax.persistence.Transient;
 import org.joda.time.DateTime;
 
 /**
@@ -38,7 +50,13 @@ import org.joda.time.DateTime;
  * scoped on {@link EntityGroupRoot}.
  */
 @Entity
-public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
+@javax.persistence.Entity
+@IdClass(CursorId.class)
+@InCrossTld
+public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
+
+  /** The scope of a global cursor. A global cursor is a cursor that is not specific to one tld. */
+  public static final String GLOBAL = "GLOBAL";
 
   /** The types of cursors, used as the string id field for each cursor in Datastore. */
   public enum CursorType {
@@ -104,9 +122,9 @@ public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
 
     /**
      * If there are multiple cursors for a given cursor type, a cursor must also have a scope
-     * defined (distinct from a parent, which is always the EntityGroupRoot key). For instance,
-     * for a cursor that is defined at the registry level, the scope type will be Registry.class.
-     * For a cursor (theoretically) defined for each EPP resource, the scope type will be
+     * defined (distinct from a parent, which is always the EntityGroupRoot key). For instance, for
+     * a cursor that is defined at the registry level, the scope type will be Registry.class. For a
+     * cursor (theoretically) defined for each EPP resource, the scope type will be
      * EppResource.class. For a global cursor, i.e. one that applies per environment, this will be
      * {@link EntityGroupRoot}.
      */
@@ -115,24 +133,73 @@ public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
     }
   }
 
-  @Parent
-  Key<EntityGroupRoot> parent = getCrossTldKey();
+  @Transient @Parent Key<EntityGroupRoot> parent = getCrossTldKey();
 
-  @Id
-  String id;
+  @Transient @Id String id;
 
+  @Ignore
+  @Enumerated(EnumType.STRING)
+  @Column(nullable = false)
+  @javax.persistence.Id
+  CursorType type;
+
+  @Ignore
+  @Column(nullable = false)
+  @javax.persistence.Id
+  String scope;
+
+  @Column(nullable = false)
   DateTime cursorTime = START_OF_TIME;
 
   /** An automatically managed timestamp of when this object was last written to Datastore. */
+  @Column(nullable = false)
   UpdateAutoTimestamp lastUpdateTime = UpdateAutoTimestamp.create(null);
 
+  @OnLoad
+  void onLoad() {
+    scope = getScopeFromId(id);
+    type = getTypeFromId(id);
+  }
+
+  @PostLoad
+  void postLoad() {
+    // "Generate" the ID based on the scope and type
+    Key<? extends ImmutableObject> scopeKey =
+        scope.equals(GLOBAL)
+            ? getCrossTldKey()
+            : Key.create(getCrossTldKey(), Registry.class, scope);
+    id = generateId(type, scopeKey);
+  }
+
+  public static VKey<Cursor> createVKey(Key<Cursor> key) {
+    String id = key.getName();
+    return VKey.create(Cursor.class, new CursorId(getTypeFromId(id), getScopeFromId(id)), key);
+  }
+
+  public VKey<Cursor> createVKey() {
+    return createVKey(type, scope);
+  }
+
+  public static VKey<Cursor> createGlobalVKey(CursorType type) {
+    return createVKey(type, GLOBAL);
+  }
+
+  public static VKey<Cursor> createVKey(CursorType type, String scope) {
+    Key<Cursor> key =
+        scope.equals(GLOBAL) ? createGlobalKey(type) : createKey(type, Registry.get(scope));
+    return VKey.create(Cursor.class, new CursorId(type, scope), key);
+  }
+
   public DateTime getLastUpdateTime() {
     return lastUpdateTime.getTimestamp();
   }
 
+  public String getScope() {
+    return scope;
+  }
+
   public CursorType getType() {
-    List<String> id = Splitter.on('_').splitToList(this.id);
-    return CursorType.valueOf(String.join("_", id.subList(1, id.size())));
+    return type;
   }
 
   /**
@@ -142,10 +209,12 @@ public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
   private static void checkValidCursorTypeForScope(
       CursorType cursorType, Key<? extends ImmutableObject> scope) {
     checkArgument(
-        cursorType.getScopeClass().equals(
-            scope.equals(EntityGroupRoot.getCrossTldKey())
-                ? EntityGroupRoot.class
-                : ofy().factory().getMetadata(scope).getEntityClass()),
+        cursorType
+            .getScopeClass()
+            .equals(
+                scope.equals(getCrossTldKey())
+                    ? EntityGroupRoot.class
+                    : ofy().factory().getMetadata(scope).getEntityClass()),
         "Class required for cursor does not match scope class");
   }
 
@@ -154,6 +223,20 @@ public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
     return String.format("%s_%s", scope.getString(), cursorType.name());
   }
 
+  private static String getScopeFromId(String id) {
+    List<String> idSplit = Splitter.on('_').splitToList(id);
+    // The "parent" is always the crossTldKey; in order to find the scope (either Registry or
+    // cross-tld-key) we have to parse the part of the ID
+    Key<?> scopeKey = Key.valueOf(idSplit.get(0));
+    return scopeKey.equals(getCrossTldKey()) ? GLOBAL : scopeKey.getName();
+  }
+
+  private static CursorType getTypeFromId(String id) {
+    List<String> idSplit = Splitter.on('_').splitToList(id);
+    // The cursor type is the second part of the ID string
+    return CursorType.valueOf(String.join("_", idSplit.subList(1, idSplit.size())));
+  }
+
   /** Creates a unique key for a given scope and cursor type. */
   public static Key<Cursor> createKey(CursorType cursorType, ImmutableObject scope) {
     Key<? extends ImmutableObject> scopeKey = Key.create(scope);
@@ -166,13 +249,12 @@ public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
     checkArgument(
         cursorType.getScopeClass().equals(EntityGroupRoot.class),
         "Cursor type is not a global cursor.");
-    return Key.create(
-        getCrossTldKey(), Cursor.class, generateId(cursorType, EntityGroupRoot.getCrossTldKey()));
+    return Key.create(getCrossTldKey(), Cursor.class, generateId(cursorType, getCrossTldKey()));
   }
 
   /** Creates a new global cursor instance. */
   public static Cursor createGlobal(CursorType cursorType, DateTime cursorTime) {
-    return create(cursorType, cursorTime, EntityGroupRoot.getCrossTldKey());
+    return create(cursorType, cursorTime, getCrossTldKey());
   }
 
   /** Creates a new cursor instance with a given {@link Key} scope. */
@@ -184,8 +266,10 @@ public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
     checkNotNull(cursorType, "Cursor type cannot be null");
     checkValidCursorTypeForScope(cursorType, scope);
     instance.id = generateId(cursorType, scope);
+    instance.type = cursorType;
+    instance.scope = scope.equals(getCrossTldKey()) ? GLOBAL : scope.getName();
     return instance;
-   }
+  }
 
   /** Creates a new cursor instance with a given {@link ImmutableObject} scope. */
   public static Cursor create(CursorType cursorType, DateTime cursorTime, ImmutableObject scope) {
@@ -203,4 +287,17 @@ public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
   public DateTime getCursorTime() {
     return cursorTime;
   }
+
+  static class CursorId extends ImmutableObject implements Serializable {
+
+    public CursorType type;
+    public String scope;
+
+    private CursorId() {}
+
+    public CursorId(CursorType type, String scope) {
+      this.type = type;
+      this.scope = scope;
+    }
+  }
 }

core/src/main/java/google/registry/model/common/DatabaseTransitionSchedule.java

@@ -32,11 +32,13 @@ import com.googlecode.objectify.annotation.Mapify;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.common.TimedTransitionProperty.TimeMapper;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.model.registry.label.PremiumList;
 import google.registry.model.registry.label.ReservedList;
 import google.registry.model.smd.SignedMarkRevocationList;
+import google.registry.model.tmch.ClaimsListShard;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreOnlyEntity;
 import java.util.Optional;
@@ -45,6 +47,7 @@ import org.joda.time.DateTime;
 
 @Entity
 @Immutable
+@InCrossTld
 public class DatabaseTransitionSchedule extends ImmutableObject implements DatastoreOnlyEntity {
 
   /**
@@ -58,6 +61,8 @@ public class DatabaseTransitionSchedule extends ImmutableObject implements Datas
 
   /** The id of the transition schedule. */
   public enum TransitionId {
+    /** The schedule for migration of {@link ClaimsListShard} entities. */
+    CLAIMS_LIST,
     /** The schedule for the migration of {@link PremiumList} and {@link ReservedList}. */
     DOMAIN_LABEL_LISTS,
     /** The schedule for the migration of the {@link SignedMarkRevocationList} entity. */

core/src/main/java/google/registry/model/common/GaeUserIdConverter.java

@@ -16,8 +16,7 @@ package google.registry.model.common;
 
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 
 import com.google.appengine.api.users.User;
 import com.google.common.base.Splitter;
@@ -56,13 +55,22 @@ public class GaeUserIdConverter extends ImmutableObject {
     try {
       // Perform these operations in a transactionless context to avoid enlisting in some outer
       // transaction (if any).
-      tm().doTransactionless(() -> ofy().saveWithoutBackup().entity(gaeUserIdConverter).now());
+      ofyTm()
+          .doTransactionless(
+              () -> {
+                ofyTm().putWithoutBackup(gaeUserIdConverter);
+                return null;
+              });
 
       // The read must be done in its own transaction to avoid reading from the session cache.
-      return tm()
-          .transactNew(() -> ofy().load().entity(gaeUserIdConverter).safe().user.getUserId());
+      return ofyTm().transactNew(() -> ofyTm().loadByEntity(gaeUserIdConverter).user.getUserId());
     } finally {
-      tm().doTransactionless(() -> ofy().deleteWithoutBackup().entity(gaeUserIdConverter).now());
+      ofyTm()
+          .doTransactionless(
+              () -> {
+                ofyTm().deleteWithoutBackup(gaeUserIdConverter);
+                return null;
+              });
     }
   }
 }

core/src/main/java/google/registry/model/contact/ContactBase.java

@@ -21,6 +21,7 @@ import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderA
 import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ResourceWithTransferData;
@@ -189,6 +190,17 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
             + " use ContactResource instead");
   }
 
+  @OnLoad
+  void onLoad() {
+    if (voice != null && voice.hasNullFields()) {
+      voice = null;
+    }
+
+    if (fax != null && fax.hasNullFields()) {
+      fax = null;
+    }
+  }
+
   public String getContactId() {
     return contactId;
   }
@@ -325,11 +337,17 @@ public class ContactBase extends EppResource implements ResourceWithTransferData
     }
 
     public B setVoiceNumber(ContactPhoneNumber voiceNumber) {
+      if (voiceNumber != null && voiceNumber.hasNullFields()) {
+        voiceNumber = null;
+      }
       getInstance().voice = voiceNumber;
       return thisCastToDerived();
     }
 
     public B setFaxNumber(ContactPhoneNumber faxNumber) {
+      if (faxNumber != null && faxNumber.hasNullFields()) {
+        faxNumber = null;
+      }
       getInstance().fax = faxNumber;
       return thisCastToDerived();
     }

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.contact;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -114,6 +116,12 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the contactBase field during asynchronous replay
+  public static void beforeSqlSave(ContactHistory contactHistory) {
+    contactHistory.contactBase =
+        jpaTm().loadByKey(VKey.createSql(ContactResource.class, contactHistory.getContactRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link ContactHistory} entity. */
   public static class ContactHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -23,8 +23,9 @@ import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.intersection;
 import static google.registry.model.EppResourceUtils.projectResourceOntoBuilderAtTime;
 import static google.registry.model.EppResourceUtils.setAutomaticTransferSuccessProperties;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.CollectionUtils.forceEmptyToNull;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
@@ -302,12 +303,6 @@ public class DomainContent extends EppResource
 
   @OnLoad
   void load() {
-    // Back fill with correct END_OF_TIME sentinel value.
-    // TODO(mcilwain): Remove this once back-filling is complete.
-    if (autorenewEndTime == null) {
-      autorenewEndTime = END_OF_TIME;
-    }
-
     // Reconstitute all of the contacts so that they have VKeys.
     allContacts =
         allContacts.stream().map(DesignatedContact::reconstitute).collect(toImmutableSet());
@@ -321,9 +316,6 @@ public class DomainContent extends EppResource
         nullToEmptyImmutableCopy(gracePeriods).stream()
             .map(gracePeriod -> gracePeriod.cloneAfterOfyLoad(getRepoId()))
             .collect(toImmutableSet());
-    // TODO(b/169873747): Remove this method after explicitly re-saving all domain entities.
-    // See also: GradePeriod.onLoad.
-    gracePeriods.forEach(GracePeriod::onLoad);
 
     // Restore history record ids.
     autorenewPollMessageHistoryId = getHistoryId(autorenewPollMessage);
@@ -346,7 +338,7 @@ public class DomainContent extends EppResource
 
   @PostLoad
   @SuppressWarnings("UnusedMethod")
-  private final void postLoad() {
+  private void postLoad() {
     // Reconstitute the contact list.
     ImmutableSet.Builder<DesignatedContact> contactsBuilder = new ImmutableSet.Builder<>();
 
@@ -387,13 +379,11 @@ public class DomainContent extends EppResource
   public static void beforeSqlDelete(VKey<DomainBase> key) {
     // Delete all grace periods associated with the domain.
     jpaTm()
-        .getEntityManager()
-        .createQuery("DELETE FROM GracePeriod WHERE domain_repo_id = :repo_id")
+        .query("DELETE FROM GracePeriod WHERE domain_repo_id = :repo_id")
         .setParameter("repo_id", key.getSqlKey())
         .executeUpdate();
     jpaTm()
-        .getEntityManager()
-        .createQuery("DELETE FROM DelegationSignerData WHERE domain_repo_id = :repo_id")
+        .query("DELETE FROM DelegationSignerData WHERE domain_repo_id = :repo_id")
         .setParameter("repo_id", key.getSqlKey())
         .executeUpdate();
   }
@@ -443,12 +433,7 @@ public class DomainContent extends EppResource
    * purposes of more legible business logic.
    */
   public Optional<DateTime> getAutorenewEndTime() {
-    // TODO(mcilwain): Remove null handling for autorenewEndTime once data migration away from null
-    //                 is complete.
-    return Optional.ofNullable(
-        (autorenewEndTime == null || autorenewEndTime.equals(END_OF_TIME))
-            ? null
-            : autorenewEndTime);
+    return Optional.ofNullable(autorenewEndTime.equals(END_OF_TIME) ? null : autorenewEndTime);
   }
 
   @Override
@@ -686,13 +671,11 @@ public class DomainContent extends EppResource
 
   /** Loads and returns the fully qualified host names of all linked nameservers. */
   public ImmutableSortedSet<String> loadNameserverHostNames() {
-    return ofy()
-        .load()
-        .keys(getNameservers().stream().map(VKey::getOfyKey).collect(toImmutableSet()))
-        .values()
-        .stream()
-        .map(HostResource::getHostName)
-        .collect(toImmutableSortedSet(Ordering.natural()));
+    return transactIfJpaTm(
+        () ->
+            tm().loadByKeys(getNameservers()).values().stream()
+                .map(HostResource::getHostName)
+                .collect(toImmutableSortedSet(Ordering.natural())));
   }
 
   /** A key to the registrant who registered this domain. */

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -15,6 +15,7 @@
 package google.registry.model.domain;
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.collect.ImmutableSet;
@@ -263,6 +264,12 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the domainContent field during asynchronous replay
+  public static void beforeSqlSave(DomainHistory domainHistory) {
+    domainHistory.domainContent =
+        jpaTm().loadByKey(VKey.createSql(DomainBase.class, domainHistory.getDomainRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link DomainHistory} entity. */
   public static class DomainHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -54,17 +54,6 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
     return super.getGracePeriodId();
   }
 
-  // TODO(b/169873747): Remove this method after explicitly re-saving all domain entities.
-  // This method is invoked from DomainContent.load(): Objectify's @OnLoad annotation
-  // apparently does not work on embedded objects inside an entity.
-  // Changing signature to void onLoad(@AlsoLoad("gracePeriodId") Long gracePeriodId)
-  // would not work. Method is not called if gracePeriodId is null.
-  void onLoad() {
-    if (gracePeriodId == null) {
-      gracePeriodId = ObjectifyService.allocateId();
-    }
-  }
-
   private static GracePeriod createInternal(
       GracePeriodStatus type,
       String domainRepoId,

core/src/main/java/google/registry/model/eppcommon/PhoneNumber.java

@@ -71,6 +71,11 @@ public class PhoneNumber extends ImmutableObject {
     return phoneNumber + (extension != null ? " x" + extension : "");
   }
 
+  /** Returns true if both fields of the phone number are null. */
+  public boolean hasNullFields() {
+    return phoneNumber == null && extension == null;
+  }
+
   /** A builder for constructing {@link PhoneNumber}. */
   public static class Builder<T extends PhoneNumber> extends Buildable.Builder<T> {
     @Override

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.host;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -115,6 +117,12 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the hostBase field during asynchronous replay
+  public static void beforeSqlSave(HostHistory hostHistory) {
+    hostHistory.hostBase =
+        jpaTm().loadByKey(VKey.createSql(HostResource.class, hostHistory.getHostRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link HostHistory} entity. */
   public static class HostHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -16,17 +16,20 @@ package google.registry.model.index;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 import static google.registry.util.TypeUtils.instantiate;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableBiMap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -45,10 +48,11 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.NonFinalForTesting;
+import java.util.Collection;
 import java.util.Comparator;
-import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
@@ -81,10 +85,10 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource>
       implements DatastoreOnlyEntity {}
 
-  private static final ImmutableMap<
+  private static final ImmutableBiMap<
           Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
       RESOURCE_CLASS_TO_FKI_CLASS =
-          ImmutableMap.of(
+          ImmutableBiMap.of(
               ContactResource.class, ForeignKeyContactIndex.class,
               DomainBase.class, ForeignKeyDomainIndex.class,
               HostResource.class, ForeignKeyHostIndex.class);
@@ -184,72 +188,90 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   }
 
   /**
-   * Load a list of {@link ForeignKeyIndex} instances by class and id strings that are active at or
+   * Load a map of {@link ForeignKeyIndex} instances by class and id strings that are active at or
    * after the specified moment in time.
    *
    * <p>The returned map will omit any keys for which the {@link ForeignKeyIndex} doesn't exist or
    * has been soft deleted.
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
-      Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
+      Class<E> clazz, Collection<String> foreignKeys, final DateTime now) {
+    return loadIndexesFromStore(clazz, foreignKeys).entrySet().stream()
+        .filter(e -> now.isBefore(e.getValue().getDeletionTime()))
+        .collect(entriesToImmutableMap());
+  }
+
+  /**
+   * Helper method to load all of the most recent {@link ForeignKeyIndex}es for the given foreign
+   * keys, regardless of whether or not they have been soft-deleted.
+   *
+   * <p>Used by both the cached (w/o deletion check) and the non-cached (with deletion check) calls.
+   */
+  private static <E extends EppResource>
+      ImmutableMap<String, ForeignKeyIndex<E>> loadIndexesFromStore(
+          Class<E> clazz, Collection<String> foreignKeys) {
     if (tm().isOfy()) {
-      return ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys).entrySet().stream()
-          .filter(e -> now.isBefore(e.getValue().deletionTime))
-          .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+      return ImmutableMap.copyOf(
+          tm().doTransactionless(() -> ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys)));
     } else {
       String property = RESOURCE_CLASS_TO_FKI_PROPERTY.get(clazz);
-      List<E> entities =
+      ImmutableList<ForeignKeyIndex<E>> indexes =
           tm().transact(
-                  () -> {
-                    String entityName =
-                        jpaTm().getEntityManager().getMetamodel().entity(clazz).getName();
-                    return jpaTm()
-                        .getEntityManager()
-                        .createQuery(
-                            String.format(
-                                "FROM %s WHERE %s IN :propertyValue and deletionTime > :now ",
-                                entityName, property),
-                            clazz)
-                        .setParameter("propertyValue", foreignKeys)
-                        .setParameter("now", now)
-                        .getResultList();
-                  });
+                  () ->
+                      jpaTm()
+                          .getEntityManager()
+                          .createQuery(
+                              CriteriaQueryBuilder.create(clazz)
+                                  .whereFieldIsIn(property, foreignKeys)
+                                  .build())
+                          .getResultStream()
+                          .map(e -> ForeignKeyIndex.create(e, e.getDeletionTime()))
+                          .collect(toImmutableList()));
       // We need to find and return the entities with the maximum deletionTime for each foreign key.
-      return Multimaps.index(entities, EppResource::getForeignKey).asMap().entrySet().stream()
+      return Multimaps.index(indexes, ForeignKeyIndex::getForeignKey).asMap().entrySet().stream()
           .map(
               entry ->
                   Maps.immutableEntry(
                       entry.getKey(),
                       entry.getValue().stream()
-                          .max(Comparator.comparing(EppResource::getDeletionTime))
+                          .max(Comparator.comparing(ForeignKeyIndex::getDeletionTime))
                           .get()))
-          .collect(
-              toImmutableMap(
-                  Map.Entry::getKey,
-                  entry -> create(entry.getValue(), entry.getValue().getDeletionTime())));
+          .collect(entriesToImmutableMap());
     }
   }
 
-  static final CacheLoader<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>> CACHE_LOADER =
-      new CacheLoader<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>() {
+  static final CacheLoader<VKey<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>> CACHE_LOADER =
+      new CacheLoader<VKey<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>() {
 
         @Override
-        public Optional<ForeignKeyIndex<?>> load(Key<ForeignKeyIndex<?>> key) {
-          return Optional.ofNullable(tm().doTransactionless(() -> ofy().load().key(key).now()));
+        public Optional<ForeignKeyIndex<?>> load(VKey<ForeignKeyIndex<?>> key) {
+          String foreignKey = key.getSqlKey().toString();
+          return Optional.ofNullable(
+              loadIndexesFromStore(
+                      RESOURCE_CLASS_TO_FKI_CLASS.inverse().get(key.getKind()),
+                      ImmutableSet.of(foreignKey))
+                  .get(foreignKey));
         }
 
         @Override
-        public Map<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>> loadAll(
-            Iterable<? extends Key<ForeignKeyIndex<?>>> keys) {
-          ImmutableSet<Key<ForeignKeyIndex<?>>> typedKeys = ImmutableSet.copyOf(keys);
-          Map<Key<ForeignKeyIndex<?>>, ForeignKeyIndex<?>> existingFkis =
-              tm().doTransactionless(() -> ofy().load().keys(typedKeys));
+        public Map<VKey<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>> loadAll(
+            Iterable<? extends VKey<ForeignKeyIndex<?>>> keys) {
+          if (!keys.iterator().hasNext()) {
+            return ImmutableMap.of();
+          }
+          Class<? extends EppResource> resourceClass =
+              RESOURCE_CLASS_TO_FKI_CLASS.inverse().get(keys.iterator().next().getKind());
+          ImmutableSet<String> foreignKeys =
+              Streams.stream(keys).map(v -> v.getSqlKey().toString()).collect(toImmutableSet());
+          ImmutableSet<VKey<ForeignKeyIndex<?>>> typedKeys = ImmutableSet.copyOf(keys);
+          ImmutableMap<String, ? extends ForeignKeyIndex<? extends EppResource>> existingFkis =
+              loadIndexesFromStore(resourceClass, foreignKeys);
           // ofy() omits keys that don't have values in Datastore, so re-add them in
           // here with Optional.empty() values.
           return Maps.asMap(
               typedKeys,
-              (Key<ForeignKeyIndex<?>> key) ->
-                  Optional.ofNullable(existingFkis.getOrDefault(key, null)));
+              (VKey<ForeignKeyIndex<?>> key) ->
+                  Optional.ofNullable(existingFkis.getOrDefault(key.getSqlKey().toString(), null)));
         }
       };
 
@@ -267,10 +289,10 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * given IDs (blah) don't exist."
    */
   @NonFinalForTesting
-  private static LoadingCache<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>
+  private static LoadingCache<VKey<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>
       cacheForeignKeyIndexes = createForeignKeyIndexesCache(getEppResourceCachingDuration());
 
-  private static LoadingCache<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>
+  private static LoadingCache<VKey<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>>
       createForeignKeyIndexesCache(Duration expiry) {
     return CacheBuilder.newBuilder()
         .expireAfterWrite(java.time.Duration.ofMillis(expiry.getMillis()))
@@ -295,25 +317,28 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * reasons, and are OK with the trade-offs in loss of transactional consistency.
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> loadCached(
-      Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
+      Class<E> clazz, Collection<String> foreignKeys, final DateTime now) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
       return tm().doTransactionless(() -> load(clazz, foreignKeys, now));
     }
-    ImmutableList<Key<ForeignKeyIndex<?>>> fkiKeys =
+    Class<? extends ForeignKeyIndex<?>> fkiClass = mapToFkiClass(clazz);
+    // Safe to cast VKey<FKI<E>> to VKey<FKI<?>>
+    @SuppressWarnings("unchecked")
+    ImmutableList<VKey<ForeignKeyIndex<?>>> fkiVKeys =
         Streams.stream(foreignKeys)
-            .map(fk -> Key.<ForeignKeyIndex<?>>create(mapToFkiClass(clazz), fk))
+            .map(fk -> (VKey<ForeignKeyIndex<?>>) VKey.create(fkiClass, fk))
             .collect(toImmutableList());
     try {
       // This cast is safe because when we loaded ForeignKeyIndexes above we used type clazz, which
       // is scoped to E.
       @SuppressWarnings("unchecked")
       ImmutableMap<String, ForeignKeyIndex<E>> fkisFromCache =
-          cacheForeignKeyIndexes.getAll(fkiKeys).entrySet().stream()
+          cacheForeignKeyIndexes.getAll(fkiVKeys).entrySet().stream()
               .filter(entry -> entry.getValue().isPresent())
               .filter(entry -> now.isBefore(entry.getValue().get().getDeletionTime()))
               .collect(
                   toImmutableMap(
-                      entry -> entry.getKey().getName(),
+                      entry -> entry.getKey().getSqlKey().toString(),
                       entry -> (ForeignKeyIndex<E>) entry.getValue().get()));
       return fkisFromCache;
     } catch (ExecutionException e) {

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -17,6 +17,7 @@ package google.registry.model.ofy;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -29,6 +30,8 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.Result;
+import com.googlecode.objectify.cmd.Query;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.contact.ContactHistory;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.host.HostHistory;
@@ -251,7 +254,13 @@ public class DatastoreTransactionManager implements TransactionManager {
 
   @Override
   public <T> ImmutableList<T> loadAllOf(Class<T> clazz) {
-    return ImmutableList.copyOf(getOfy().load().type(clazz));
+    Query<T> query = getOfy().load().type(clazz);
+    // If the entity is in the cross-TLD entity group, then we can take advantage of an ancestor
+    // query to give us strong transactional consistency.
+    if (clazz.isAnnotationPresent(InCrossTld.class)) {
+      query = query.ancestor(getCrossTldKey());
+    }
+    return ImmutableList.copyOf(query);
   }
 
   @Override
@@ -298,6 +307,11 @@ public class DatastoreTransactionManager implements TransactionManager {
     getOfy().clearSessionCache();
   }
 
+  @Override
+  public boolean isOfy() {
+    return true;
+  }
+
   /**
    * Executes the given {@link Result} instance synchronously if not in a transaction.
    *

core/src/main/java/google/registry/model/ofy/EntityWritePriorities.java

@@ -41,10 +41,11 @@ public class EntityWritePriorities {
    */
   static final ImmutableMap<String, Integer> CLASS_PRIORITIES =
       ImmutableMap.of(
-          "ContactResource", -15,
-          "HistoryEntry", -10,
           "AllocationToken", -9,
-          "DomainBase", 10);
+          "ContactResource", 8,
+          "HostResource", 9,
+          "DomainBase", 10,
+          "HistoryEntry", 20);
 
   // The beginning of the range of priority numbers reserved for delete.  This must be greater than
   // any of the values in CLASS_PRIORITIES by enough overhead to accommodate  any negative values in

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -106,7 +106,7 @@ public abstract class PollMessage extends ImmutableObject
   @Column(name = "poll_message_id")
   Long id;
 
-  @Parent @DoNotHydrate @Transient Key<HistoryEntry> parent;
+  @Parent @DoNotHydrate @Transient Key<? extends HistoryEntry> parent;
 
   /** The registrar that this poll message will be delivered to. */
   @Index
@@ -134,7 +134,7 @@ public abstract class PollMessage extends ImmutableObject
 
   @Ignore Long hostHistoryRevisionId;
 
-  public Key<HistoryEntry> getParentKey() {
+  public Key<? extends HistoryEntry> getParentKey() {
     return parent;
   }
 
@@ -239,7 +239,7 @@ public abstract class PollMessage extends ImmutableObject
       return thisCastToDerived();
     }
 
-    public B setParentKey(Key<HistoryEntry> parentKey) {
+    public B setParentKey(Key<? extends HistoryEntry> parentKey) {
       getInstance().parent = parentKey;
       return thisCastToDerived();
     }
@@ -398,6 +398,7 @@ public abstract class PollMessage extends ImmutableObject
       }
       if (!isNullOrEmpty(domainPendingActionNotificationResponses)) {
         pendingActionNotificationResponse = domainPendingActionNotificationResponses.get(0);
+        fullyQualifiedDomainName = pendingActionNotificationResponse.nameOrId.value;
       }
       if (!isNullOrEmpty(domainTransferResponses)) {
         fullyQualifiedDomainName = domainTransferResponses.get(0).getFullyQualifiedDomainName();
@@ -414,21 +415,23 @@ public abstract class PollMessage extends ImmutableObject
       // Take the SQL-specific fields and map them to the Objectify-specific fields, if applicable
       if (pendingActionNotificationResponse != null) {
         if (contactId != null) {
-          contactPendingActionNotificationResponses =
-              ImmutableList.of(
-                  ContactPendingActionNotificationResponse.create(
-                      pendingActionNotificationResponse.nameOrId.value,
-                      pendingActionNotificationResponse.getActionResult(),
-                      pendingActionNotificationResponse.getTrid(),
-                      pendingActionNotificationResponse.processedDate));
+          ContactPendingActionNotificationResponse contactPendingResponse =
+              ContactPendingActionNotificationResponse.create(
+                  pendingActionNotificationResponse.nameOrId.value,
+                  pendingActionNotificationResponse.getActionResult(),
+                  pendingActionNotificationResponse.getTrid(),
+                  pendingActionNotificationResponse.processedDate);
+          pendingActionNotificationResponse = contactPendingResponse;
+          contactPendingActionNotificationResponses = ImmutableList.of(contactPendingResponse);
         } else if (fullyQualifiedDomainName != null) {
-          domainPendingActionNotificationResponses =
-              ImmutableList.of(
-                  DomainPendingActionNotificationResponse.create(
-                      pendingActionNotificationResponse.nameOrId.value,
-                      pendingActionNotificationResponse.getActionResult(),
-                      pendingActionNotificationResponse.getTrid(),
-                      pendingActionNotificationResponse.processedDate));
+          DomainPendingActionNotificationResponse domainPendingResponse =
+              DomainPendingActionNotificationResponse.create(
+                  pendingActionNotificationResponse.nameOrId.value,
+                  pendingActionNotificationResponse.getActionResult(),
+                  pendingActionNotificationResponse.getTrid(),
+                  pendingActionNotificationResponse.processedDate);
+          pendingActionNotificationResponse = domainPendingResponse;
+          domainPendingActionNotificationResponses = ImmutableList.of(domainPendingResponse);
         }
       }
       if (transferResponse != null) {
@@ -474,38 +477,35 @@ public abstract class PollMessage extends ImmutableObject
       }
 
       public Builder setResponseData(ImmutableList<? extends ResponseData> responseData) {
-        getInstance().contactPendingActionNotificationResponses =
+        OneTime instance = getInstance();
+        instance.contactPendingActionNotificationResponses =
             forceEmptyToNull(
-                responseData
-                    .stream()
+                responseData.stream()
                     .filter(ContactPendingActionNotificationResponse.class::isInstance)
                     .map(ContactPendingActionNotificationResponse.class::cast)
                     .collect(toImmutableList()));
 
-        getInstance().contactTransferResponses =
+        instance.contactTransferResponses =
             forceEmptyToNull(
-                responseData
-                    .stream()
+                responseData.stream()
                     .filter(ContactTransferResponse.class::isInstance)
                     .map(ContactTransferResponse.class::cast)
                     .collect(toImmutableList()));
 
-        getInstance().domainPendingActionNotificationResponses =
+        instance.domainPendingActionNotificationResponses =
             forceEmptyToNull(
-                responseData
-                    .stream()
+                responseData.stream()
                     .filter(DomainPendingActionNotificationResponse.class::isInstance)
                     .map(DomainPendingActionNotificationResponse.class::cast)
                     .collect(toImmutableList()));
-        getInstance().domainTransferResponses =
+        instance.domainTransferResponses =
             forceEmptyToNull(
-                responseData
-                    .stream()
+                responseData.stream()
                     .filter(DomainTransferResponse.class::isInstance)
                     .map(DomainTransferResponse.class::cast)
                     .collect(toImmutableList()));
 
-        getInstance().hostPendingActionNotificationResponses =
+        instance.hostPendingActionNotificationResponses =
             forceEmptyToNull(
                 responseData.stream()
                     .filter(HostPendingActionNotificationResponse.class::isInstance)
@@ -513,26 +513,30 @@ public abstract class PollMessage extends ImmutableObject
                     .collect(toImmutableList()));
 
         // Set the generic pending-action field as appropriate
-        if (getInstance().contactPendingActionNotificationResponses != null) {
-          getInstance().pendingActionNotificationResponse =
-              getInstance().contactPendingActionNotificationResponses.get(0);
-        } else if (getInstance().domainPendingActionNotificationResponses != null) {
-          getInstance().pendingActionNotificationResponse =
-              getInstance().domainPendingActionNotificationResponses.get(0);
-        } else if (getInstance().hostPendingActionNotificationResponses != null) {
-          getInstance().pendingActionNotificationResponse =
-              getInstance().hostPendingActionNotificationResponses.get(0);
+        if (instance.contactPendingActionNotificationResponses != null) {
+          instance.pendingActionNotificationResponse =
+              instance.contactPendingActionNotificationResponses.get(0);
+          instance.contactId =
+              instance.contactPendingActionNotificationResponses.get(0).nameOrId.value;
+        } else if (instance.domainPendingActionNotificationResponses != null) {
+          instance.pendingActionNotificationResponse =
+              instance.domainPendingActionNotificationResponses.get(0);
+          instance.fullyQualifiedDomainName =
+              instance.domainPendingActionNotificationResponses.get(0).nameOrId.value;
+        } else if (instance.hostPendingActionNotificationResponses != null) {
+          instance.pendingActionNotificationResponse =
+              instance.hostPendingActionNotificationResponses.get(0);
         }
         // Set the generic transfer response field as appropriate
-        if (getInstance().contactTransferResponses != null) {
-          getInstance().contactId = getInstance().contactTransferResponses.get(0).getContactId();
-          getInstance().transferResponse = getInstance().contactTransferResponses.get(0);
-        } else if (getInstance().domainTransferResponses != null) {
-          getInstance().fullyQualifiedDomainName =
-              getInstance().domainTransferResponses.get(0).getFullyQualifiedDomainName();
-          getInstance().transferResponse = getInstance().domainTransferResponses.get(0);
-          getInstance().extendedRegistrationExpirationTime =
-              getInstance().domainTransferResponses.get(0).getExtendedRegistrationExpirationTime();
+        if (instance.contactTransferResponses != null) {
+          instance.contactId = getInstance().contactTransferResponses.get(0).getContactId();
+          instance.transferResponse = getInstance().contactTransferResponses.get(0);
+        } else if (instance.domainTransferResponses != null) {
+          instance.fullyQualifiedDomainName =
+              instance.domainTransferResponses.get(0).getFullyQualifiedDomainName();
+          instance.transferResponse = getInstance().domainTransferResponses.get(0);
+          instance.extendedRegistrationExpirationTime =
+              instance.domainTransferResponses.get(0).getExtendedRegistrationExpirationTime();
         }
         return this;
       }

core/src/main/java/google/registry/model/poll/PollMessageExternalKeyConverter.java

@@ -24,6 +24,7 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import java.util.List;
 
 /**
@@ -78,14 +79,14 @@ public class PollMessageExternalKeyConverter {
   /**
    * Returns an Objectify Key to a PollMessage corresponding with the external ID.
    *
-   * <p>Note that the year field that is included at the end of the poll message isn't actually
-   * used for anything; it exists solely to create unique externally visible IDs for autorenews. We
-   * thus ignore it (for now) for backwards compatibility reasons, so that registrars can still ACK
+   * <p>Note that the year field that is included at the end of the poll message isn't actually used
+   * for anything; it exists solely to create unique externally visible IDs for autorenews. We thus
+   * ignore it (for now) for backwards compatibility reasons, so that registrars can still ACK
    * existing poll message IDs they may have lying around.
    *
    * @throws PollMessageExternalKeyParseException if the external key has an invalid format.
    */
-  public static Key<PollMessage> parsePollMessageExternalId(String externalKey) {
+  public static VKey<PollMessage> parsePollMessageExternalId(String externalKey) {
     List<String> idComponents = Splitter.on('-').splitToList(externalKey);
     if (idComponents.size() != 6) {
       throw new PollMessageExternalKeyParseException();
@@ -96,16 +97,17 @@ public class PollMessageExternalKeyConverter {
       if (resourceClazz == null) {
         throw new PollMessageExternalKeyParseException();
       }
-      return Key.create(
+      return VKey.from(
           Key.create(
               Key.create(
-                  null,
-                  resourceClazz,
-                  String.format("%s-%s", idComponents.get(1), idComponents.get(2))),
-              HistoryEntry.class,
-              Long.parseLong(idComponents.get(3))),
-          PollMessage.class,
-          Long.parseLong(idComponents.get(4)));
+                  Key.create(
+                      null,
+                      resourceClazz,
+                      String.format("%s-%s", idComponents.get(1), idComponents.get(2))),
+                  HistoryEntry.class,
+                  Long.parseLong(idComponents.get(3))),
+              PollMessage.class,
+              Long.parseLong(idComponents.get(4))));
       // Note that idComponents.get(5) is entirely ignored; we never use the year field internally.
     } catch (NumberFormatException e) {
       throw new PollMessageExternalKeyParseException();

core/src/main/java/google/registry/model/rde/RdeRevision.java

@@ -17,6 +17,7 @@ package google.registry.model.rde;
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.model.rde.RdeNamingUtils.makePartialName;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 
 import com.google.common.base.VerifyException;
 import com.googlecode.objectify.Key;
@@ -97,7 +98,8 @@ public final class RdeRevision extends BackupGroupRoot implements NonReplicatedE
     RdeRevisionId sqlKey = RdeRevisionId.create(tld, date.toLocalDate(), mode);
     Key<RdeRevision> ofyKey = Key.create(RdeRevision.class, id);
     Optional<RdeRevision> revisionOptional =
-        tm().loadByKeyIfPresent(VKey.create(RdeRevision.class, sqlKey, ofyKey));
+        transactIfJpaTm(
+            () -> tm().loadByKeyIfPresent(VKey.create(RdeRevision.class, sqlKey, ofyKey)));
     return revisionOptional.map(rdeRevision -> rdeRevision.revision + 1).orElse(0);
   }
 

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -72,6 +72,7 @@ import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
 import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.registrar.Registrar.BillingAccountEntry.CurrencyMapper;
@@ -112,6 +113,7 @@ import org.joda.time.DateTime;
           columnList = "ianaIdentifier",
           name = "registrar_iana_identifier_idx"),
     })
+@InCrossTld
 public class Registrar extends ImmutableObject
     implements Buildable, DatastoreAndSqlEntity, Jsonifiable {
 
@@ -651,8 +653,7 @@ public class Registrar extends ImmutableObject
       return tm().transact(
               () ->
                   jpaTm()
-                      .getEntityManager()
-                      .createQuery(
+                      .query(
                           "FROM RegistrarPoc WHERE registrarId = :registrarId",
                           RegistrarContact.class)
                       .setParameter("registrarId", clientIdentifier)
@@ -986,9 +987,7 @@ public class Registrar extends ImmutableObject
 
   /** Loads all registrar entities directly from Datastore. */
   public static Iterable<Registrar> loadAll() {
-    return tm().isOfy()
-        ? ImmutableList.copyOf(ofy().load().type(Registrar.class).ancestor(getCrossTldKey()))
-        : tm().transact(() -> tm().loadAllOf(Registrar.class));
+    return transactIfJpaTm(() -> tm().loadAllOf(Registrar.class));
   }
 
   /** Loads all registrar entities using an in-memory cache. */

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -23,6 +23,7 @@ import static com.google.common.io.BaseEncoding.base64;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.registrar.Registrar.checkValidEmail;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableSortedCopy;
 import static google.registry.util.PasswordUtils.SALT_SUPPLIER;
@@ -44,6 +45,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.JsonMapBuilder;
 import google.registry.model.Jsonifiable;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registrar.RegistrarContact.RegistrarPocId;
 import google.registry.persistence.VKey;
@@ -77,6 +79,7 @@ import javax.persistence.Transient;
       @javax.persistence.Index(columnList = "gaeUserId", name = "registrarpoc_gae_user_id_idx")
     })
 @IdClass(RegistrarPocId.class)
+@InCrossTld
 public class RegistrarContact extends ImmutableObject
     implements DatastoreAndSqlEntity, Jsonifiable {
 
@@ -198,17 +201,34 @@ public class RegistrarContact extends ImmutableObject
    * relevant Registrar entity with the {@link Registrar#contactsRequireSyncing} field set to true.
    */
   public static void updateContacts(
-      final Registrar registrar, final Set<RegistrarContact> contacts) {
+      final Registrar registrar, final ImmutableSet<RegistrarContact> contacts) {
     tm().transact(
             () -> {
-              ofy()
-                  .delete()
-                  .keys(
-                      difference(
-                          ImmutableSet.copyOf(
-                              ofy().load().type(RegistrarContact.class).ancestor(registrar).keys()),
-                          contacts.stream().map(Key::create).collect(toImmutableSet())));
-              ofy().save().entities(contacts);
+              if (tm().isOfy()) {
+                ImmutableSet<Key<RegistrarContact>> existingKeys =
+                    ImmutableSet.copyOf(
+                        ofy().load().type(RegistrarContact.class).ancestor(registrar).keys());
+                tm().delete(
+                        difference(
+                                existingKeys,
+                                contacts.stream().map(Key::create).collect(toImmutableSet()))
+                            .stream()
+                            .map(key -> VKey.createOfy(RegistrarContact.class, key))
+                            .collect(toImmutableSet()));
+              } else {
+                ImmutableSet<String> emailAddressesToKeep =
+                    contacts.stream()
+                        .map(RegistrarContact::getEmailAddress)
+                        .collect(toImmutableSet());
+                jpaTm()
+                    .query(
+                        "DELETE FROM RegistrarPoc WHERE registrarId = :registrarId AND "
+                            + "emailAddress NOT IN :emailAddressesToKeep")
+                    .setParameter("registrarId", registrar.getClientId())
+                    .setParameter("emailAddressesToKeep", emailAddressesToKeep)
+                    .executeUpdate();
+              }
+              tm().putAll(contacts);
             });
   }
 

core/src/main/java/google/registry/model/registry/Registry.java

@@ -52,6 +52,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.common.TimedTransitionProperty;
@@ -86,6 +87,7 @@ import org.joda.time.Duration;
 @ReportedOn
 @Entity
 @javax.persistence.Entity(name = "Tld")
+@InCrossTld
 public class Registry extends ImmutableObject implements Buildable, DatastoreAndSqlEntity {
 
   @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();

core/src/main/java/google/registry/model/registry/RegistryLockDao.java

@@ -20,7 +20,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import com.google.common.collect.ImmutableList;
 import google.registry.schema.domain.RegistryLock;
 import java.util.Optional;
-import javax.persistence.EntityManager;
 
 /** Data access object for {@link google.registry.schema.domain.RegistryLock}. */
 public final class RegistryLockDao {
@@ -34,15 +33,16 @@ public final class RegistryLockDao {
   /** Returns the most recent version of the {@link RegistryLock} referred to by the code. */
   public static Optional<RegistryLock> getByVerificationCode(String verificationCode) {
     jpaTm().assertInTransaction();
-    EntityManager em = jpaTm().getEntityManager();
     Long revisionId =
-        em.createQuery(
+        jpaTm()
+            .query(
                 "SELECT MAX(revisionId) FROM RegistryLock WHERE verificationCode ="
                     + " :verificationCode",
                 Long.class)
             .setParameter("verificationCode", verificationCode)
             .getSingleResult();
-    return Optional.ofNullable(revisionId).map(revision -> em.find(RegistryLock.class, revision));
+    return Optional.ofNullable(revisionId)
+        .map(revision -> jpaTm().getEntityManager().find(RegistryLock.class, revision));
   }
 
   /** Returns all lock objects that this registrar has created, including pending locks. */
@@ -50,12 +50,9 @@ public final class RegistryLockDao {
     jpaTm().assertInTransaction();
     return ImmutableList.copyOf(
         jpaTm()
-            .getEntityManager()
-            .createQuery(
-                "SELECT lock FROM RegistryLock lock"
-                    + " WHERE lock.registrarId = :registrarId"
-                    + " AND lock.unlockCompletionTimestamp IS NULL"
-                    + " ORDER BY lock.domainName ASC",
+            .query(
+                "SELECT lock FROM RegistryLock lock WHERE lock.registrarId = :registrarId"
+                    + " AND lock.unlockCompletionTime IS NULL ORDER BY lock.domainName ASC",
                 RegistryLock.class)
             .setParameter("registrarId", registrarId)
             .getResultList());
@@ -69,8 +66,7 @@ public final class RegistryLockDao {
   public static Optional<RegistryLock> getMostRecentByRepoId(String repoId) {
     jpaTm().assertInTransaction();
     return jpaTm()
-        .getEntityManager()
-        .createQuery(
+        .query(
             "SELECT lock FROM RegistryLock lock WHERE lock.repoId = :repoId"
                 + " ORDER BY lock.revisionId DESC",
             RegistryLock.class)
@@ -89,12 +85,10 @@ public final class RegistryLockDao {
   public static Optional<RegistryLock> getMostRecentVerifiedLockByRepoId(String repoId) {
     jpaTm().assertInTransaction();
     return jpaTm()
-        .getEntityManager()
-        .createQuery(
+        .query(
             "SELECT lock FROM RegistryLock lock WHERE lock.repoId = :repoId AND"
-                + " lock.lockCompletionTimestamp IS NOT NULL AND"
-                + " lock.unlockCompletionTimestamp IS NULL ORDER BY lock.revisionId"
-                + " DESC",
+                + " lock.lockCompletionTime IS NOT NULL AND lock.unlockCompletionTime IS NULL"
+                + " ORDER BY lock.revisionId DESC",
             RegistryLock.class)
         .setParameter("repoId", repoId)
         .setMaxResults(1)
@@ -111,11 +105,9 @@ public final class RegistryLockDao {
   public static Optional<RegistryLock> getMostRecentVerifiedUnlockByRepoId(String repoId) {
     jpaTm().assertInTransaction();
     return jpaTm()
-        .getEntityManager()
-        .createQuery(
+        .query(
             "SELECT lock FROM RegistryLock lock WHERE lock.repoId = :repoId AND"
-                + " lock.unlockCompletionTimestamp IS NOT NULL ORDER BY lock.revisionId"
-                + " DESC",
+                + " lock.unlockCompletionTime IS NOT NULL ORDER BY lock.revisionId DESC",
             RegistryLock.class)
         .setParameter("repoId", repoId)
         .setMaxResults(1)

core/src/main/java/google/registry/model/registry/label/BaseDomainLabelList.java

@@ -35,6 +35,7 @@ import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.ReservedList.ReservedListEntry;
@@ -59,6 +60,7 @@ import org.joda.time.DateTime;
  *     must subclass {@link DomainLabelEntry}.
  */
 @MappedSuperclass
+@InCrossTld
 public abstract class BaseDomainLabelList<T extends Comparable<?>, R extends DomainLabelEntry<T, ?>>
     extends ImmutableObject implements Buildable {
 

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -31,6 +31,7 @@ import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registry.Registry;
 import google.registry.schema.replay.DatastoreOnlyEntity;
@@ -96,6 +97,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   /** Virtual parent entity for premium list entry entities associated with a single revision. */
   @ReportedOn
   @Entity
+  @InCrossTld
   public static class PremiumListRevision extends ImmutableObject implements DatastoreOnlyEntity {
 
     @Parent Key<PremiumList> parent;
@@ -195,6 +197,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
    */
   @ReportedOn
   @Entity
+  @InCrossTld
   public static class PremiumListEntry extends DomainLabelEntry<Money, PremiumListEntry>
       implements Buildable, DatastoreOnlyEntity {
 

core/src/main/java/google/registry/model/registry/label/PremiumListDualDao.java

@@ -16,9 +16,10 @@ package google.registry.model.registry.label;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.Streams;
+import google.registry.model.DatabaseMigrationUtils;
+import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.PremiumList.PremiumListEntry;
 import google.registry.schema.tld.PremiumListSqlDao;
@@ -46,8 +47,7 @@ public class PremiumListDualDao {
    * or absent if no such list exists.
    */
   public static Optional<PremiumList> getLatestRevision(String premiumListName) {
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       return PremiumListDatastoreDao.getLatestRevision(premiumListName);
     } else {
       return PremiumListSqlDao.getLatestRevision(premiumListName);
@@ -68,16 +68,14 @@ public class PremiumListDualDao {
     }
     String premiumListName = registry.getPremiumList().getName();
     Optional<Money> primaryResult;
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       primaryResult =
           PremiumListDatastoreDao.getPremiumPrice(premiumListName, label, registry.getTldStr());
     } else {
       primaryResult = PremiumListSqlDao.getPremiumPrice(premiumListName, label);
     }
     // Also load the value from the secondary DB, compare the two results, and log if different.
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       suppressExceptionUnlessInTest(
           () -> {
             Optional<Money> secondaryResult =
@@ -120,8 +118,7 @@ public class PremiumListDualDao {
    */
   public static PremiumList save(String name, List<String> inputData) {
     PremiumList result;
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       result = PremiumListDatastoreDao.save(name, inputData);
       suppressExceptionUnlessInTest(
           () -> PremiumListSqlDao.save(name, inputData), "Error when saving premium list to SQL.");
@@ -141,8 +138,7 @@ public class PremiumListDualDao {
    * secondary database.
    */
   public static void delete(PremiumList premiumList) {
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       PremiumListDatastoreDao.delete(premiumList);
       suppressExceptionUnlessInTest(
           () -> PremiumListSqlDao.delete(premiumList),
@@ -159,8 +155,7 @@ public class PremiumListDualDao {
   public static boolean exists(String premiumListName) {
     // It may seem like overkill, but loading the list has ways been the way we check existence and
     // given that we usually load the list around the time we check existence, we'll hit the cache
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       return PremiumListDatastoreDao.getLatestRevision(premiumListName).isPresent();
     } else {
       return PremiumListSqlDao.getLatestRevision(premiumListName).isPresent();
@@ -179,8 +174,7 @@ public class PremiumListDualDao {
                 () ->
                     new IllegalArgumentException(
                         String.format("No premium list with name %s.", premiumListName)));
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       return PremiumListDatastoreDao.loadPremiumListEntriesUncached(premiumList);
     } else {
       CurrencyUnit currencyUnit = premiumList.getCurrency();

core/src/main/java/google/registry/model/registry/label/ReservedList.java

@@ -16,10 +16,10 @@ package google.registry.model.registry.label;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
 import static google.registry.model.registry.label.ReservationType.FULLY_BLOCKED;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static org.joda.time.DateTimeZone.UTC;
 
@@ -36,6 +36,7 @@ import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Mapify;
 import com.googlecode.objectify.mapper.Mapper;
 import google.registry.model.Buildable;
+import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.DomainLabelMetrics.MetricsReservedListMatch;
 import google.registry.schema.replay.NonReplicatedEntity;
@@ -63,6 +64,7 @@ import org.joda.time.DateTime;
  * revisionId. This is fine though, because we only use the list with the highest revisionId.
  */
 @Entity
+@ReportedOn
 @javax.persistence.Entity
 @Table(indexes = {@Index(columnList = "name", name = "reservedlist_name_idx")})
 public final class ReservedList
@@ -121,6 +123,12 @@ public final class ReservedList
       return new ReservedListEntry.Builder(clone(this));
     }
 
+    @Override
+    public String toString() {
+      return String.format(
+          "%s,%s%s", label, reservationType, isNullOrEmpty(comment) ? "" : " # " + comment);
+    }
+
     /** A builder for constructing {@link ReservedListEntry} objects, since they are immutable. */
     private static class Builder
         extends DomainLabelEntry.Builder<ReservedListEntry, ReservedListEntry.Builder> {
@@ -247,9 +255,7 @@ public final class ReservedList
               new CacheLoader<String, ReservedList>() {
                 @Override
                 public ReservedList load(String listName) {
-                  return tm().isOfy()
-                      ? ReservedListDualDatabaseDao.getLatestRevision(listName).orElse(null)
-                      : ReservedListSqlDao.getLatestRevision(listName).orElse(null);
+                  return ReservedListDualDatabaseDao.getLatestRevision(listName).orElse(null);
                 }
               });
 

core/src/main/java/google/registry/model/registry/label/ReservedListDatastoreDao.java

@@ -0,0 +1,50 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.registry.label;
+
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
+import com.googlecode.objectify.Key;
+import google.registry.persistence.VKey;
+import java.util.Optional;
+
+/** A {@link ReservedList} DAO for Datastore. */
+public class ReservedListDatastoreDao {
+
+  private ReservedListDatastoreDao() {}
+
+  /** Persist a new reserved list to Datastore. */
+  public static void save(ReservedList reservedList) {
+    ofyTm().transact(() -> ofyTm().put(reservedList));
+  }
+
+  /** Delete a reserved list from Datastore. */
+  public static void delete(ReservedList reservedList) {
+    ofyTm().transact(() -> ofyTm().delete(reservedList));
+  }
+
+  /**
+   * Returns the most recent revision of the {@link ReservedList} with the specified name, if it
+   * exists.
+   */
+  public static Optional<ReservedList> getLatestRevision(String reservedListName) {
+    return ofyTm()
+        .loadByKeyIfPresent(
+            VKey.createOfy(
+                ReservedList.class,
+                Key.create(getCrossTldKey(), ReservedList.class, reservedListName)));
+  }
+}

core/src/main/java/google/registry/model/registry/label/ReservedListDualDatabaseDao.java

@@ -15,43 +15,55 @@
 package google.registry.model.registry.label;
 
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
-import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.model.DatabaseMigrationUtils.isDatastore;
 
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.MapDifference.ValueDifference;
 import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.model.DatabaseMigrationUtils;
+import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
 import google.registry.model.registry.label.ReservedList.ReservedListEntry;
-import google.registry.persistence.VKey;
 import java.util.Map;
 import java.util.Optional;
 
 /**
- * A {@link ReservedList} DAO that does dual-write and dual-read against Datastore and Cloud SQL. It
- * still uses Datastore as the primary storage and suppresses any exception thrown by Cloud SQL.
+ * A {@link ReservedList} DAO that does dual-write and dual-read against Datastore and Cloud SQL.
  *
  * <p>TODO(b/160993806): Delete this DAO and switch to use the SQL only DAO after migrating to Cloud
  * SQL.
  */
 public class ReservedListDualDatabaseDao {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
   private ReservedListDualDatabaseDao() {}
 
-  /** Persist a new reserved list to Cloud SQL. */
+  /** Persist a new reserved list to the database. */
   public static void save(ReservedList reservedList) {
-    ofyTm().transact(() -> ofyTm().put(reservedList));
-      logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName());
-    DatabaseMigrationUtils.suppressExceptionUnlessInTest(
-        () -> ReservedListSqlDao.save(reservedList),
-        "Error saving the reserved list to Cloud SQL.");
-      logger.atInfo().log(
-          "Saved reserved list %s with %d entries to Cloud SQL",
-          reservedList.getName(), reservedList.getReservedListEntries().size());
+    if (isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
+      ReservedListDatastoreDao.save(reservedList);
+      DatabaseMigrationUtils.suppressExceptionUnlessInTest(
+          () -> ReservedListSqlDao.save(reservedList),
+          "Error saving the reserved list to Cloud SQL.");
+    } else {
+      ReservedListSqlDao.save(reservedList);
+      DatabaseMigrationUtils.suppressExceptionUnlessInTest(
+          () -> ReservedListDatastoreDao.save(reservedList),
+          "Error saving the reserved list to Datastore.");
+    }
+  }
+
+  /** Delete a reserved list from both databases. */
+  public static void delete(ReservedList reservedList) {
+    if (isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
+      ReservedListDatastoreDao.delete(reservedList);
+      DatabaseMigrationUtils.suppressExceptionUnlessInTest(
+          () -> ReservedListSqlDao.delete(reservedList),
+          "Error deleting the reserved list from Cloud SQL.");
+    } else {
+      ReservedListSqlDao.delete(reservedList);
+      DatabaseMigrationUtils.suppressExceptionUnlessInTest(
+          () -> ReservedListDatastoreDao.delete(reservedList),
+          "Error deleting the reserved list from Datastore.");
+    }
   }
 
   /**
@@ -59,63 +71,88 @@ public class ReservedListDualDatabaseDao {
    * exists.
    */
   public static Optional<ReservedList> getLatestRevision(String reservedListName) {
-    Optional<ReservedList> maybeDatastoreList =
-        ofyTm()
-            .loadByKeyIfPresent(
-                VKey.createOfy(
-                    ReservedList.class,
-                    Key.create(getCrossTldKey(), ReservedList.class, reservedListName)));
-    // Also load the list from Cloud SQL, compare the two lists, and log if different.
+    Optional<ReservedList> maybePrimaryList =
+        isDatastore(TransitionId.DOMAIN_LABEL_LISTS)
+            ? ReservedListDatastoreDao.getLatestRevision(reservedListName)
+            : ReservedListSqlDao.getLatestRevision(reservedListName);
     DatabaseMigrationUtils.suppressExceptionUnlessInTest(
-        () -> maybeDatastoreList.ifPresent(ReservedListDualDatabaseDao::loadAndCompareCloudSqlList),
+        () -> maybePrimaryList.ifPresent(primaryList -> loadAndCompare(primaryList)),
         "Error comparing reserved lists.");
-    return maybeDatastoreList;
+    return maybePrimaryList;
   }
 
-  private static void loadAndCompareCloudSqlList(ReservedList datastoreList) {
-    Optional<ReservedList> maybeCloudSqlList =
-        ReservedListSqlDao.getLatestRevision(datastoreList.getName());
-    if (maybeCloudSqlList.isPresent()) {
-      Map<String, ReservedListEntry> datastoreLabelsToReservations =
-          datastoreList.reservedListMap.entrySet().parallelStream()
-              .collect(
-                  toImmutableMap(
-                      Map.Entry::getKey,
-                      entry ->
-                          ReservedListEntry.create(
-                              entry.getKey(),
-                              entry.getValue().reservationType,
-                              entry.getValue().comment)));
+  private static void loadAndCompare(ReservedList primaryList) {
+    Optional<ReservedList> maybeSecondaryList =
+        isDatastore(TransitionId.DOMAIN_LABEL_LISTS)
+            ? ReservedListSqlDao.getLatestRevision(primaryList.getName())
+            : ReservedListDatastoreDao.getLatestRevision(primaryList.getName());
+    if (!maybeSecondaryList.isPresent()) {
+      throw new IllegalStateException(
+          String.format(
+              "Reserved list in the secondary database (%s) is empty.",
+              isDatastore(TransitionId.DOMAIN_LABEL_LISTS) ? "Cloud SQL" : "Datastore"));
+    }
+    Map<String, ReservedListEntry> labelsToReservations =
+        primaryList.reservedListMap.entrySet().parallelStream()
+            .collect(
+                toImmutableMap(
+                    Map.Entry::getKey,
+                    entry ->
+                        ReservedListEntry.create(
+                            entry.getKey(),
+                            entry.getValue().reservationType,
+                            entry.getValue().comment)));
 
-      ReservedList cloudSqlList = maybeCloudSqlList.get();
-      MapDifference<String, ReservedListEntry> diff =
-          Maps.difference(datastoreLabelsToReservations, cloudSqlList.reservedListMap);
+    ReservedList secondaryList = maybeSecondaryList.get();
+    MapDifference<String, ReservedListEntry> diff =
+        Maps.difference(labelsToReservations, secondaryList.reservedListMap);
       if (!diff.areEqual()) {
         if (diff.entriesDiffering().size() > 10) {
-          throw new IllegalStateException(
-              String.format(
-                  "Unequal reserved lists detected, Cloud SQL list with revision"
-                      + " id %d has %d different records than the current"
-                      + " Datastore list.",
-                  cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
-        } else {
-          StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
-          diff.entriesDiffering().entrySet().stream()
-              .forEach(
-                  entry -> {
-                    String label = entry.getKey();
-                    ValueDifference<ReservedListEntry> valueDiff = entry.getValue();
-                    diffMessage.append(
-                        String.format(
-                            "Domain label %s has entry %s in Datastore and entry"
-                                + " %s in Cloud SQL.\n",
-                            label, valueDiff.leftValue(), valueDiff.rightValue()));
-                  });
-          throw new IllegalStateException(diffMessage.toString());
-        }
+        throw new IllegalStateException(
+            String.format(
+                "Unequal reserved lists detected, %s list with revision"
+                    + " id %d has %d different records than the current"
+                    + " primary database list.",
+                isDatastore(TransitionId.DOMAIN_LABEL_LISTS) ? "Cloud SQL" : "Datastore",
+                secondaryList.getRevisionId(),
+                diff.entriesDiffering().size()));
+      }
+          StringBuilder diffMessage = new StringBuilder("Unequal reserved lists detected:\n");
+      diff.entriesDiffering().entrySet().stream()
+          .forEach(
+              entry -> {
+                String label = entry.getKey();
+                ValueDifference<ReservedListEntry> valueDiff = entry.getValue();
+                diffMessage.append(
+                    String.format(
+                        "Domain label %s has entry %s in %s and entry"
+                            + " %s in the secondary database.\n",
+                        label,
+                        valueDiff.leftValue(),
+                        isDatastore(TransitionId.DOMAIN_LABEL_LISTS) ? "Datastore" : "Cloud SQL",
+                        valueDiff.rightValue()));
+              });
+      diff.entriesOnlyOnLeft().entrySet().stream()
+          .forEach(
+              entry -> {
+                String label = entry.getKey();
+                diffMessage.append(
+                    String.format(
+                        "Domain label %s has entry in %s, but not in the secondary database.\n",
+                        label,
+                        isDatastore(TransitionId.DOMAIN_LABEL_LISTS) ? "Datastore" : "Cloud SQL"));
+              });
+      diff.entriesOnlyOnRight().entrySet().stream()
+          .forEach(
+              entry -> {
+                String label = entry.getKey();
+                diffMessage.append(
+                    String.format(
+                        "Domain label %s has entry in %s, but not in the primary database.\n",
+                        label,
+                        isDatastore(TransitionId.DOMAIN_LABEL_LISTS) ? "Cloud SQL" : "Datastore"));
+              });
+          throw new IllegalStateException(diffMessage.toString());
       }
-    } else {
-      throw new IllegalStateException("Reserved list in Cloud SQL is empty.");
-    }
   }
 }

core/src/main/java/google/registry/model/registry/label/ReservedListSqlDao.java

@@ -17,6 +17,7 @@ package google.registry.model.registry.label;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
+import com.google.common.flogger.FluentLogger;
 import java.util.Optional;
 
 /**
@@ -26,12 +27,23 @@ import java.util.Optional;
  */
 public class ReservedListSqlDao {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   private ReservedListSqlDao() {}
 
   /** Persist a new reserved list to Cloud SQL. */
   public static void save(ReservedList reservedList) {
     checkArgumentNotNull(reservedList, "Must specify reservedList");
+    logger.atInfo().log("Saving reserved list %s to Cloud SQL", reservedList.getName());
     jpaTm().transact(() -> jpaTm().insert(reservedList));
+    logger.atInfo().log(
+        "Saved reserved list %s with %d entries to Cloud SQL",
+        reservedList.getName(), reservedList.getReservedListEntries().size());
+  }
+
+  /** Deletes a reserved list from Cloud SQL. */
+  public static void delete(ReservedList reservedList) {
+    jpaTm().transact(() -> jpaTm().delete(reservedList));
   }
 
   /**
@@ -43,8 +55,7 @@ public class ReservedListSqlDao {
         .transact(
             () ->
                 jpaTm()
-                    .getEntityManager()
-                    .createQuery(
+                    .query(
                         "FROM ReservedList rl LEFT JOIN FETCH rl.reservedListMap WHERE"
                             + " rl.revisionId IN (SELECT MAX(revisionId) FROM ReservedList subrl"
                             + " WHERE subrl.name = :name)",
@@ -64,8 +75,7 @@ public class ReservedListSqlDao {
         .transact(
             () ->
                 jpaTm()
-                        .getEntityManager()
-                        .createQuery("SELECT 1 FROM ReservedList WHERE name = :name", Integer.class)
+                        .query("SELECT 1 FROM ReservedList WHERE name = :name", Integer.class)
                         .setParameter("name", reservedListName)
                         .setMaxResults(1)
                         .getResultList()

core/src/main/java/google/registry/model/reporting/HistoryEntryDao.java

@@ -21,7 +21,9 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
+import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
 import google.registry.model.EppResource;
 import google.registry.model.contact.ContactHistory;
 import google.registry.model.contact.ContactResource;
@@ -30,8 +32,11 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.host.HostHistory;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import java.util.Comparator;
-import javax.persistence.EntityManager;
+import java.util.stream.Stream;
+import javax.persistence.criteria.CriteriaBuilder;
+import javax.persistence.criteria.CriteriaQuery;
 import org.joda.time.DateTime;
 
 /**
@@ -86,22 +91,56 @@ public class HistoryEntryDao {
     }
   }
 
+  /** Loads all history objects from all time from the given registrars. */
+  public static Iterable<? extends HistoryEntry> loadHistoryObjectsByRegistrars(
+      ImmutableCollection<String> registrarIds) {
+    if (tm().isOfy()) {
+      return ofy()
+          .load()
+          .type(HistoryEntry.class)
+          .filter("clientId in", registrarIds)
+          .order("modificationTime");
+    } else {
+      return jpaTm()
+          .transact(
+              () ->
+                  Streams.concat(
+                          loadHistoryObjectFromSqlByRegistrars(ContactHistory.class, registrarIds),
+                          loadHistoryObjectFromSqlByRegistrars(DomainHistory.class, registrarIds),
+                          loadHistoryObjectFromSqlByRegistrars(HostHistory.class, registrarIds))
+                      .sorted(Comparator.comparing(HistoryEntry::getModificationTime))
+                      .collect(toImmutableList()));
+    }
+  }
+
+  private static Stream<? extends HistoryEntry> loadHistoryObjectFromSqlByRegistrars(
+      Class<? extends HistoryEntry> historyClass, ImmutableCollection<String> registrarIds) {
+    return jpaTm()
+        .getEntityManager()
+        .createQuery(
+            CriteriaQueryBuilder.create(historyClass)
+                .whereFieldIsIn("clientId", registrarIds)
+                .build())
+        .getResultStream();
+  }
+
   private static Iterable<? extends HistoryEntry> loadHistoryObjectsForResourceFromSql(
       VKey<? extends EppResource> parentKey, DateTime afterTime, DateTime beforeTime) {
+    // The class we're searching from is based on which parent type (e.g. Domain) we have
     Class<? extends HistoryEntry> historyClass = getHistoryClassFromParent(parentKey.getKind());
+    // The field representing repo ID unfortunately varies by history class
     String repoIdFieldName = getRepoIdFieldNameFromHistoryClass(historyClass);
-    EntityManager entityManager = jpaTm().getEntityManager();
-    String tableName = entityManager.getMetamodel().entity(historyClass).getName();
-    String queryString =
-        String.format(
-            "SELECT entry FROM %s entry WHERE entry.modificationTime >= :afterTime AND "
-                + "entry.modificationTime <= :beforeTime AND entry.%s = :parentKey",
-            tableName, repoIdFieldName);
-    return entityManager
-        .createQuery(queryString, historyClass)
-        .setParameter("afterTime", afterTime)
-        .setParameter("beforeTime", beforeTime)
-        .setParameter("parentKey", parentKey.getSqlKey().toString())
+    CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
+    CriteriaQuery<? extends HistoryEntry> criteriaQuery =
+        CriteriaQueryBuilder.create(historyClass)
+            .where("modificationTime", criteriaBuilder::greaterThanOrEqualTo, afterTime)
+            .where("modificationTime", criteriaBuilder::lessThanOrEqualTo, beforeTime)
+            .where(repoIdFieldName, criteriaBuilder::equal, parentKey.getSqlKey().toString())
+            .build();
+
+    return jpaTm()
+        .getEntityManager()
+        .createQuery(criteriaQuery)
         .getResultStream()
         .sorted(Comparator.comparing(HistoryEntry::getModificationTime))
         .collect(toImmutableList());
@@ -129,16 +168,14 @@ public class HistoryEntryDao {
 
   private static Iterable<? extends HistoryEntry> loadAllHistoryObjectsFromSql(
       Class<? extends HistoryEntry> historyClass, DateTime afterTime, DateTime beforeTime) {
-    EntityManager entityManager = jpaTm().getEntityManager();
-    return entityManager
+    CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
+    return jpaTm()
+        .getEntityManager()
         .createQuery(
-            String.format(
-                "SELECT entry FROM %s entry WHERE entry.modificationTime >= :afterTime AND "
-                    + "entry.modificationTime <= :beforeTime",
-                entityManager.getMetamodel().entity(historyClass).getName()),
-            historyClass)
-        .setParameter("afterTime", afterTime)
-        .setParameter("beforeTime", beforeTime)
+            CriteriaQueryBuilder.create(historyClass)
+                .where("modificationTime", criteriaBuilder::greaterThanOrEqualTo, afterTime)
+                .where("modificationTime", criteriaBuilder::lessThanOrEqualTo, beforeTime)
+                .build())
         .getResultList();
   }
 }

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatchDao.java

@@ -32,8 +32,7 @@ public class Spec11ThreatMatchDao {
   public static void deleteEntriesByDate(JpaTransactionManager jpaTm, LocalDate date) {
     jpaTm.assertInTransaction();
     jpaTm
-        .getEntityManager()
-        .createQuery("DELETE FROM Spec11ThreatMatch WHERE check_date = :date")
+        .query("DELETE FROM Spec11ThreatMatch WHERE check_date = :date")
         .setParameter("date", DateTimeUtils.toSqlDate(date), TemporalType.DATE)
         .executeUpdate();
   }
@@ -44,8 +43,7 @@ public class Spec11ThreatMatchDao {
     jpaTm.assertInTransaction();
     return ImmutableList.copyOf(
         jpaTm
-            .getEntityManager()
-            .createQuery(
+            .query(
                 "SELECT match FROM Spec11ThreatMatch match WHERE match.checkDate = :date",
                 Spec11ThreatMatch.class)
             .setParameter("date", date)

core/src/main/java/google/registry/model/server/KmsSecret.java

@@ -21,6 +21,7 @@ import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
 import google.registry.schema.replay.DatastoreOnlyEntity;
@@ -28,6 +29,7 @@ import google.registry.schema.replay.DatastoreOnlyEntity;
 /** Pointer to the latest {@link KmsSecretRevision}. */
 @Entity
 @ReportedOn
+@InCrossTld
 public class KmsSecret extends ImmutableObject implements DatastoreOnlyEntity {
 
   /** The unique name of this {@link KmsSecret}. */

core/src/main/java/google/registry/model/server/KmsSecretRevision.java

@@ -26,6 +26,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.schema.replay.NonReplicatedEntity;
 import javax.persistence.Column;
@@ -58,6 +59,7 @@ import javax.persistence.Transient;
 @ReportedOn
 @javax.persistence.Entity(name = "KmsSecret")
 @Table(indexes = {@Index(columnList = "secretName")})
+@InCrossTld
 public class KmsSecretRevision extends ImmutableObject implements NonReplicatedEntity {
 
   /**

core/src/main/java/google/registry/model/server/KmsSecretRevisionSqlDao.java

@@ -42,8 +42,7 @@ public class KmsSecretRevisionSqlDao {
     checkArgument(!isNullOrEmpty(secretName), "secretName cannot be null or empty");
     jpaTm().assertInTransaction();
     return jpaTm()
-        .getEntityManager()
-        .createQuery(
+        .query(
             "FROM KmsSecret ks WHERE ks.revisionKey IN (SELECT MAX(revisionKey) FROM "
                 + "KmsSecret subKs WHERE subKs.secretName = :secretName)",
             KmsSecretRevision.class)

core/src/main/java/google/registry/model/server/Lock.java

@@ -15,19 +15,20 @@
 package google.registry.model.server;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.flogger.FluentLogger;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
@@ -190,12 +191,17 @@ public class Lock extends ImmutableObject implements DatastoreOnlyEntity, Serial
     // access to resources like GCS that can't be transactionally rolled back. Therefore, the lock
     // must be definitively acquired before it is used, even when called inside another transaction.
     AcquireResult acquireResult =
-        tm().transactNew(
+        ofyTm()
+            .transactNew(
                 () -> {
-                  DateTime now = tm().getTransactionTime();
+                  DateTime now = ofyTm().getTransactionTime();
 
                   // Checking if an unexpired lock still exists - if so, the lock can't be acquired.
-                  Lock lock = ofy().load().type(Lock.class).id(lockId).now();
+                  Lock lock =
+                      ofyTm()
+                          .loadByKeyIfPresent(
+                              VKey.createOfy(Lock.class, Key.create(Lock.class, lockId)))
+                          .orElse(null);
                   if (lock != null) {
                     logger.atInfo().log(
                         "Loaded existing lock: %s for request: %s", lock.lockId, lock.requestLogId);
@@ -218,7 +224,7 @@ public class Lock extends ImmutableObject implements DatastoreOnlyEntity, Serial
                   // Locks are not parented under an EntityGroupRoot (so as to avoid write
                   // contention) and
                   // don't need to be backed up.
-                  ofy().saveWithoutBackup().entity(newLock);
+                  ofyTm().putWithoutBackup(newLock);
 
                   return AcquireResult.create(now, lock, newLock, lockState);
                 });
@@ -231,21 +237,26 @@ public class Lock extends ImmutableObject implements DatastoreOnlyEntity, Serial
   /** Release the lock. */
   public void release() {
     // Just use the default clock because we aren't actually doing anything that will use the clock.
-    tm().transact(
+    ofyTm()
+        .transact(
             () -> {
               // To release a lock, check that no one else has already obtained it and if not
               // delete it. If the lock in Datastore was different then this lock is gone already;
               // this can happen if release() is called around the expiration time and the lock
               // expires underneath us.
-              Lock loadedLock = ofy().load().type(Lock.class).id(lockId).now();
+              Lock loadedLock =
+                  ofyTm()
+                      .loadByKeyIfPresent(
+                          VKey.createOfy(Lock.class, Key.create(Lock.class, lockId)))
+                      .orElse(null);
               if (Lock.this.equals(loadedLock)) {
                 // Use noBackupOfy() so that we don't create a commit log entry for deleting the
                 // lock.
                 logger.atInfo().log("Deleting lock: %s", lockId);
-                ofy().deleteWithoutBackup().entity(Lock.this);
+                ofyTm().deleteWithoutBackup(Lock.this);
 
                 lockMetrics.recordRelease(
-                    resourceName, tld, new Duration(acquiredTime, tm().getTransactionTime()));
+                    resourceName, tld, new Duration(acquiredTime, ofyTm().getTransactionTime()));
               } else {
                 logger.atSevere().log(
                     "The lock we acquired was transferred to someone else before we"

core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java

@@ -30,6 +30,7 @@ import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.OnSave;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.EntityGroupRoot;
@@ -67,6 +68,7 @@ import org.joda.time.DateTime;
 @Entity
 @javax.persistence.Entity
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
+@InCrossTld
 public class SignedMarkRevocationList extends ImmutableObject implements NonReplicatedEntity {
 
   @VisibleForTesting static final int SHARD_SIZE = 10000;

core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java

@@ -25,6 +25,7 @@ import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.smd.SignedMarkRevocationList.SHARD_SIZE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -40,7 +41,6 @@ import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
 import google.registry.util.CollectionUtils;
 import java.util.Map;
 import java.util.Optional;
-import javax.persistence.EntityManager;
 import org.joda.time.DateTime;
 
 public class SignedMarkRevocationListDao {
@@ -128,7 +128,8 @@ public class SignedMarkRevocationListDao {
 
   /** Loads the shards from Datastore and combines them into one list. */
   private static Optional<SignedMarkRevocationList> loadFromDatastore() {
-    return tm().transactNewReadOnly(
+    return ofyTm()
+        .transactNewReadOnly(
             () -> {
               Iterable<SignedMarkRevocationList> shards =
                   ofy().load().type(SignedMarkRevocationList.class).ancestor(getCrossTldKey());
@@ -153,11 +154,12 @@ public class SignedMarkRevocationListDao {
     return jpaTm()
         .transact(
             () -> {
-              EntityManager em = jpaTm().getEntityManager();
               Long revisionId =
-                  em.createQuery("SELECT MAX(revisionId) FROM SignedMarkRevocationList", Long.class)
+                  jpaTm()
+                      .query("SELECT MAX(revisionId) FROM SignedMarkRevocationList", Long.class)
                       .getSingleResult();
-              return em.createQuery(
+              return jpaTm()
+                  .query(
                       "FROM SignedMarkRevocationList smrl LEFT JOIN FETCH smrl.revokes "
                           + "WHERE smrl.revisionId = :revisionId",
                       SignedMarkRevocationList.class)
@@ -196,7 +198,7 @@ public class SignedMarkRevocationListDao {
   }
 
   private static void saveToCloudSql(SignedMarkRevocationList signedMarkRevocationList) {
-    jpaTm().transact(() -> jpaTm().getEntityManager().persist(signedMarkRevocationList));
+    jpaTm().transact(() -> jpaTm().insert(signedMarkRevocationList));
     logger.atInfo().log(
         "Inserted %,d signed mark revocations into Cloud SQL.",
         signedMarkRevocationList.revokes.size());

core/src/main/java/google/registry/model/tmch/ClaimsListDao.java

@@ -1,89 +0,0 @@
-// Copyright 2019 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.model.tmch;
-
-import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
-import static google.registry.model.CacheUtils.tryMemoizeWithExpiration;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-
-import com.google.common.base.Supplier;
-import com.google.common.flogger.FluentLogger;
-import google.registry.util.NonFinalForTesting;
-import java.util.Optional;
-import javax.persistence.EntityManager;
-
-/** Data access object for {@link ClaimsListShard}. */
-public class ClaimsListDao {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
-  /** In-memory cache for claims list. */
-  @NonFinalForTesting
-  private static Supplier<Optional<ClaimsListShard>> cacheClaimsList =
-      tryMemoizeWithExpiration(getDomainLabelListCacheDuration(), ClaimsListDao::getLatestRevision);
-
-  private static void save(ClaimsListShard claimsList) {
-    jpaTm().transact(() -> jpaTm().getEntityManager().persist(claimsList));
-  }
-
-  /**
-   * Try to save the given {@link ClaimsListShard} into Cloud SQL. If the save fails, the error will
-   * be logged but no exception will be thrown.
-   *
-   * <p>This method is used during the dual-write phase of database migration as Datastore is still
-   * the authoritative database.
-   */
-  static void trySave(ClaimsListShard claimsList) {
-    try {
-      ClaimsListDao.save(claimsList);
-      logger.atInfo().log(
-          "Inserted %,d claims into Cloud SQL, created at %s",
-          claimsList.getLabelsToKeys().size(), claimsList.getTmdbGenerationTime());
-    } catch (Throwable e) {
-      logger.atSevere().withCause(e).log("Error inserting claims into Cloud SQL");
-    }
-  }
-
-  /**
-   * Returns the most recent revision of the {@link ClaimsListShard} in Cloud SQL, if it exists.
-   * TODO(b/177569979): Change this method to package level access after dual-read phase.
-   * ClaimsListShard uses this method to retrieve claims list in Cloud SQL for the comparison, and
-   * ClaimsListShard is not in this package.
-   */
-  public static Optional<ClaimsListShard> getLatestRevision() {
-    return jpaTm()
-        .transact(
-            () -> {
-              EntityManager em = jpaTm().getEntityManager();
-              Long revisionId =
-                  em.createQuery("SELECT MAX(revisionId) FROM ClaimsList", Long.class)
-                      .getSingleResult();
-              return em.createQuery(
-                      "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
-                          + " :revisionId",
-                      ClaimsListShard.class)
-                  .setParameter("revisionId", revisionId)
-                  .getResultStream()
-                  .findFirst();
-            });
-  }
-
-  /** Returns the most recent revision of the {@link ClaimsListShard}, from cache. */
-  public static Optional<ClaimsListShard> getLatestRevisionCached() {
-    return cacheClaimsList.get();
-  }
-
-  private ClaimsListDao() {}
-}

core/src/main/java/google/registry/model/tmch/ClaimsListDualDatabaseDao.java

@@ -0,0 +1,148 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.tmch;
+
+import static google.registry.config.RegistryConfig.getDomainLabelListCacheDuration;
+import static google.registry.model.CacheUtils.tryMemoizeWithExpiration;
+import static google.registry.model.DatabaseMigrationUtils.isDatastore;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
+import static google.registry.model.common.DatabaseTransitionSchedule.TransitionId.CLAIMS_LIST;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import google.registry.util.NonFinalForTesting;
+import java.util.Optional;
+
+/**
+ * DAO for {@link ClaimsListShard} objects that handles the branching paths for SQL and Datastore.
+ *
+ * <p>For write actions, this class will perform the action against the primary database then, after
+ * * that success or failure, against the secondary database. If the secondary database fails, an
+ * error is logged (but not thrown).
+ *
+ * <p>For read actions, we will log if the primary and secondary databases * have different values
+ * (or if the retrieval from the second database fails).
+ */
+public class ClaimsListDualDatabaseDao {
+
+  /** In-memory cache for claims list. */
+  @NonFinalForTesting
+  private static Supplier<ClaimsListShard> claimsListCache =
+      tryMemoizeWithExpiration(
+          getDomainLabelListCacheDuration(), ClaimsListDualDatabaseDao::getUncached);
+
+  /**
+   * Saves the given {@link ClaimsListShard} to both the primary and secondary databases, logging
+   * and skipping errors in the secondary DB.
+   */
+  public static void save(ClaimsListShard claimsList) {
+    if (isDatastore(CLAIMS_LIST)) {
+      claimsList.saveToDatastore();
+      suppressExceptionUnlessInTest(
+          () -> ClaimsListSqlDao.save(claimsList), "Error saving ClaimsList to SQL.");
+    } else {
+      ClaimsListSqlDao.save(claimsList);
+      suppressExceptionUnlessInTest(
+          claimsList::saveToDatastore, "Error saving ClaimsListShard to Datastore.");
+    }
+  }
+
+  /** Returns the most recent revision of the {@link ClaimsListShard}, from cache. */
+  public static ClaimsListShard get() {
+    return claimsListCache.get();
+  }
+
+  /** Retrieves and compares the latest revision from the databases. */
+  private static ClaimsListShard getUncached() {
+    Optional<ClaimsListShard> primaryResult;
+    if (isDatastore(CLAIMS_LIST)) {
+      primaryResult = ClaimsListShard.getFromDatastore();
+      suppressExceptionUnlessInTest(
+          () -> {
+            Optional<ClaimsListShard> secondaryResult = ClaimsListSqlDao.get();
+            compareClaimsLists(primaryResult, secondaryResult);
+          },
+          "Error loading ClaimsList from SQL.");
+    } else {
+      primaryResult = ClaimsListSqlDao.get();
+      suppressExceptionUnlessInTest(
+          () -> {
+            Optional<ClaimsListShard> secondaryResult = ClaimsListShard.getFromDatastore();
+            compareClaimsLists(primaryResult, secondaryResult);
+          },
+          "Error loading ClaimsListShard from Datastore.");
+    }
+    return primaryResult.orElse(ClaimsListShard.create(START_OF_TIME, ImmutableMap.of()));
+  }
+
+  private static void compareClaimsLists(
+      Optional<ClaimsListShard> maybePrimary, Optional<ClaimsListShard> maybeSecondary) {
+    if (maybePrimary.isPresent() && !maybeSecondary.isPresent()) {
+      throw new IllegalStateException("Claims list found in primary DB but not in secondary DB.");
+    }
+    if (!maybePrimary.isPresent() && maybeSecondary.isPresent()) {
+      throw new IllegalStateException("Claims list found in secondary DB but not in primary DB.");
+    }
+    if (!maybePrimary.isPresent()) {
+      return;
+    }
+    ClaimsListShard primary = maybePrimary.get();
+    ClaimsListShard secondary = maybeSecondary.get();
+    MapDifference<String, String> diff =
+        Maps.difference(primary.labelsToKeys, secondary.getLabelsToKeys());
+    if (!diff.areEqual()) {
+      if (diff.entriesDiffering().size()
+              + diff.entriesOnlyOnRight().size()
+              + diff.entriesOnlyOnLeft().size()
+          > 10) {
+        throw new IllegalStateException(
+            String.format(
+                "Unequal claims lists detected, secondary list with revision id %d has %d"
+                    + " different records than the current primary list.",
+                secondary.getRevisionId(), diff.entriesDiffering().size()));
+      } else {
+        StringBuilder diffMessage = new StringBuilder("Unequal claims lists detected:\n");
+        diff.entriesDiffering()
+            .forEach(
+                (label, valueDiff) ->
+                    diffMessage.append(
+                        String.format(
+                            "Domain label %s has key %s in the primary DB and key %s "
+                                + "in the secondary DB.\n",
+                            label, valueDiff.leftValue(), valueDiff.rightValue())));
+        diff.entriesOnlyOnLeft()
+            .forEach(
+                (label, valueDiff) ->
+                    diffMessage.append(
+                        String.format(
+                            "Domain label %s with key %s only appears in the primary DB.\n",
+                            label, valueDiff)));
+        diff.entriesOnlyOnRight()
+            .forEach(
+                (label, valueDiff) ->
+                    diffMessage.append(
+                        String.format(
+                            "Domain label %s with key %s only appears in the secondary DB.\n",
+                            label, valueDiff)));
+        throw new IllegalStateException(diffMessage.toString());
+      }
+    }
+  }
+
+  private ClaimsListDualDatabaseDao() {}
+}

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -18,19 +18,13 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Throwables.throwIfUnchecked;
 import static com.google.common.base.Verify.verify;
-import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.MapDifference.ValueDifference;
-import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.google.common.util.concurrent.UncheckedExecutionException;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EmbedMap;
@@ -41,6 +35,7 @@ import com.googlecode.objectify.annotation.OnSave;
 import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
+import google.registry.model.annotations.InCrossTld;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.annotations.VirtualEntity;
@@ -71,8 +66,8 @@ import org.joda.time.DateTime;
  * A list of TMCH claims labels and their associated claims keys.
  *
  * <p>The claims list is actually sharded into multiple {@link ClaimsListShard} entities to work
- * around the Datastore limitation of 1M max size per entity. However, when calling {@link #get} all
- * of the shards are recombined into one {@link ClaimsListShard} object.
+ * around the Datastore limitation of 1M max size per entity. However, when calling {@link
+ * #getFromDatastore} all of the shards are recombined into one {@link ClaimsListShard} object.
  *
  * <p>ClaimsList shards are tied to a specific revision and are persisted individually, then the
  * entire claims list is atomically shifted over to using the new shards by persisting the new
@@ -95,10 +90,9 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 @javax.persistence.Entity(name = "ClaimsList")
 @Table
+@InCrossTld
 public class ClaimsListShard extends ImmutableObject implements NonReplicatedEntity {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
   /** The number of claims list entries to store per shard. */
   private static final int SHARD_SIZE = 10000;
 
@@ -143,107 +137,58 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
 
   private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);
 
-  private static ClaimsListShard loadClaimsListShard() {
+  private static Optional<ClaimsListShard> loadClaimsListShard() {
     // Find the most recent revision.
     Key<ClaimsListRevision> revisionKey = getCurrentRevision();
+    if (revisionKey == null) {
+      return Optional.empty();
+    }
 
     Map<String, String> combinedLabelsToKeys = new HashMap<>();
     DateTime creationTime = START_OF_TIME;
-    if (revisionKey != null) {
-      // Grab all of the keys for the shards that belong to the current revision.
-      final List<Key<ClaimsListShard>> shardKeys =
-          ofy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list();
+    // Grab all of the keys for the shards that belong to the current revision.
+    final List<Key<ClaimsListShard>> shardKeys =
+        ofy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list();
 
-      List<ClaimsListShard> shards;
-      try {
-        // Load all of the shards concurrently, each in a separate transaction.
-        shards =
-            Concurrent.transform(
-                shardKeys,
-                key ->
-                    tm().transactNewReadOnly(
-                            () -> {
-                              ClaimsListShard claimsListShard = ofy().load().key(key).now();
-                              checkState(
-                                  claimsListShard != null,
-                                  "Key not found when loading claims list shards.");
-                              return claimsListShard;
-                            }));
-      } catch (UncheckedExecutionException e) {
-        // We retry on IllegalStateException. However, there's a checkState inside the
-        // Concurrent.transform, so if it's thrown it'll be wrapped in an
-        // UncheckedExecutionException. We want to unwrap it so it's caught by the retrier.
-        if (e.getCause() != null) {
-          throwIfUnchecked(e.getCause());
-        }
-        throw e;
-      }
-
-      // Combine the shards together and return the concatenated ClaimsList.
-      if (!shards.isEmpty()) {
-        creationTime = shards.get(0).creationTime;
-        for (ClaimsListShard shard : shards) {
-          combinedLabelsToKeys.putAll(shard.labelsToKeys);
-          checkState(
-              creationTime.equals(shard.creationTime),
-              "Inconsistent claims list shard creation times.");
-        }
-      }
-    }
-
-    ClaimsListShard datastoreList = create(creationTime, ImmutableMap.copyOf(combinedLabelsToKeys));
-    // Also load the list from Cloud SQL, compare the two lists, and log if different.
+    List<ClaimsListShard> shards;
     try {
-      loadAndCompareCloudSqlList(datastoreList);
-    } catch (Throwable t) {
-      logger.atSevere().withCause(t).log("Error comparing claims lists.");
-    }
-    return datastoreList;
-  };
-
-  private static void loadAndCompareCloudSqlList(ClaimsListShard datastoreList) {
-    Optional<ClaimsListShard> maybeCloudSqlList = ClaimsListDao.getLatestRevision();
-    if (maybeCloudSqlList.isPresent()) {
-      ClaimsListShard cloudSqlList = maybeCloudSqlList.get();
-      MapDifference<String, String> diff =
-          Maps.difference(datastoreList.labelsToKeys, cloudSqlList.getLabelsToKeys());
-      if (!diff.areEqual()) {
-        if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
-              String.format(
-                  "Unequal claims lists detected, Cloud SQL list with revision id %d has %d"
-                      + " different records than the current Datastore list.",
-                  cloudSqlList.getRevisionId(), diff.entriesDiffering().size()));
-        } else {
-          StringBuilder diffMessage = new StringBuilder("Unequal claims lists detected:\n");
-          diff.entriesDiffering().entrySet().stream()
-              .forEach(
-                  entry -> {
-                    String label = entry.getKey();
-                    ValueDifference<String> valueDiff = entry.getValue();
-                    diffMessage.append(
-                        String.format(
-                            "Domain label %s has key %s in Datastore and key %s in Cloud"
-                                + " SQL.\n",
-                            label, valueDiff.leftValue(), valueDiff.rightValue()));
-                  });
-          logger.atWarning().log(diffMessage.toString());
-        }
+      // Load all of the shards concurrently, each in a separate transaction.
+      shards =
+          Concurrent.transform(
+              shardKeys,
+              key ->
+                  ofyTm()
+                      .transactNewReadOnly(
+                          () -> {
+                            ClaimsListShard claimsListShard = ofy().load().key(key).now();
+                            checkState(
+                                claimsListShard != null,
+                                "Key not found when loading claims list shards.");
+                            return claimsListShard;
+                          }));
+    } catch (UncheckedExecutionException e) {
+      // We retry on IllegalStateException. However, there's a checkState inside the
+      // Concurrent.transform, so if it's thrown it'll be wrapped in an
+      // UncheckedExecutionException. We want to unwrap it so it's caught by the retrier.
+      if (e.getCause() != null) {
+        throwIfUnchecked(e.getCause());
       }
-    } else {
-      logger.atWarning().log("Claims list in Cloud SQL is empty.");
+      throw e;
     }
-  }
 
-  /**
-   * A cached supplier that fetches the claims list shards from Datastore and recombines them into a
-   * single {@link ClaimsListShard} object.
-   */
-  private static final Supplier<ClaimsListShard> CACHE =
-      memoizeWithShortExpiration(
-          () ->
-              LOADER_RETRIER.callWithRetry(
-                  ClaimsListShard::loadClaimsListShard, IllegalStateException.class));
+    // Combine the shards together and return the concatenated ClaimsList.
+    if (!shards.isEmpty()) {
+      creationTime = shards.get(0).creationTime;
+      for (ClaimsListShard shard : shards) {
+        combinedLabelsToKeys.putAll(shard.labelsToKeys);
+        checkState(
+            creationTime.equals(shard.creationTime),
+            "Inconsistent claims list shard creation times.");
+      }
+    }
+
+    return Optional.of(create(creationTime, ImmutableMap.copyOf(combinedLabelsToKeys)));
+  }
 
   /** Returns the revision id of this claims list, or throws exception if it is null. */
   public Long getRevisionId() {
@@ -286,9 +231,8 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
    * Save the Claims list to Datastore by writing the new shards in a series of transactions,
    * switching over to using them atomically, then deleting the old ones.
    */
-  public void save() {
+  void saveToDatastore() {
     saveToDatastore(SHARD_SIZE);
-    ClaimsListDao.trySave(this);
   }
 
   @VisibleForTesting
@@ -301,7 +245,8 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
     Concurrent.transform(
         CollectionUtils.partitionMap(labelsToKeys, shardSize),
         (final ImmutableMap<String, String> labelsToKeysShard) ->
-            tm().transactNew(
+            ofyTm()
+                .transact(
                     () -> {
                       ClaimsListShard shard = create(creationTime, labelsToKeysShard);
                       shard.isShard = true;
@@ -311,7 +256,8 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
                     }));
 
     // Persist the new revision, thus causing the newly created shards to go live.
-    tm().transactNew(
+    ofyTm()
+        .transact(
             () -> {
               verify(
                   (getCurrentRevision() == null && oldRevision == null)
@@ -337,9 +283,9 @@ public class ClaimsListShard extends ImmutableObject implements NonReplicatedEnt
   }
 
   /** Return a single logical instance that combines all Datastore shards. */
-  @Nullable
-  public static ClaimsListShard get() {
-    return CACHE.get();
+  static Optional<ClaimsListShard> getFromDatastore() {
+    return LOADER_RETRIER.callWithRetry(
+        ClaimsListShard::loadClaimsListShard, IllegalStateException.class);
   }
 
   /** As a safety mechanism, fail if someone tries to save this class directly. */

core/src/main/java/google/registry/model/tmch/ClaimsListSqlDao.java

@@ -0,0 +1,53 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.tmch;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import java.util.Optional;
+
+/** Data access object for {@link ClaimsListShard}. */
+public class ClaimsListSqlDao {
+
+  /** Saves the given {@link ClaimsListShard} to Cloud SQL. */
+  static void save(ClaimsListShard claimsList) {
+    jpaTm().transact(() -> jpaTm().insert(claimsList));
+  }
+
+  /**
+   * Returns the most recent revision of the {@link ClaimsListShard} in SQL or an empty list if it
+   * doesn't exist.
+   */
+  static Optional<ClaimsListShard> get() {
+    return jpaTm()
+        .transact(
+            () -> {
+              Long revisionId =
+                  jpaTm()
+                      .query("SELECT MAX(revisionId) FROM ClaimsList", Long.class)
+                      .getSingleResult();
+              return jpaTm()
+                  .query(
+                      "FROM ClaimsList cl LEFT JOIN FETCH cl.labelsToKeys WHERE cl.revisionId ="
+                          + " :revisionId",
+                      ClaimsListShard.class)
+                  .setParameter("revisionId", revisionId)
+                  .getResultStream()
+                  .findFirst();
+            });
+  }
+
+  private ClaimsListSqlDao() {}
+}

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -79,10 +79,7 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
                   .transactNew(
                       () -> {
                         // Delete the old one and insert the new one
-                        jpaTm()
-                            .getEntityManager()
-                            .createQuery("DELETE FROM TmchCrl")
-                            .executeUpdate();
+                        jpaTm().query("DELETE FROM TmchCrl").executeUpdate();
                         jpaTm().putWithoutBackup(tmchCrl);
                       });
             });

core/src/main/java/google/registry/model/transfer/DomainTransferData.java

@@ -15,6 +15,7 @@
 package google.registry.model.transfer;
 
 import static google.registry.util.CollectionUtils.forceEmptyToNull;
+import static google.registry.util.CollectionUtils.isNullOrEmpty;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
@@ -23,6 +24,7 @@ import com.googlecode.objectify.annotation.AlsoLoad;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.IgnoreSave;
+import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.annotation.Unindex;
 import com.googlecode.objectify.condition.IfNull;
 import google.registry.model.billing.BillingEvent;
@@ -31,6 +33,7 @@ import google.registry.model.domain.Period;
 import google.registry.model.domain.Period.Unit;
 import google.registry.model.poll.PollMessage;
 import google.registry.persistence.VKey;
+import java.util.Set;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeOverride;
 import javax.persistence.AttributeOverrides;
@@ -80,7 +83,11 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
 
   @Ignore
   @Column(name = "transfer_billing_cancellation_id")
-  Long billingCancellationId;
+  public VKey<BillingEvent.Cancellation> billingCancellationId;
+
+  @Ignore
+  @Column(name = "transfer_billing_cancellation_history_id")
+  Long billingCancellationHistoryId;
 
   /**
    * The regular one-time billing event that will be charged for a server-approved transfer.
@@ -146,6 +153,17 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     serverApproveAutorenewPollMessage =
         DomainBase.restoreOfyFrom(
             rootKey, serverApproveAutorenewPollMessage, serverApproveAutorenewPollMessageHistoryId);
+    billingCancellationId =
+        DomainBase.restoreOfyFrom(rootKey, billingCancellationId, billingCancellationHistoryId);
+
+    // Reconstruct server approve entities.  We currently have to call postLoad() a _second_ time
+    // if the billing cancellation id has been reconstituted, as it is part of that set.
+    // TODO(b/183010623): Normalize the approaches to VKey reconstitution for the TransferData
+    // hierarchy (the logic currently lives either in PostLoad or here, depending on the key).
+    if (billingCancellationId != null) {
+      serverApproveEntities = null;
+      postLoad();
+    }
   }
 
   /**
@@ -176,6 +194,12 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     serverApproveAutorenewPollMessageHistoryId = DomainBase.getHistoryId(val);
   }
 
+  @SuppressWarnings("unused") // For Hibernate.
+  private void billingCancellationHistoryId(
+      @AlsoLoad("billingCancellationHistoryId") VKey<BillingEvent.Cancellation> val) {
+    billingCancellationHistoryId = DomainBase.getHistoryId(val);
+  }
+
   public Period getTransferPeriod() {
     return transferPeriod;
   }
@@ -218,11 +242,20 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     return serverApproveAutorenewPollMessageHistoryId;
   }
 
+  @OnLoad
+  @Override
+  void onLoad(
+      @AlsoLoad("serverApproveEntities")
+          Set<VKey<? extends TransferServerApproveEntity>> serverApproveEntities) {
+    super.onLoad(serverApproveEntities);
+    mapBillingCancellationEntityToField(serverApproveEntities, this);
+  }
+
   @PostLoad
   @Override
   void postLoad() {
-    // The superclass's serverApproveEntities should include the billing events if present
     super.postLoad();
+    // The superclass's serverApproveEntities should include the billing events if present
     ImmutableSet.Builder<VKey<? extends TransferServerApproveEntity>> serverApproveEntitiesBuilder =
         new ImmutableSet.Builder<>();
     if (serverApproveEntities != null) {
@@ -234,8 +267,8 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     if (serverApproveAutorenewEvent != null) {
       serverApproveEntitiesBuilder.add(serverApproveAutorenewEvent);
     }
-    if (serverApproveAutorenewPollMessage != null) {
-      serverApproveEntitiesBuilder.add(serverApproveAutorenewPollMessage);
+    if (billingCancellationId != null) {
+      serverApproveEntitiesBuilder.add(billingCancellationId);
     }
     serverApproveEntities = forceEmptyToNull(serverApproveEntitiesBuilder.build());
   }
@@ -250,6 +283,28 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
     return new Builder(clone(this));
   }
 
+  /** Maps serverApproveEntities set to the individual fields. */
+  @SuppressWarnings("unchecked")
+  static void mapBillingCancellationEntityToField(
+      Set<VKey<? extends TransferServerApproveEntity>> serverApproveEntities,
+      DomainTransferData domainTransferData) {
+    if (isNullOrEmpty(serverApproveEntities)) {
+      domainTransferData.billingCancellationId = null;
+      domainTransferData.billingCancellationHistoryId = null;
+    } else {
+      domainTransferData.billingCancellationId =
+          (VKey<BillingEvent.Cancellation>)
+              serverApproveEntities.stream()
+                  .filter(k -> k.getKind().equals(BillingEvent.Cancellation.class))
+                  .findFirst()
+                  .orElse(null);
+      domainTransferData.billingCancellationHistoryId =
+          domainTransferData.billingCancellationId != null
+              ? DomainBase.getHistoryId(domainTransferData.billingCancellationId)
+              : null;
+    }
+  }
+
   public static class Builder extends TransferData.Builder<DomainTransferData, Builder> {
     /** Create a {@link DomainTransferData.Builder} wrapping a new instance. */
     public Builder() {}
@@ -259,6 +314,12 @@ public class DomainTransferData extends TransferData<DomainTransferData.Builder>
       super(instance);
     }
 
+    @Override
+    public DomainTransferData build() {
+      mapBillingCancellationEntityToField(getInstance().serverApproveEntities, getInstance());
+      return super.build();
+    }
+
     public Builder setTransferPeriod(Period transferPeriod) {
       getInstance().transferPeriod = transferPeriod;
       return this;

core/src/main/java/google/registry/module/backend/BackendComponent.java

@@ -36,6 +36,8 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.backend.BackendRequestComponent.BackendRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.persistence.PersistenceModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.JSchModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
@@ -71,6 +73,8 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      PersistenceModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       SheetsServiceModule.class,
       StackdriverModule.class,

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -30,6 +30,8 @@ import google.registry.batch.RefreshDnsOnHostRenameAction;
 import google.registry.batch.RelockDomainAction;
 import google.registry.batch.ResaveAllEppResourcesAction;
 import google.registry.batch.ResaveEntityAction;
+import google.registry.batch.WipeOutCloudSqlAction;
+import google.registry.batch.WipeoutDatastoreAction;
 import google.registry.cron.CommitLogFanoutAction;
 import google.registry.cron.CronModule;
 import google.registry.cron.TldFanoutAction;
@@ -205,6 +207,10 @@ interface BackendRequestComponent {
 
   PublishInvoicesAction uploadInvoicesAction();
 
+  WipeOutCloudSqlAction wipeOutCloudSqlAction();
+
+  WipeoutDatastoreAction wipeoutDatastoreAction();
+
   @Subcomponent.Builder
   abstract class Builder implements RequestComponentBuilder<BackendRequestComponent> {
 

core/src/main/java/google/registry/persistence/HibernateSchemaExporter.java

@@ -51,7 +51,7 @@ public class HibernateSchemaExporter {
   }
 
   /** Exports DDL script to the {@code outputFile} for the given {@code entityClasses}. */
-  public void export(ImmutableList<Class> entityClasses, File outputFile) {
+  public void export(ImmutableList<Class<?>> entityClasses, File outputFile) {
     // Configure Hibernate settings.
     Map<String, String> settings = Maps.newHashMap();
     settings.put(Environment.DIALECT, NomulusPostgreSQLDialect.class.getName());
@@ -85,7 +85,7 @@ public class HibernateSchemaExporter {
     }
   }
 
-  private ImmutableList<Class> findAllConverters() {
+  private ImmutableList<Class<?>> findAllConverters() {
     return PersistenceXmlUtility.getManagedClasses().stream()
         .filter(AttributeConverter.class::isAssignableFrom)
         .collect(toImmutableList());

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -31,7 +31,6 @@ import dagger.BindsOptionalOf;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
-import google.registry.keyring.kms.KmsKeyring;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
@@ -44,10 +43,14 @@ import google.registry.tools.AuthModule.CloudSqlClientCredential;
 import google.registry.util.Clock;
 import java.lang.annotation.Documented;
 import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Properties;
+import java.util.function.Supplier;
 import javax.annotation.Nullable;
 import javax.inject.Provider;
 import javax.inject.Qualifier;
@@ -163,22 +166,46 @@ public abstract class PersistenceModule {
     return ImmutableMap.copyOf(overrides);
   }
 
+  /**
+   * Provides a {@link Supplier} of single-use JDBC {@link Connection connections} that can manage
+   * the database DDL schema.
+   */
+  @Provides
+  @Singleton
+  @SchemaManagerConnection
+  static Supplier<Connection> provideSchemaManagerConnectionSupplier(
+      SqlCredentialStore credentialStore,
+      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs) {
+    SqlCredential credential =
+        credentialStore.getCredential(new RobotUser(RobotId.SCHEMA_DEPLOYER));
+    String user = credential.login();
+    String password = credential.password();
+    return () -> createJdbcConnection(user, password, cloudSqlConfigs);
+  }
+
+  private static Connection createJdbcConnection(
+      String user, String password, ImmutableMap<String, String> cloudSqlConfigs) {
+    Properties properties = new Properties();
+    properties.put("user", user);
+    properties.put("password", password);
+    properties.put("cloudSqlInstance", cloudSqlConfigs.get(HIKARI_DS_CLOUD_SQL_INSTANCE));
+    properties.put("socketFactory", cloudSqlConfigs.get(HIKARI_DS_SOCKET_FACTORY));
+    try {
+      return DriverManager.getConnection(cloudSqlConfigs.get(Environment.URL), properties);
+    } catch (SQLException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   @Provides
   @Singleton
   @AppEngineJpaTm
   static JpaTransactionManager provideAppEngineJpaTm(
-      @Config("cloudSqlUsername") String username,
-      KmsKeyring kmsKeyring,
       SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
-    validateAndSetCredential(
-        credentialStore,
-        new RobotUser(RobotId.NOMULUS),
-        overrides,
-        username,
-        kmsKeyring.getCloudSqlPassword());
+    setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides);
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -223,20 +250,13 @@ public abstract class PersistenceModule {
   @Singleton
   @NomulusToolJpaTm
   static JpaTransactionManager provideNomulusToolJpaTm(
-      @Config("toolsCloudSqlUsername") String username,
-      KmsKeyring kmsKeyring,
       SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       @CloudSqlClientCredential Credential credential,
       Clock clock) {
     CloudSqlCredentialSupplier.setupCredentialSupplier(credential);
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
-    validateAndSetCredential(
-        credentialStore,
-        new RobotUser(RobotId.TOOL),
-        overrides,
-        username,
-        kmsKeyring.getToolsCloudSqlPassword());
+    setSqlCredential(credentialStore, new RobotUser(RobotId.TOOL), overrides);
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -314,35 +334,15 @@ public abstract class PersistenceModule {
     return emf;
   }
 
-  /**
-   * Verifies that the credential from the Secret Manager matches the one currently in use, and
-   * configures JPA with the credential from the Secret Manager.
-   *
-   * <p>This is a helper for the transition to the Secret Manager, and will be removed once data and
-   * permissions are properly set up for all projects.
-   */
-  private static void validateAndSetCredential(
-      SqlCredentialStore credentialStore,
-      SqlUser sqlUser,
-      Map<String, String> overrides,
-      String expectedLogin,
-      String expectedPassword) {
+  /** Configures JPA with the credential from the Secret Manager. */
+  private static void setSqlCredential(
+      SqlCredentialStore credentialStore, SqlUser sqlUser, Map<String, String> overrides) {
     try {
       SqlCredential credential = credentialStore.getCredential(sqlUser);
-      checkState(
-          credential.login().equals(expectedLogin),
-          "Wrong login for %s. Expecting %s, found %s.",
-          sqlUser.geUserName(),
-          expectedLogin,
-          credential.login());
-      checkState(
-          credential.password().equals(expectedPassword),
-          "Wrong password for %s.",
-          sqlUser.geUserName());
       overrides.put(Environment.USER, credential.login());
       overrides.put(Environment.PASS, credential.password());
-      logger.atWarning().log("Credentials in the kerying and the secret manager match.");
     } catch (Throwable e) {
+      // TODO(b/184631990): after SQL becomes primary, throw an exception to fail fast
       logger.atSevere().withCause(e).log("Failed to get SQL credential from Secret Manager");
     }
   }
@@ -378,6 +378,11 @@ public abstract class PersistenceModule {
     }
   }
 
+  /** Dagger qualifier for JDBC {@link Connection} with schema management privilege. */
+  @Qualifier
+  @Documented
+  public @interface SchemaManagerConnection {}
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
   @Qualifier
   @Documented