Upload latest version of RDE report to icann (#1089)

* Upload latest version of RDE report to icann

Currently the RdeReportAction is hard coded to load the initial version
of a report. This is wrong when reports have been regenerated.

Changed lines are copied from RdeUploadAction.

build.gradle

@@ -24,7 +24,7 @@ buildscript {
   }
 
   dependencies {
-    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.0.1'
+    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.4.1'
     classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.6.1'
     classpath 'org.sonatype.aether:aether-api:1.13.1'
     classpath 'org.sonatype.aether:aether-impl:1.13.1'

common/src/testing/java/google/registry/testing/FakeClock.java

@@ -35,6 +35,8 @@ public final class FakeClock implements Clock {
   // threads should see a consistent flow.
   private final AtomicLong currentTimeMillis = new AtomicLong();
 
+  private volatile long autoIncrementStepMs;
+
   /** Creates a FakeClock that starts at START_OF_TIME. */
   public FakeClock() {
     this(START_OF_TIME);
@@ -48,7 +50,21 @@ public final class FakeClock implements Clock {
   /** Returns the current time. */
   @Override
   public DateTime nowUtc() {
-    return new DateTime(currentTimeMillis.get(), UTC);
+    return new DateTime(currentTimeMillis.addAndGet(autoIncrementStepMs), UTC);
+  }
+
+  /**
+   * Sets the increment applied to the clock whenever it is queried. The increment is zero by
+   * default: the clock is left unchanged when queried.
+   *
+   * <p>Passing a duration of zero to this method effectively unsets the auto increment mode.
+   *
+   * @param autoIncrementStep the new auto increment duration
+   * @return this
+   */
+  public FakeClock setAutoIncrementStep(ReadableDuration autoIncrementStep) {
+    this.autoIncrementStepMs = autoIncrementStep.getMillis();
+    return this;
   }
 
   /** Advances clock by one millisecond. */

core/build.gradle

@@ -423,7 +423,7 @@ task jaxbToJava {
       }
     }
     execInBash(
-        'find . -name *.java -exec sed -i /\\*\\ \\<p\\>\\$/d {} +',
+        "find . -name *.java -exec sed -i -e '/" + /\* <p>$/ + "/d' {} +",
         generatedDir)
   }
 }

core/gradle/dependency-locks/compile.lockfile

@@ -253,10 +253,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -246,10 +246,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/default.lockfile

@@ -267,10 +267,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -266,10 +266,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -253,10 +253,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -247,10 +247,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -265,10 +265,10 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -265,10 +265,10 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/runtime.lockfile

@@ -265,10 +265,10 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -266,10 +266,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testCompile.lockfile

@@ -300,12 +300,12 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -294,12 +294,12 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testRuntime.lockfile

@@ -313,12 +313,12 @@ org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -314,12 +314,12 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -154,7 +154,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
     Object ofyPojo = ofy().toPojo(entity);
     if (ofyPojo instanceof DatastoreEntity) {
       DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
-      datastoreEntity.toSqlEntity().ifPresent(jpaTm()::put);
+      datastoreEntity
+          .toSqlEntity()
+          .ifPresent(
+              sqlEntity -> {
+                ReplaySpecializer.beforeSqlSave(sqlEntity);
+                jpaTm().put(sqlEntity);
+              });
     } else {
       // this should never happen, but we shouldn't fail on it
       logger.atSevere().log(

core/src/main/java/google/registry/batch/RefreshDnsOnHostRenameAction.java

@@ -25,7 +25,7 @@ import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
 import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
 import static google.registry.model.EppResourceUtils.isActive;
 import static google.registry.model.EppResourceUtils.isDeleted;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.latestOf;
 import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.SECONDS;
@@ -44,7 +44,6 @@ import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.flogger.FluentLogger;
-import com.googlecode.objectify.Key;
 import google.registry.batch.AsyncTaskMetrics.OperationResult;
 import google.registry.dns.DnsQueue;
 import google.registry.mapreduce.MapreduceRunner;
@@ -64,6 +63,7 @@ import google.registry.util.SystemClock;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.logging.Level;
 import javax.annotation.Nullable;
@@ -123,7 +123,7 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
     }
 
     ImmutableList.Builder<DnsRefreshRequest> requestsBuilder = new ImmutableList.Builder<>();
-    ImmutableList.Builder<Key<HostResource>> hostKeys = new ImmutableList.Builder<>();
+    ImmutableList.Builder<VKey<HostResource>> hostKeys = new ImmutableList.Builder<>();
     final List<DnsRefreshRequest> requestsToDelete = new ArrayList<>();
 
     for (TaskHandle task : tasks) {
@@ -204,10 +204,10 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
         emit(true, true);
         return;
       }
-      Key<HostResource> referencingHostKey = null;
+      VKey<HostResource> referencingHostKey = null;
       for (DnsRefreshRequest request : refreshRequests) {
         if (isActive(domain, request.lastUpdateTime())
-            && domain.getNameservers().contains(VKey.from(request.hostKey()))) {
+            && domain.getNameservers().contains(request.hostKey())) {
           referencingHostKey = request.hostKey();
           break;
         }
@@ -293,7 +293,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
 
     private static final long serialVersionUID = 1772812852271288622L;
 
-    abstract Key<HostResource> hostKey();
+    abstract VKey<HostResource> hostKey();
+
     abstract DateTime lastUpdateTime();
     abstract DateTime requestedTime();
     abstract boolean isRefreshNeeded();
@@ -301,7 +302,8 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
 
     @AutoValue.Builder
     abstract static class Builder {
-      abstract Builder setHostKey(Key<HostResource> hostKey);
+      abstract Builder setHostKey(VKey<HostResource> hostKey);
+
       abstract Builder setLastUpdateTime(DateTime lastUpdateTime);
       abstract Builder setRequestedTime(DateTime requestedTime);
       abstract Builder setIsRefreshNeeded(boolean isRefreshNeeded);
@@ -314,10 +316,12 @@ public class RefreshDnsOnHostRenameAction implements Runnable {
      */
     static DnsRefreshRequest createFromTask(TaskHandle task, DateTime now) throws Exception {
       ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
-      Key<HostResource> hostKey =
-          Key.create(checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
+      VKey<HostResource> hostKey =
+          VKey.fromWebsafeKey(
+              checkNotNull(params.get(PARAM_HOST_KEY), "Host to refresh not specified"));
       HostResource host =
-          checkNotNull(ofy().load().key(hostKey).now(), "Host to refresh doesn't exist");
+          tm().transact(() -> tm().loadByKeyIfPresent(hostKey))
+              .orElseThrow(() -> new NoSuchElementException("Host to refresh doesn't exist"));
       boolean isHostDeleted =
           isDeleted(host, latestOf(now, host.getUpdateTimestamp().getTimestamp()));
       if (isHostDeleted) {

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateParameter;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+/**
+ * Wipes out all Cloud Datastore data in a Nomulus GCP environment.
+ *
+ * <p>This class is created for the QA environment, where migration testing with production data
+ * will happen. A regularly scheduled wipeout is a prerequisite to using production data there.
+ */
+@Action(
+    service = Action.Service.BACKEND,
+    path = "/_dr/task/wipeOutDatastore",
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class WipeoutDatastoreAction implements Runnable {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private static final String PIPELINE_NAME = "bulk_delete_datastore_pipeline";
+
+  // As a short-lived class, hardcode allowed projects here instead of using config files.
+  private static final ImmutableSet<String> ALLOWED_PROJECTS =
+      ImmutableSet.of("domain-registry-qa");
+
+  private final String projectId;
+  private final String jobRegion;
+  private final Response response;
+  private final Dataflow dataflow;
+  private final String stagingBucketUrl;
+
+  @Inject
+  WipeoutDatastoreAction(
+      @Config("projectId") String projectId,
+      @Config("defaultJobRegion") String jobRegion,
+      @Config("beamStagingBucketUrl") String stagingBucketUrl,
+      Response response,
+      Dataflow dataflow) {
+    this.projectId = projectId;
+    this.jobRegion = jobRegion;
+    this.stagingBucketUrl = stagingBucketUrl;
+    this.response = response;
+    this.dataflow = dataflow;
+  }
+
+  @Override
+  public void run() {
+    response.setContentType(PLAIN_TEXT_UTF_8);
+
+    if (!ALLOWED_PROJECTS.contains(projectId)) {
+      response.setStatus(SC_FORBIDDEN);
+      response.setPayload("Wipeout is not allowed in " + projectId);
+      return;
+    }
+
+    try {
+      LaunchFlexTemplateParameter parameters =
+          new LaunchFlexTemplateParameter()
+              // Job name must be unique and in [-a-z0-9].
+              .setJobName(
+                  "bulk-delete-datastore-"
+                      + DateTime.now(DateTimeZone.UTC).toString("yyyy-MM-dd'T'HH-mm-ss'Z'"))
+              .setContainerSpecGcsPath(
+                  String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
+              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+      LaunchFlexTemplateResponse launchResponse =
+          dataflow
+              .projects()
+              .locations()
+              .flexTemplates()
+              .launch(
+                  projectId,
+                  jobRegion,
+                  new LaunchFlexTemplateRequest().setLaunchParameter(parameters))
+              .execute();
+      response.setStatus(SC_OK);
+      response.setPayload("Launched " + launchResponse.getJob().getName());
+    } catch (Exception e) {
+      String msg = String.format("Failed to launch %s.", PIPELINE_NAME);
+      logger.atSevere().withCause(e).log(msg);
+      response.setStatus(SC_INTERNAL_SERVER_ERROR);
+      response.setPayload(msg);
+    }
+  }
+}

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -403,12 +403,6 @@ public final class RegistryConfig {
       return config.cloudSql.jdbcUrl;
     }
 
-    @Provides
-    @Config("cloudSqlUsername")
-    public static String providesCloudSqlUsername(RegistryConfigSettings config) {
-      return config.cloudSql.username;
-    }
-
     @Provides
     @Config("cloudSqlInstanceConnectionName")
     public static String providesCloudSqlInstanceConnectionName(RegistryConfigSettings config) {
@@ -1342,12 +1336,6 @@ public final class RegistryConfig {
       return config.registryTool.clientSecret;
     }
 
-    @Provides
-    @Config("toolsCloudSqlUsername")
-    public static String providesToolsCloudSqlUsername(RegistryConfigSettings config) {
-      return config.registryTool.username;
-    }
-
     @Provides
     @Config("rdapTos")
     public static ImmutableList<String> provideRdapTos(RegistryConfigSettings config) {

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -123,6 +123,7 @@ public class RegistryConfigSettings {
   /** Configuration for Cloud SQL. */
   public static class CloudSql {
     public String jdbcUrl;
+    // TODO(05012021): remove username field after it is removed from all yaml files.
     public String username;
     public String instanceConnectionName;
     public boolean replicateTransactions;
@@ -221,6 +222,7 @@ public class RegistryConfigSettings {
   public static class RegistryTool {
     public String clientId;
     public String clientSecret;
+    // TODO(05012021): remove username field after it is removed from all yaml files.
     public String username;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -225,8 +225,6 @@ cloudSql:
   # If jdbcUrl in this file is moved elsewhere, be sure to move this notice
   # with it until the change is applied.
   jdbcUrl: jdbc:postgresql://localhost
-  # Username for the database user.
-  username: username
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
   # Set this to true to replicate cloud SQL transactions to datastore in the
@@ -447,7 +445,6 @@ registryTool:
   clientId: YOUR_CLIENT_ID
   # OAuth client secret used by the tool.
   clientSecret: YOUR_CLIENT_SECRET
-  username: toolusername
 
 # Configuration options for checking SSL certificates.
 sslCertificateValidation:

core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml

@@ -385,6 +385,12 @@
     <url-pattern>/_dr/task/wipeOutCloudSql</url-pattern>
   </servlet-mapping>
 
+  <!-- Action to wipeout Cloud Datastore data -->
+  <servlet-mapping>
+    <servlet-name>backend-servlet</servlet-name>
+    <url-pattern>/_dr/task/wipeOutDatastore</url-pattern>
+  </servlet-mapping>
+
   <!-- Security config -->
   <security-constraint>
     <web-resource-collection>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -91,4 +91,13 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutDatastore]]></url>
+    <description>
+      This job runs an action that deletes all data in Cloud Datastore.
+    </description>
+    <schedule>every saturday 03:07</schedule>
+    <target>backend</target>
+  </cron>
+
 </cronentries>

core/src/main/java/google/registry/keyring/api/InMemoryKeyring.java

@@ -39,8 +39,6 @@ public final class InMemoryKeyring implements Keyring {
   private final String marksdbLordnPassword;
   private final String marksdbSmdrlLoginAndPassword;
   private final String jsonCredential;
-  private final String cloudSqlPassword;
-  private final String toolsCloudSqlPassword;
 
   public InMemoryKeyring(
       PGPKeyPair rdeStagingKey,
@@ -83,8 +81,6 @@ public final class InMemoryKeyring implements Keyring {
     this.marksdbSmdrlLoginAndPassword =
         checkNotNull(marksdbSmdrlLoginAndPassword, "marksdbSmdrlLoginAndPassword");
     this.jsonCredential = checkNotNull(jsonCredential, "jsonCredential");
-    this.cloudSqlPassword = checkNotNull(cloudSqlPassword, "cloudSqlPassword");
-    this.toolsCloudSqlPassword = checkNotNull(toolsCloudSqlPassword, "toolsCloudSqlPassword");
   }
 
   @Override
@@ -157,16 +153,6 @@ public final class InMemoryKeyring implements Keyring {
     return jsonCredential;
   }
 
-  @Override
-  public String getCloudSqlPassword() {
-    return cloudSqlPassword;
-  }
-
-  @Override
-  public String getToolsCloudSqlPassword() {
-    return toolsCloudSqlPassword;
-  }
-
   /** Does nothing. */
   @Override
   public void close() {}

core/src/main/java/google/registry/keyring/api/KeyModule.java

@@ -36,18 +36,6 @@ public final class KeyModule {
     String value();
   }
 
-  @Provides
-  @Key("cloudSqlPassword")
-  static String providesCloudSqlPassword(Keyring keyring) {
-    return keyring.getCloudSqlPassword();
-  }
-
-  @Provides
-  @Key("toolsCloudSqlPassword")
-  static String providesToolsCloudSqlPassword(Keyring keyring) {
-    return keyring.getToolsCloudSqlPassword();
-  }
-
   @Provides
   @Key("brdaReceiverKey")
   static PGPPublicKey provideBrdaReceiverKey(Keyring keyring) {

core/src/main/java/google/registry/keyring/api/Keyring.java

@@ -28,12 +28,6 @@ import org.bouncycastle.openpgp.PGPPublicKey;
 @ThreadSafe
 public interface Keyring extends AutoCloseable {
 
-  /** Returns the password which is used by App Engine to connect to the Cloud SQL database. */
-  String getCloudSqlPassword();
-
-  /** Returns the password which is used by nomulus tool to connect to the Cloud SQL database. */
-  String getToolsCloudSqlPassword();
-
   /**
    * Returns the key which should be used to sign RDE deposits being uploaded to a third-party.
    *

core/src/main/java/google/registry/keyring/kms/KmsKeyring.java

@@ -21,6 +21,10 @@ import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
@@ -29,8 +33,11 @@ import google.registry.keyring.api.KeyringException;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
 import google.registry.model.server.KmsSecretRevisionSqlDao;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Optional;
+import java.util.stream.Stream;
 import javax.inject.Inject;
 import org.bouncycastle.openpgp.PGPException;
 import org.bouncycastle.openpgp.PGPKeyPair;
@@ -46,6 +53,8 @@ import org.bouncycastle.openpgp.PGPPublicKey;
  */
 public class KmsKeyring implements Keyring {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   /** Key labels for private key secrets. */
   enum PrivateKeyLabel {
     BRDA_SIGNING_PRIVATE,
@@ -72,7 +81,6 @@ public class KmsKeyring implements Keyring {
 
   /** Key labels for string secrets. */
   enum StringKeyLabel {
-    CLOUD_SQL_PASSWORD_STRING,
     SAFE_BROWSING_API_KEY,
     ICANN_REPORTING_PASSWORD_STRING,
     JSON_CREDENTIAL_STRING,
@@ -80,8 +88,7 @@ public class KmsKeyring implements Keyring {
     MARKSDB_LORDN_PASSWORD_STRING,
     MARKSDB_SMDRL_LOGIN_STRING,
     RDE_SSH_CLIENT_PRIVATE_STRING,
-    RDE_SSH_CLIENT_PUBLIC_STRING,
-    TOOLS_CLOUD_SQL_PASSWORD_STRING;
+    RDE_SSH_CLIENT_PUBLIC_STRING;
 
     String getLabel() {
       return UPPER_UNDERSCORE.to(LOWER_HYPHEN, name());
@@ -89,20 +96,13 @@ public class KmsKeyring implements Keyring {
   }
 
   private final KmsConnection kmsConnection;
+  private final KeyringSecretStore secretStore;
 
   @Inject
-  KmsKeyring(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
+  KmsKeyring(
+      @Config("defaultKmsConnection") KmsConnection kmsConnection, KeyringSecretStore secretStore) {
     this.kmsConnection = kmsConnection;
-  }
-
-  @Override
-  public String getCloudSqlPassword() {
-    return getString(StringKeyLabel.CLOUD_SQL_PASSWORD_STRING);
-  }
-
-  @Override
-  public String getToolsCloudSqlPassword() {
-    return getString(StringKeyLabel.TOOLS_CLOUD_SQL_PASSWORD_STRING);
+    this.secretStore = secretStore;
   }
 
   @Override
@@ -204,7 +204,7 @@ public class KmsKeyring implements Keyring {
     return getKeyPair(keyLabel).getPrivateKey();
   }
 
-  private byte[] getDecryptedData(String keyName) {
+  private byte[] getDecryptedDataFromDatastore(String keyName) {
     String encryptedData;
     if (tm().isOfy()) {
       KmsSecret secret =
@@ -225,4 +225,56 @@ public class KmsKeyring implements Keyring {
           String.format("CloudKMS decrypt operation failed for secret %s", keyName), e);
     }
   }
+
+  private byte[] getDataFromSecretStore(String keyName) {
+    try {
+      return secretStore.getSecret(keyName);
+    } catch (Exception e) {
+      return new byte[0];
+    }
+  }
+
+  private byte[] getDecryptedData(String keyName) {
+    byte[] dsData = getDecryptedDataFromDatastore(keyName);
+    byte[] secretStoreData = getDataFromSecretStore(keyName);
+
+    if (Arrays.equals(dsData, secretStoreData)) {
+      logger.atInfo().log("Values for %s in Datastore and Secret Manager match.", keyName);
+      return secretStoreData;
+    }
+    logger.atWarning().log("Values for %s in Datastore and Secret Manager do not match.", keyName);
+    return dsData;
+  }
+
+  /**
+   * Generates the tasks to migrate secrets from Datastore to Secret Manager.
+   *
+   * <p>The keys in the returned {@link ImmutableMap} are the names of the secrets that need
+   * migration. The values in the map are {@link Runnable Runnables} that copy secret data from
+   * Datastore to Secret Manager for their corresponding keys. Only secrets that are absent in
+   * Secret Manager or have inconsistent values are included in the returned map.
+   */
+  public ImmutableMap<String, Runnable> migrationPlan() {
+
+    ImmutableMap.Builder<String, Runnable> tasks = new ImmutableMap.Builder<>();
+
+    ImmutableList<String> labels =
+        Streams.concat(
+                Stream.of(PrivateKeyLabel.values()).map(PrivateKeyLabel::getLabel),
+                Stream.of(PublicKeyLabel.values()).map(PublicKeyLabel::getLabel),
+                Stream.of(StringKeyLabel.values()).map(StringKeyLabel::getLabel))
+            .collect(ImmutableList.toImmutableList());
+
+    for (String keyName : labels) {
+      byte[] dsData = getDecryptedDataFromDatastore(keyName);
+      byte[] secretStoreData = getDataFromSecretStore(keyName);
+      if (Arrays.equals(dsData, secretStoreData)) {
+        logger.atInfo().log("%s is already up to date.\n", keyName);
+        continue;
+      }
+      logger.atInfo().log("%s needs to be migrated.\n", keyName);
+      tasks.put(keyName, () -> secretStore.createOrUpdateSecret(keyName, dsData));
+    }
+    return tasks.build();
+  }
 }

core/src/main/java/google/registry/keyring/kms/KmsUpdater.java

@@ -24,7 +24,6 @@ import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.BRDA_SIGNING
 import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.RDE_RECEIVER_PUBLIC;
 import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.RDE_SIGNING_PUBLIC;
 import static google.registry.keyring.kms.KmsKeyring.PublicKeyLabel.RDE_STAGING_PUBLIC;
-import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.CLOUD_SQL_PASSWORD_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.ICANN_REPORTING_PASSWORD_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.JSON_CREDENTIAL_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.MARKSDB_DNL_LOGIN_STRING;
@@ -33,7 +32,6 @@ import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.MARKSDB_SMDR
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.RDE_SSH_CLIENT_PRIVATE_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.RDE_SSH_CLIENT_PUBLIC_STRING;
 import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.SAFE_BROWSING_API_KEY;
-import static google.registry.keyring.kms.KmsKeyring.StringKeyLabel.TOOLS_CLOUD_SQL_PASSWORD_STRING;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -45,6 +43,7 @@ import google.registry.keyring.kms.KmsKeyring.PublicKeyLabel;
 import google.registry.keyring.kms.KmsKeyring.StringKeyLabel;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -61,20 +60,19 @@ import org.bouncycastle.openpgp.PGPPublicKey;
 public final class KmsUpdater {
 
   private final KmsConnection kmsConnection;
+  private final KeyringSecretStore secretStore;
   private final HashMap<String, byte[]> secretValues;
 
   @Inject
-  public KmsUpdater(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
+  public KmsUpdater(
+      @Config("defaultKmsConnection") KmsConnection kmsConnection, KeyringSecretStore secretStore) {
     this.kmsConnection = kmsConnection;
+    this.secretStore = secretStore;
 
     // Use LinkedHashMap to preserve insertion order on update() to simplify testing and debugging
     this.secretValues = new LinkedHashMap<>();
   }
 
-  public KmsUpdater setCloudSqlPassword(String password) {
-    return setString(password, CLOUD_SQL_PASSWORD_STRING);
-  }
-
   public KmsUpdater setRdeSigningKey(PGPKeyPair keyPair) throws IOException, PGPException {
     return setKeyPair(keyPair, RDE_SIGNING_PRIVATE, RDE_SIGNING_PUBLIC);
   }
@@ -107,10 +105,6 @@ public final class KmsUpdater {
     return setString(apiKey, SAFE_BROWSING_API_KEY);
   }
 
-  public KmsUpdater setToolsCloudSqlPassword(String password) {
-    return setString(password, TOOLS_CLOUD_SQL_PASSWORD_STRING);
-  }
-
   public KmsUpdater setIcannReportingPassword(String password) {
     return setString(password, ICANN_REPORTING_PASSWORD_STRING);
   }
@@ -142,6 +136,19 @@ public final class KmsUpdater {
     checkState(!secretValues.isEmpty(), "At least one Keyring value must be persisted");
 
     persistEncryptedValues(encryptValues(secretValues));
+
+    // Errors when writing to secret store can be thrown to the top, since writes are always
+    // executed by a human user using the UpdateKmsKeyringCommand.
+    try {
+      secretValues
+          .entrySet()
+          .forEach(e -> secretStore.createOrUpdateSecret(e.getKey(), e.getValue()));
+    } catch (RuntimeException e) {
+      throw new RuntimeException(
+          "Failed to persist secrets to Secret Manager. "
+              + "Please check the status of Secret Manager and re-run the command.",
+          e);
+    }
   }
 
   /**

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -194,30 +194,6 @@ public final class EppResourceUtils {
     return ForeignKeyIndex.load(clazz, uniqueIds, now).keySet();
   }
 
-  /**
-   * Loads resources that match some filter and that have {@link EppResource#deletionTime} that is
-   * not before "now".
-   *
-   * <p>This is an eventually consistent query.
-   *
-   * @param clazz the resource type to load
-   * @param now the logical time of the check
-   * @param filterDefinition the filter to apply when loading resources
-   * @param filterValue the acceptable value for the filter
-   */
-  public static <T extends EppResource> Iterable<T> queryNotDeleted(
-      Class<T> clazz, DateTime now, String filterDefinition, Object filterValue) {
-    return ofy()
-        .load()
-        .type(clazz)
-        .filter(filterDefinition, filterValue)
-        .filter("deletionTime >", now.toDate())
-        .list()
-        .stream()
-        .map(EppResourceUtils.transformAtTime(now))
-        .collect(toImmutableSet());
-  }
-
   /**
    * Returns a Function that transforms an EppResource to the given DateTime, suitable for use with
    * Iterables.transform() over a collection of EppResources.

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -17,6 +17,7 @@ package google.registry.model;
 import static com.google.common.collect.Iterables.transform;
 import static com.google.common.collect.Maps.transformValues;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static java.lang.annotation.ElementType.FIELD;
 import static java.lang.annotation.RetentionPolicy.RUNTIME;
 import static java.util.stream.Collectors.toCollection;
@@ -187,7 +188,10 @@ public abstract class ImmutableObject implements Cloneable {
   /** Helper function to recursively hydrate an ImmutableObject. */
   private static Object hydrate(Object value) {
     if (value instanceof Key) {
-      return hydrate(ofy().load().key((Key<?>) value).now());
+      if (tm().isOfy()) {
+        return hydrate(ofy().load().key((Key<?>) value).now());
+      }
+      return value;
     } else if (value instanceof Map) {
       return transformValues((Map<?, ?>) value, ImmutableObject::hydrate);
     } else if (value instanceof Collection) {

core/src/main/java/google/registry/model/ResourceTransferUtils.java

@@ -17,7 +17,6 @@ package google.registry.model;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
@@ -107,14 +106,14 @@ public final class ResourceTransferUtils {
   /** Update the relevant {@link ForeignKeyIndex} to cache the new deletion time. */
   public static <R extends EppResource> void updateForeignKeyIndexDeletionTime(R resource) {
     if (resource instanceof ForeignKeyedEppResource) {
-      ofy().save().entity(ForeignKeyIndex.create(resource, resource.getDeletionTime()));
+      tm().insert(ForeignKeyIndex.create(resource, resource.getDeletionTime()));
     }
   }
 
   /** If there is a transfer out, delete the server-approve entities and enqueue a poll message. */
   public static <R extends EppResource & ResourceWithTransferData>
       void handlePendingTransferOnDelete(
-            R resource, R newResource, DateTime now, HistoryEntry historyEntry) {
+          R resource, R newResource, DateTime now, HistoryEntry historyEntry) {
     if (resource.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) {
       TransferData oldTransferData = resource.getTransferData();
       tm().delete(oldTransferData.getServerApproveEntities());

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -18,7 +18,8 @@ import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.CollectionUtils.forceEmptyToNull;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
@@ -772,10 +773,10 @@ public abstract class BillingEvent extends ImmutableObject
         Modification instance = getInstance();
         checkNotNull(instance.reason);
         checkNotNull(instance.eventRef);
-        BillingEvent.OneTime billingEvent = ofy().load().key(instance.eventRef).now();
-        checkArgument(Objects.equals(
-            instance.cost.getCurrencyUnit(),
-            billingEvent.cost.getCurrencyUnit()),
+        BillingEvent.OneTime billingEvent =
+            transactIfJpaTm(() -> tm().loadByKey(VKey.from(instance.eventRef)));
+        checkArgument(
+            Objects.equals(instance.cost.getCurrencyUnit(), billingEvent.cost.getCurrencyUnit()),
             "Referenced billing event is in a different currency");
         return super.build();
       }

core/src/main/java/google/registry/model/common/CrossTldSingleton.java

@@ -31,7 +31,7 @@ public abstract class CrossTldSingleton extends ImmutableObject {
 
   public static final long SINGLETON_ID = 1; // There is always exactly one of these.
 
-  @Id @Transient long id = SINGLETON_ID;
+  @Id @javax.persistence.Id long id = SINGLETON_ID;
 
   @Transient @Parent Key<EntityGroupRoot> parent = getCrossTldKey();
 }

core/src/main/java/google/registry/model/common/Cursor.java

@@ -36,6 +36,7 @@ import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.io.Serializable;
 import java.util.List;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
@@ -280,8 +281,8 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
   /**
    * Returns the current time for a given cursor, or {@code START_OF_TIME} if the cursor is null.
    */
-  public static DateTime getCursorTimeOrStartOfTime(Cursor cursor) {
-    return cursor != null ? cursor.getCursorTime() : START_OF_TIME;
+  public static DateTime getCursorTimeOrStartOfTime(Optional<Cursor> cursor) {
+    return cursor.map(Cursor::getCursorTime).orElse(START_OF_TIME);
   }
 
   public DateTime getCursorTime() {

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.contact;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -114,6 +116,12 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the contactBase field during asynchronous replay
+  public static void beforeSqlSave(ContactHistory contactHistory) {
+    contactHistory.contactBase =
+        jpaTm().loadByKey(VKey.createSql(ContactResource.class, contactHistory.getContactRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link ContactHistory} entity. */
   public static class ContactHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -15,6 +15,7 @@
 package google.registry.model.domain;
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.collect.ImmutableSet;
@@ -263,6 +264,12 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the domainContent field during asynchronous replay
+  public static void beforeSqlSave(DomainHistory domainHistory) {
+    domainHistory.domainContent =
+        jpaTm().loadByKey(VKey.createSql(DomainBase.class, domainHistory.getDomainRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link DomainHistory} entity. */
   public static class DomainHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.host;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -115,6 +117,12 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the hostBase field during asynchronous replay
+  public static void beforeSqlSave(HostHistory hostHistory) {
+    hostHistory.hostBase =
+        jpaTm().loadByKey(VKey.createSql(HostResource.class, hostHistory.getHostRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link HostHistory} entity. */
   public static class HostHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -48,8 +48,10 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.CriteriaQueryBuilder;
 import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.NonFinalForTesting;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.Map;
 import java.util.Optional;
@@ -193,8 +195,8 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * has been soft deleted.
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
-      Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
-    return loadIndexesFromStore(clazz, foreignKeys).entrySet().stream()
+      Class<E> clazz, Collection<String> foreignKeys, final DateTime now) {
+    return loadIndexesFromStore(clazz, foreignKeys, true).entrySet().stream()
         .filter(e -> now.isBefore(e.getValue().getDeletionTime()))
         .collect(entriesToImmutableMap());
   }
@@ -204,30 +206,36 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * keys, regardless of whether or not they have been soft-deleted.
    *
    * <p>Used by both the cached (w/o deletion check) and the non-cached (with deletion check) calls.
+   *
+   * <p>Note that in the cached case, we wish to run this outside of any transaction because we may
+   * be loading many entities, going over the Datastore limit on the number of enrolled entity
+   * groups per transaction (25). If we require consistency, however, we must use a transaction.
+   *
+   * @param inTransaction whether or not to use an Objectify transaction
    */
   private static <E extends EppResource>
       ImmutableMap<String, ForeignKeyIndex<E>> loadIndexesFromStore(
-          Class<E> clazz, Iterable<String> foreignKeys) {
+          Class<E> clazz, Collection<String> foreignKeys, boolean inTransaction) {
     if (tm().isOfy()) {
+      Class<ForeignKeyIndex<E>> fkiClass = mapToFkiClass(clazz);
       return ImmutableMap.copyOf(
-          tm().doTransactionless(() -> ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys)));
+          inTransaction
+              ? ofy().load().type(fkiClass).ids(foreignKeys)
+              : tm().doTransactionless(() -> ofy().load().type(fkiClass).ids(foreignKeys)));
     } else {
       String property = RESOURCE_CLASS_TO_FKI_PROPERTY.get(clazz);
       ImmutableList<ForeignKeyIndex<E>> indexes =
           tm().transact(
-                  () -> {
-                    String entityName =
-                        jpaTm().getEntityManager().getMetamodel().entity(clazz).getName();
-                    return jpaTm()
-                        .query(
-                            String.format(
-                                "FROM %s WHERE %s IN :propertyValue", entityName, property),
-                            clazz)
-                        .setParameter("propertyValue", foreignKeys)
-                        .getResultStream()
-                        .map(e -> ForeignKeyIndex.create(e, e.getDeletionTime()))
-                        .collect(toImmutableList());
-                  });
+                  () ->
+                      jpaTm()
+                          .getEntityManager()
+                          .createQuery(
+                              CriteriaQueryBuilder.create(clazz)
+                                  .whereFieldIsIn(property, foreignKeys)
+                                  .build())
+                          .getResultStream()
+                          .map(e -> ForeignKeyIndex.create(e, e.getDeletionTime()))
+                          .collect(toImmutableList()));
       // We need to find and return the entities with the maximum deletionTime for each foreign key.
       return Multimaps.index(indexes, ForeignKeyIndex::getForeignKey).asMap().entrySet().stream()
           .map(
@@ -250,7 +258,8 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
           return Optional.ofNullable(
               loadIndexesFromStore(
                       RESOURCE_CLASS_TO_FKI_CLASS.inverse().get(key.getKind()),
-                      ImmutableSet.of(foreignKey))
+                      ImmutableSet.of(foreignKey),
+                      false)
                   .get(foreignKey));
         }
 
@@ -266,7 +275,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               Streams.stream(keys).map(v -> v.getSqlKey().toString()).collect(toImmutableSet());
           ImmutableSet<VKey<ForeignKeyIndex<?>>> typedKeys = ImmutableSet.copyOf(keys);
           ImmutableMap<String, ? extends ForeignKeyIndex<? extends EppResource>> existingFkis =
-              loadIndexesFromStore(resourceClass, foreignKeys);
+              loadIndexesFromStore(resourceClass, foreignKeys, false);
           // ofy() omits keys that don't have values in Datastore, so re-add them in
           // here with Optional.empty() values.
           return Maps.asMap(
@@ -318,7 +327,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * reasons, and are OK with the trade-offs in loss of transactional consistency.
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> loadCached(
-      Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
+      Class<E> clazz, Collection<String> foreignKeys, final DateTime now) {
     if (!RegistryConfig.isEppResourceCachingEnabled()) {
       return tm().doTransactionless(() -> load(clazz, foreignKeys, now));
     }

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -37,12 +37,17 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.host.HostHistory;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.QueryComposer;
 import google.registry.persistence.transaction.TransactionManager;
+import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import javax.annotation.Nullable;
+import javax.persistence.NoResultException;
+import javax.persistence.NonUniqueResultException;
 import org.joda.time.DateTime;
 
 /** Datastore implementation of {@link TransactionManager}. */
@@ -302,6 +307,11 @@ public class DatastoreTransactionManager implements TransactionManager {
     syncIfTransactionless(getOfy().deleteWithoutBackup().entity(entity));
   }
 
+  @Override
+  public <T> QueryComposer<T> createQueryComposer(Class<T> entity) {
+    return new DatastoreQueryComposerImpl(entity);
+  }
+
   @Override
   public void clearSessionCache() {
     getOfy().clearSessionCache();
@@ -363,4 +373,45 @@ public class DatastoreTransactionManager implements TransactionManager {
     }
     return obj;
   }
+
+  private static class DatastoreQueryComposerImpl<T> extends QueryComposer<T> {
+    DatastoreQueryComposerImpl(Class<T> entityClass) {
+      super(entityClass);
+    }
+
+    Query<T> buildQuery() {
+      Query<T> result = ofy().load().type(entityClass);
+      for (WhereClause pred : predicates) {
+        result = result.filter(pred.fieldName + pred.comparator.getDatastoreString(), pred.value);
+      }
+
+      if (orderBy != null) {
+        result = result.order(orderBy);
+      }
+
+      return result;
+    }
+
+    @Override
+    public Optional<T> first() {
+      return Optional.ofNullable(buildQuery().first().now());
+    }
+
+    @Override
+    public T getSingleResult() {
+      List<T> results = buildQuery().limit(2).list();
+      if (results.size() == 0) {
+        // The exception text here is the same as what we get for JPA queries.
+        throw new NoResultException("No entity found for query");
+      } else if (results.size() > 1) {
+        throw new NonUniqueResultException("More than one result found for getSingleResult query");
+      }
+      return results.get(0);
+    }
+
+    @Override
+    public Stream<T> stream() {
+      return Streams.stream(buildQuery());
+    }
+  }
 }

core/src/main/java/google/registry/model/ofy/EntityWritePriorities.java

@@ -41,10 +41,11 @@ public class EntityWritePriorities {
    */
   static final ImmutableMap<String, Integer> CLASS_PRIORITIES =
       ImmutableMap.of(
-          "ContactResource", -15,
-          "HistoryEntry", -10,
           "AllocationToken", -9,
-          "DomainBase", 10);
+          "ContactResource", 8,
+          "HostResource", 9,
+          "DomainBase", 10,
+          "HistoryEntry", 20);
 
   // The beginning of the range of priority numbers reserved for delete.  This must be greater than
   // any of the values in CLASS_PRIORITIES by enough overhead to accommodate  any negative values in

core/src/main/java/google/registry/model/rde/RdeRevision.java

@@ -103,6 +103,15 @@ public final class RdeRevision extends BackupGroupRoot implements NonReplicatedE
     return revisionOptional.map(rdeRevision -> rdeRevision.revision + 1).orElse(0);
   }
 
+  /** Returns the latest revision of the report already generated for the given triplet. */
+  public static Optional<Integer> getCurrentRevision(String tld, DateTime date, RdeMode mode) {
+    int nextRevision = getNextRevision(tld, date, mode);
+    if (nextRevision == 0) {
+      return Optional.empty();
+    }
+    return Optional.of(nextRevision - 1);
+  }
+
   /**
    * Sets the revision ID for a given triplet.
    *

core/src/main/java/google/registry/model/registry/label/PremiumListDualDao.java

@@ -16,14 +16,16 @@ package google.registry.model.registry.label;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.Streams;
+import google.registry.model.DatabaseMigrationUtils;
+import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.label.PremiumList.PremiumListEntry;
 import google.registry.schema.tld.PremiumListSqlDao;
 import java.util.List;
 import java.util.Optional;
+import org.joda.money.BigMoney;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 
@@ -46,8 +48,7 @@ public class PremiumListDualDao {
    * or absent if no such list exists.
    */
   public static Optional<PremiumList> getLatestRevision(String premiumListName) {
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       return PremiumListDatastoreDao.getLatestRevision(premiumListName);
     } else {
       return PremiumListSqlDao.getLatestRevision(premiumListName);
@@ -68,16 +69,14 @@ public class PremiumListDualDao {
     }
     String premiumListName = registry.getPremiumList().getName();
     Optional<Money> primaryResult;
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       primaryResult =
           PremiumListDatastoreDao.getPremiumPrice(premiumListName, label, registry.getTldStr());
     } else {
       primaryResult = PremiumListSqlDao.getPremiumPrice(premiumListName, label);
     }
     // Also load the value from the secondary DB, compare the two results, and log if different.
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       suppressExceptionUnlessInTest(
           () -> {
             Optional<Money> secondaryResult =
@@ -120,8 +119,7 @@ public class PremiumListDualDao {
    */
   public static PremiumList save(String name, List<String> inputData) {
     PremiumList result;
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       result = PremiumListDatastoreDao.save(name, inputData);
       suppressExceptionUnlessInTest(
           () -> PremiumListSqlDao.save(name, inputData), "Error when saving premium list to SQL.");
@@ -141,8 +139,7 @@ public class PremiumListDualDao {
    * secondary database.
    */
   public static void delete(PremiumList premiumList) {
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       PremiumListDatastoreDao.delete(premiumList);
       suppressExceptionUnlessInTest(
           () -> PremiumListSqlDao.delete(premiumList),
@@ -159,8 +156,7 @@ public class PremiumListDualDao {
   public static boolean exists(String premiumListName) {
     // It may seem like overkill, but loading the list has ways been the way we check existence and
     // given that we usually load the list around the time we check existence, we'll hit the cache
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       return PremiumListDatastoreDao.getLatestRevision(premiumListName).isPresent();
     } else {
       return PremiumListSqlDao.getLatestRevision(premiumListName).isPresent();
@@ -179,8 +175,7 @@ public class PremiumListDualDao {
                 () ->
                     new IllegalArgumentException(
                         String.format("No premium list with name %s.", premiumListName)));
-    // TODO(gbrodman): Use Sarah's DB scheduler instead of this isOfy check
-    if (tm().isOfy()) {
+    if (DatabaseMigrationUtils.isDatastore(TransitionId.DOMAIN_LABEL_LISTS)) {
       return PremiumListDatastoreDao.loadPremiumListEntriesUncached(premiumList);
     } else {
       CurrencyUnit currencyUnit = premiumList.getCurrency();
@@ -188,7 +183,7 @@ public class PremiumListDualDao {
           .map(
               premiumEntry ->
                   new PremiumListEntry.Builder()
-                      .setPrice(Money.of(currencyUnit, premiumEntry.getPrice()))
+                      .setPrice(BigMoney.of(currencyUnit, premiumEntry.getPrice()).toMoney())
                       .setLabel(premiumEntry.getDomainLabel())
                       .build())
           .collect(toImmutableList());

core/src/main/java/google/registry/model/server/Lock.java

@@ -15,19 +15,20 @@
 package google.registry.model.server;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.flogger.FluentLogger;
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
+import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
@@ -190,12 +191,17 @@ public class Lock extends ImmutableObject implements DatastoreOnlyEntity, Serial
     // access to resources like GCS that can't be transactionally rolled back. Therefore, the lock
     // must be definitively acquired before it is used, even when called inside another transaction.
     AcquireResult acquireResult =
-        tm().transactNew(
+        ofyTm()
+            .transactNew(
                 () -> {
-                  DateTime now = tm().getTransactionTime();
+                  DateTime now = ofyTm().getTransactionTime();
 
                   // Checking if an unexpired lock still exists - if so, the lock can't be acquired.
-                  Lock lock = ofy().load().type(Lock.class).id(lockId).now();
+                  Lock lock =
+                      ofyTm()
+                          .loadByKeyIfPresent(
+                              VKey.createOfy(Lock.class, Key.create(Lock.class, lockId)))
+                          .orElse(null);
                   if (lock != null) {
                     logger.atInfo().log(
                         "Loaded existing lock: %s for request: %s", lock.lockId, lock.requestLogId);
@@ -218,7 +224,7 @@ public class Lock extends ImmutableObject implements DatastoreOnlyEntity, Serial
                   // Locks are not parented under an EntityGroupRoot (so as to avoid write
                   // contention) and
                   // don't need to be backed up.
-                  ofy().saveWithoutBackup().entity(newLock);
+                  ofyTm().putWithoutBackup(newLock);
 
                   return AcquireResult.create(now, lock, newLock, lockState);
                 });
@@ -231,21 +237,26 @@ public class Lock extends ImmutableObject implements DatastoreOnlyEntity, Serial
   /** Release the lock. */
   public void release() {
     // Just use the default clock because we aren't actually doing anything that will use the clock.
-    tm().transact(
+    ofyTm()
+        .transact(
             () -> {
               // To release a lock, check that no one else has already obtained it and if not
               // delete it. If the lock in Datastore was different then this lock is gone already;
               // this can happen if release() is called around the expiration time and the lock
               // expires underneath us.
-              Lock loadedLock = ofy().load().type(Lock.class).id(lockId).now();
+              Lock loadedLock =
+                  ofyTm()
+                      .loadByKeyIfPresent(
+                          VKey.createOfy(Lock.class, Key.create(Lock.class, lockId)))
+                      .orElse(null);
               if (Lock.this.equals(loadedLock)) {
                 // Use noBackupOfy() so that we don't create a commit log entry for deleting the
                 // lock.
                 logger.atInfo().log("Deleting lock: %s", lockId);
-                ofy().deleteWithoutBackup().entity(Lock.this);
+                ofyTm().deleteWithoutBackup(Lock.this);
 
                 lockMetrics.recordRelease(
-                    resourceName, tld, new Duration(acquiredTime, tm().getTransactionTime()));
+                    resourceName, tld, new Duration(acquiredTime, ofyTm().getTransactionTime()));
               } else {
                 logger.atSevere().log(
                     "The lock we acquired was transferred to someone else before we"

core/src/main/java/google/registry/model/server/ServerSecret.java

@@ -15,8 +15,6 @@
 package google.registry.model.server;
 
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -35,10 +33,10 @@ import google.registry.model.common.CrossTldSingleton;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.NonReplicatedEntity;
 import java.nio.ByteBuffer;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import javax.persistence.Column;
-import javax.persistence.Id;
 import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 
@@ -67,22 +65,28 @@ public class ServerSecret extends CrossTldSingleton implements NonReplicatedEnti
               });
 
   private static ServerSecret retrieveAndSaveSecret() {
-    VKey<ServerSecret> key =
+    VKey<ServerSecret> vkey =
         VKey.create(
             ServerSecret.class,
             SINGLETON_ID,
             Key.create(getCrossTldKey(), ServerSecret.class, SINGLETON_ID));
+    if (tm().isOfy()) {
+      // Attempt a quick load if we're in ofy first to short-circuit sans transaction
+      Optional<ServerSecret> secretWithoutTransaction = tm().loadByKeyIfPresent(vkey);
+      if (secretWithoutTransaction.isPresent()) {
+        return secretWithoutTransaction.get();
+      }
+    }
     return tm().transact(
             () -> {
-              // transactionally create a new ServerSecret (once per app setup) if necessary.
-              // return the ofy() result during Datastore-primary phase
-              ServerSecret secret =
-                  ofyTm().loadByKeyIfPresent(key).orElseGet(() -> create(UUID.randomUUID()));
-              // During a dual-write period, write it to both Datastore and SQL
-              // even if we didn't have to retrieve it from the DB
-              ofyTm().transact(() -> ofyTm().putWithoutBackup(secret));
-              jpaTm().transact(() -> jpaTm().putWithoutBackup(secret));
-              return secret;
+              // Make sure we're in a transaction and attempt to load any existing secret, then
+              // create it if it's absent.
+              Optional<ServerSecret> secret = tm().loadByKeyIfPresent(vkey);
+              if (!secret.isPresent()) {
+                secret = Optional.of(create(UUID.randomUUID()));
+                tm().insertWithoutBackup(secret.get());
+              }
+              return secret.get();
             });
   }
 
@@ -102,7 +106,6 @@ public class ServerSecret extends CrossTldSingleton implements NonReplicatedEnti
   @Transient long leastSignificant;
 
   /** The UUID value itself. */
-  @Id
   @Column(columnDefinition = "uuid")
   @Ignore
   UUID secret;

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -25,15 +25,11 @@ import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
-import google.registry.model.tmch.TmchCrl.TmchCrlId;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.NonReplicatedEntity;
-import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
 import javax.persistence.Column;
-import javax.persistence.Id;
-import javax.persistence.IdClass;
 import org.joda.time.DateTime;
 
 /** Datastore singleton for ICANN's TMCH CA certificate revocation list (CRL). */
@@ -41,22 +37,26 @@ import org.joda.time.DateTime;
 @javax.persistence.Entity
 @Immutable
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-@IdClass(TmchCrlId.class)
 public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEntity {
 
-  @Id String crl;
+  @Column(name = "certificateRevocations", nullable = false)
+  String crl;
 
-  @Id DateTime updated;
+  @Column(name = "updateTimestamp", nullable = false)
+  DateTime updated;
 
-  @Id String url;
+  @Column(nullable = false)
+  String url;
 
   /** Returns the singleton instance of this entity, without memoization. */
   public static Optional<TmchCrl> get() {
-    VKey<TmchCrl> key =
-        VKey.create(
-            TmchCrl.class, SINGLETON_ID, Key.create(getCrossTldKey(), TmchCrl.class, SINGLETON_ID));
-    // return the ofy() result during Datastore-primary phase
-    return ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(key));
+    return tm().transact(
+            () ->
+                tm().loadByKeyIfPresent(
+                        VKey.create(
+                            TmchCrl.class,
+                            SINGLETON_ID,
+                            Key.create(getCrossTldKey(), TmchCrl.class, SINGLETON_ID))));
   }
 
   /**
@@ -75,13 +75,7 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
               tmchCrl.crl = checkNotNull(crl, "crl");
               tmchCrl.url = checkNotNull(url, "url");
               ofyTm().transactNew(() -> ofyTm().putWithoutBackup(tmchCrl));
-              jpaTm()
-                  .transactNew(
-                      () -> {
-                        // Delete the old one and insert the new one
-                        jpaTm().query("DELETE FROM TmchCrl").executeUpdate();
-                        jpaTm().putWithoutBackup(tmchCrl);
-                      });
+              jpaTm().transactNew(() -> jpaTm().putWithoutBackup(tmchCrl));
             });
   }
 
@@ -99,26 +93,4 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
   public final DateTime getUpdated() {
     return updated;
   }
-
-  static class TmchCrlId implements Serializable {
-
-    @Column(name = "certificateRevocations")
-    String crl;
-
-    @Column(name = "updateTimestamp")
-    DateTime updated;
-
-    String url;
-
-    /** Hibernate requires this default constructor. */
-    private TmchCrlId() {}
-
-    static TmchCrlId create(String crl, DateTime updated, String url) {
-      TmchCrlId result = new TmchCrlId();
-      result.crl = crl;
-      result.updated = updated;
-      result.url = url;
-      return result;
-    }
-  }
 }

core/src/main/java/google/registry/module/backend/BackendRequestComponent.java

@@ -31,6 +31,7 @@ import google.registry.batch.RelockDomainAction;
 import google.registry.batch.ResaveAllEppResourcesAction;
 import google.registry.batch.ResaveEntityAction;
 import google.registry.batch.WipeOutCloudSqlAction;
+import google.registry.batch.WipeoutDatastoreAction;
 import google.registry.cron.CommitLogFanoutAction;
 import google.registry.cron.CronModule;
 import google.registry.cron.TldFanoutAction;
@@ -208,6 +209,8 @@ interface BackendRequestComponent {
 
   WipeOutCloudSqlAction wipeOutCloudSqlAction();
 
+  WipeoutDatastoreAction wipeoutDatastoreAction();
+
   @Subcomponent.Builder
   abstract class Builder implements RequestComponentBuilder<BackendRequestComponent> {
 

core/src/main/java/google/registry/module/frontend/FrontendComponent.java

@@ -30,6 +30,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.frontend.FrontendRequestComponent.FrontendRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
@@ -58,6 +59,7 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
       UrlFetchTransportModule.class,

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -30,6 +30,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.pubapi.PubApiRequestComponent.PubApiRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
@@ -56,6 +57,7 @@ import javax.inject.Singleton;
       KmsModule.class,
       NetHttpTransportModule.class,
       PubApiRequestComponentModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
       UrlFetchTransportModule.class,

core/src/main/java/google/registry/module/tools/ToolsComponent.java

@@ -32,6 +32,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.tools.ToolsRequestComponent.ToolsRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
@@ -61,6 +62,7 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
       ToolsRequestComponentModule.class,

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -31,7 +31,6 @@ import dagger.BindsOptionalOf;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
-import google.registry.keyring.kms.KmsKeyring;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
@@ -202,18 +201,11 @@ public abstract class PersistenceModule {
   @Singleton
   @AppEngineJpaTm
   static JpaTransactionManager provideAppEngineJpaTm(
-      @Config("cloudSqlUsername") String username,
-      KmsKeyring kmsKeyring,
       SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
-    validateAndSetCredential(
-        credentialStore,
-        new RobotUser(RobotId.NOMULUS),
-        overrides,
-        username,
-        kmsKeyring.getCloudSqlPassword());
+    setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides);
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -258,20 +250,13 @@ public abstract class PersistenceModule {
   @Singleton
   @NomulusToolJpaTm
   static JpaTransactionManager provideNomulusToolJpaTm(
-      @Config("toolsCloudSqlUsername") String username,
-      KmsKeyring kmsKeyring,
       SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       @CloudSqlClientCredential Credential credential,
       Clock clock) {
     CloudSqlCredentialSupplier.setupCredentialSupplier(credential);
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
-    validateAndSetCredential(
-        credentialStore,
-        new RobotUser(RobotId.TOOL),
-        overrides,
-        username,
-        kmsKeyring.getToolsCloudSqlPassword());
+    setSqlCredential(credentialStore, new RobotUser(RobotId.TOOL), overrides);
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -349,35 +334,15 @@ public abstract class PersistenceModule {
     return emf;
   }
 
-  /**
-   * Verifies that the credential from the Secret Manager matches the one currently in use, and
-   * configures JPA with the credential from the Secret Manager.
-   *
-   * <p>This is a helper for the transition to the Secret Manager, and will be removed once data and
-   * permissions are properly set up for all projects.
-   */
-  private static void validateAndSetCredential(
-      SqlCredentialStore credentialStore,
-      SqlUser sqlUser,
-      Map<String, String> overrides,
-      String expectedLogin,
-      String expectedPassword) {
+  /** Configures JPA with the credential from the Secret Manager. */
+  private static void setSqlCredential(
+      SqlCredentialStore credentialStore, SqlUser sqlUser, Map<String, String> overrides) {
     try {
       SqlCredential credential = credentialStore.getCredential(sqlUser);
-      checkState(
-          credential.login().equals(expectedLogin),
-          "Wrong login for %s. Expecting %s, found %s.",
-          sqlUser.geUserName(),
-          expectedLogin,
-          credential.login());
-      checkState(
-          credential.password().equals(expectedPassword),
-          "Wrong password for %s.",
-          sqlUser.geUserName());
       overrides.put(Environment.USER, credential.login());
       overrides.put(Environment.PASS, credential.password());
-      logger.atWarning().log("Credentials in the kerying and the secret manager match.");
     } catch (Throwable e) {
+      // TODO(b/184631990): after SQL becomes primary, throw an exception to fail fast
       logger.atSevere().withCause(e).log("Failed to get SQL credential from Secret Manager");
     }
   }

core/src/main/java/google/registry/persistence/VKey.java

@@ -223,4 +223,14 @@ public class VKey<T> extends ImmutableObject implements Serializable {
   public static <T> VKey<T> from(Key<T> key) {
     return VKeyTranslatorFactory.createVKey(key);
   }
+
+  /**
+   * Construct a VKey from the string representation of an ofy key.
+   *
+   * <p>TODO(b/184350590): After migration, we'll want remove the ofy key dependency from this.
+   */
+  @Nullable
+  public static <T> VKey<T> fromWebsafeKey(String ofyKeyRepr) {
+    return from(Key.create(ofyKeyRepr));
+  }
 }

core/src/main/java/google/registry/persistence/transaction/CriteriaQueryBuilder.java

@@ -18,6 +18,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import com.google.common.collect.ImmutableList;
 import java.util.Collection;
+import javax.persistence.EntityManager;
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Expression;
@@ -35,7 +36,7 @@ import javax.persistence.criteria.Root;
 public class CriteriaQueryBuilder<T> {
 
   /** Functional interface that defines the 'where' operator, e.g. {@link CriteriaBuilder#equal}. */
-  public interface WhereClause<U> {
+  public interface WhereOperator<U> {
     Predicate predicate(Expression<U> expression, U object);
   }
 
@@ -50,7 +51,8 @@ public class CriteriaQueryBuilder<T> {
   }
 
   /** Adds a WHERE clause to the query, given the specified operation, field, and value. */
-  public <V> CriteriaQueryBuilder<T> where(String fieldName, WhereClause<V> whereClause, V value) {
+  public <V> CriteriaQueryBuilder<T> where(
+      String fieldName, WhereOperator<V> whereClause, V value) {
     Expression<V> expression = root.get(fieldName);
     return where(whereClause.predicate(expression, value));
   }
@@ -94,7 +96,12 @@ public class CriteriaQueryBuilder<T> {
 
   /** Creates a query builder that will SELECT from the given class. */
   public static <T> CriteriaQueryBuilder<T> create(Class<T> clazz) {
-    CriteriaQuery<T> query = jpaTm().getEntityManager().getCriteriaBuilder().createQuery(clazz);
+    return create(jpaTm().getEntityManager(), clazz);
+  }
+
+  /** Creates a query builder for the given entity manager. */
+  public static <T> CriteriaQueryBuilder<T> create(EntityManager em, Class<T> clazz) {
+    CriteriaQuery<T> query = em.getCriteriaBuilder().createQuery(clazz);
     Root<T> root = query.from(clazz);
     query = query.select(root);
     return new CriteriaQueryBuilder<>(query, root);

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -43,10 +43,12 @@ import google.registry.util.Clock;
 import google.registry.util.Retrier;
 import google.registry.util.SystemSleeper;
 import java.lang.reflect.Field;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import javax.persistence.EntityManager;
 import javax.persistence.EntityManagerFactory;
@@ -530,6 +532,11 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     delete(entity);
   }
 
+  @Override
+  public <T> QueryComposer<T> createQueryComposer(Class<T> entity) {
+    return new JpaQueryComposerImpl<T>(entity, getEntityManager());
+  }
+
   @Override
   public void clearSessionCache() {
     // This is an intended no-op method as there is no session cache in Postgresql.
@@ -681,4 +688,44 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
       }
     }
   }
+
+  private static class JpaQueryComposerImpl<T> extends QueryComposer<T> {
+
+    EntityManager em;
+
+    JpaQueryComposerImpl(Class<T> entityClass, EntityManager em) {
+      super(entityClass);
+      this.em = em;
+    }
+
+    private TypedQuery<T> buildQuery() {
+      CriteriaQueryBuilder<T> queryBuilder = CriteriaQueryBuilder.create(em, entityClass);
+
+      for (WhereClause<?> pred : predicates) {
+        pred.addToCriteriaQueryBuilder(queryBuilder);
+      }
+
+      if (orderBy != null) {
+        queryBuilder.orderByAsc(orderBy);
+      }
+
+      return em.createQuery(queryBuilder.build());
+    }
+
+    @Override
+    public Optional<T> first() {
+      List<T> results = buildQuery().setMaxResults(1).getResultList();
+      return results.size() > 0 ? Optional.of(results.get(0)) : Optional.empty();
+    }
+
+    @Override
+    public T getSingleResult() {
+      return buildQuery().getSingleResult();
+    }
+
+    @Override
+    public Stream<T> stream() {
+      return buildQuery().getResultStream();
+    }
+  }
 }

core/src/main/java/google/registry/persistence/transaction/QueryComposer.java

@@ -0,0 +1,190 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Function;
+import google.registry.persistence.transaction.CriteriaQueryBuilder.WhereOperator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import javax.persistence.criteria.CriteriaBuilder;
+
+/**
+ * Creates queries that can be used both for objectify and JPA.
+ *
+ * <p>Example usage:
+ *
+ * <pre>
+ *    tm().createQueryComposer(EntityType.class)
+ *        .where("fieldName", Comparator.EQ, "value")
+ *        .orderBy("fieldName")
+ *        .stream()
+ * </pre>
+ */
+public abstract class QueryComposer<T> {
+
+  // The class whose entities we're querying.  Note that this limits us to single table queries in
+  // SQL.  In datastore, there's really no other kind of query.
+  protected Class<T> entityClass;
+
+  // Field to order by, if any.  Null if we don't care about order.
+  @Nullable protected String orderBy;
+
+  protected List<WhereClause<?>> predicates = new ArrayList<WhereClause<?>>();
+
+  protected QueryComposer(Class<T> entityClass) {
+    this.entityClass = entityClass;
+  }
+
+  /**
+   * Introduce a "where" clause to the query.
+   *
+   * <p>Causes the query to return only results where the field and value have the relationship
+   * specified by the comparator. For example, "field EQ value", "field GT value" etc.
+   */
+  public <U extends Comparable<? super U>> QueryComposer<T> where(
+      String fieldName, Comparator comparator, U value) {
+    predicates.add(new WhereClause(fieldName, comparator, value));
+    return this;
+  }
+
+  /**
+   * Order the query results by the value of the specified field.
+   *
+   * <p>TODO(mmuller): add the ability to do descending sort order.
+   */
+  public QueryComposer<T> orderBy(String fieldName) {
+    orderBy = fieldName;
+    return this;
+  }
+
+  /** Returns the first result of the query or an empty optional if there is none. */
+  public abstract Optional<T> first();
+
+  /**
+   * Returns the one and only result of a query.
+   *
+   * <p>Throws a {@link javax.persistence.NonUniqueResultException} if there is more than one
+   * result, throws {@link javax.persistence.NoResultException} if no results are found.
+   */
+  public abstract T getSingleResult();
+
+  /** Returns the results of the query as a stream. */
+  public abstract Stream<T> stream();
+
+  // We have to wrap the CriteriaQueryBuilder predicate factories in our own functions because at
+  // the point where we pass them to the Comparator constructor, the compiler can't determine which
+  // of the overloads to use since there is no "value" object for context.
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> equal(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::equal;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> lessThan(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::lessThan;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> lessThanOrEqualTo(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::lessThanOrEqualTo;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> greaterThanOrEqualTo(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::greaterThanOrEqualTo;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> greaterThan(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::greaterThan;
+  }
+
+  /**
+   * Enum used to specify comparison operations, e.g. {@code where("fieldName", Comparator.NE,
+   * "someval")'}.
+   *
+   * <p>These contain values that specify the comparison behavior for both objectify and criteria
+   * queries. For objectify, we provide a string to be appended to the field name in a {@code
+   * filter()} expression. For criteria queries we provide a function that knows how to obtain a
+   * {@link WhereOperator} from a {@link CriteriaBuilder}.
+   *
+   * <p>Note that the objectify strings for comparators other than equality are preceded by a space
+   * because {@code filter()} expects the fieldname to be separated from the operator by a space.
+   */
+  public enum Comparator {
+    /**
+     * Return only records whose field is equal to the value.
+     *
+     * <p>Note that the datastore string for this is empty, which is consistent with the way {@code
+     * filter()} works (it uses an unadorned field name to check for equality).
+     */
+    EQ("", QueryComposer::equal),
+
+    /** Return only records whose field is less than the value. */
+    LT(" <", QueryComposer::lessThan),
+
+    /** Return only records whose field is less than or equal to the value. */
+    LTE(" <=", QueryComposer::lessThanOrEqualTo),
+
+    /** Return only records whose field is greater than or equal to the value. */
+    GTE(" >=", QueryComposer::greaterThanOrEqualTo),
+
+    /** Return only records whose field is greater than the value. */
+    GT(" >", QueryComposer::greaterThan);
+
+    private final String datastoreString;
+
+    @SuppressWarnings("ImmutableEnumChecker") // Functions are immutable.
+    private final Function<CriteriaBuilder, WhereOperator<?>> operatorFactory;
+
+    Comparator(
+        String datastoreString, Function<CriteriaBuilder, WhereOperator<?>> operatorFactory) {
+      this.datastoreString = datastoreString;
+      this.operatorFactory = operatorFactory;
+    }
+
+    public String getDatastoreString() {
+      return datastoreString;
+    }
+
+    public Function<CriteriaBuilder, WhereOperator<?>> getComparisonFactory() {
+      return operatorFactory;
+    }
+  };
+
+  protected static class WhereClause<U extends Comparable<? super U>> {
+    public String fieldName;
+    public Comparator comparator;
+    public U value;
+
+    WhereClause(String fieldName, Comparator comparator, U value) {
+      this.fieldName = fieldName;
+      this.comparator = comparator;
+      this.value = value;
+    }
+
+    public void addToCriteriaQueryBuilder(CriteriaQueryBuilder queryBuilder) {
+      CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
+      queryBuilder.where(
+          fieldName, comparator.getComparisonFactory().apply(criteriaBuilder), value);
+    }
+  }
+}

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -273,6 +273,9 @@ public interface TransactionManager {
    */
   void deleteWithoutBackup(Object entity);
 
+  /** Returns a QueryComposer which can be used to perform queries against the current database. */
+  <T> QueryComposer<T> createQueryComposer(Class<T> entity);
+
   /** Clears the session cache if the underlying database is Datastore, otherwise it is a no-op. */
   void clearSessionCache();
 

core/src/main/java/google/registry/privileges/secretmanager/KeyringSecretStore.java

@@ -0,0 +1,60 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Storage for 'keyring' secrets, backed by the Secret Manager.
+ *
+ * <p>This store is for secrets and credentials that must be set up manually and/or do not require
+ * non-disruptive password changes, e.g., passwords to regulatory reporting websites, which are used
+ * by cron jobs.
+ *
+ * <p>In contrast, the {@link SqlCredentialStore} is designed to support non-disruptive credential
+ * changes with Cloud SQL.
+ */
+public class KeyringSecretStore {
+  static final String SECRET_NAME_PREFIX = "keyring-";
+
+  private final SecretManagerClient csmClient;
+
+  @Inject
+  public KeyringSecretStore(SecretManagerClient csmClient) {
+    this.csmClient = csmClient;
+  }
+
+  public void createOrUpdateSecret(String label, byte[] data) {
+    String secretId = decorateLabel(label);
+    csmClient.createSecretIfAbsent(secretId);
+    csmClient.addSecretVersion(secretId, new String(data, StandardCharsets.UTF_8));
+  }
+
+  public byte[] getSecret(String label) {
+    return csmClient
+        .getSecretData(decorateLabel(label), Optional.empty())
+        .getBytes(StandardCharsets.UTF_8);
+  }
+
+  static String decorateLabel(String label) {
+    checkArgument(!isNullOrEmpty(label), "null or empty label");
+    return SECRET_NAME_PREFIX + label;
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClient.java

@@ -44,6 +44,15 @@ public interface SecretManagerClient {
   /** Returns the {@link SecretVersionState} of all secrets with {@code secretId}. */
   Iterable<SecretVersionState> listSecretVersions(String secretId);
 
+  /** Creates a secret if it does not already exists. */
+  default void createSecretIfAbsent(String secretId) {
+    try {
+      createSecret(secretId);
+    } catch (SecretAlreadyExistsException ignore) {
+      // Not a problem.
+    }
+  }
+
   /**
    * Returns the version strings of all secrets in the given {@code state} with {@code secretId}.
    */

core/src/main/java/google/registry/privileges/secretmanager/SqlCredentialStore.java

@@ -17,7 +17,6 @@ package google.registry.privileges.secretmanager;
 import com.google.cloud.secretmanager.v1.SecretVersionName;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
-import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
 import java.util.Optional;
 import javax.inject.Inject;
 
@@ -74,17 +73,9 @@ public class SqlCredentialStore {
     }
   }
 
-  private void createSecretIfAbsent(String secretId) {
-    try {
-      csmClient.createSecret(secretId);
-    } catch (SecretAlreadyExistsException ignore) {
-      // Not a problem.
-    }
-  }
-
   private SecretVersionName saveCredentialData(SqlUser user, String password) {
     String credentialDataSecretId = getCredentialDataSecretId(user, dbInstance);
-    createSecretIfAbsent(credentialDataSecretId);
+    csmClient.createSecretIfAbsent(credentialDataSecretId);
     String credentialVersion =
         csmClient.addSecretVersion(
             credentialDataSecretId,
@@ -94,7 +85,7 @@ public class SqlCredentialStore {
 
   private void saveLiveLabel(SqlUser user, SecretVersionName dataVersionName) {
     String liveLabelSecretId = getLiveLabelSecretId(user, dbInstance);
-    createSecretIfAbsent(liveLabelSecretId);
+    csmClient.createSecretIfAbsent(liveLabelSecretId);
     csmClient.addSecretVersion(liveLabelSecretId, dataVersionName.toString());
   }
 

core/src/main/java/google/registry/rde/RdeReportAction.java

@@ -24,6 +24,7 @@ import static google.registry.request.Action.Method.POST;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
@@ -31,6 +32,7 @@ import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.common.Cursor;
 import google.registry.model.common.Cursor.CursorType;
 import google.registry.model.rde.RdeNamingUtils;
+import google.registry.model.rde.RdeRevision;
 import google.registry.model.registry.Registry;
 import google.registry.rde.EscrowTaskRunner.EscrowTask;
 import google.registry.request.Action;
@@ -41,6 +43,7 @@ import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.bouncycastle.openpgp.PGPPrivateKey;
 import org.joda.time.DateTime;
@@ -56,6 +59,8 @@ import org.joda.time.Duration;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public final class RdeReportAction implements Runnable, EscrowTask {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   static final String PATH = "/_dr/task/rdeReport";
 
   @Inject GcsUtils gcsUtils;
@@ -76,8 +81,9 @@ public final class RdeReportAction implements Runnable, EscrowTask {
 
   @Override
   public void runWithLock(DateTime watermark) throws Exception {
-    Cursor cursor =
-        transactIfJpaTm(() -> tm().loadByKey(Cursor.createVKey(CursorType.RDE_UPLOAD, tld)));
+    Optional<Cursor> cursor =
+        transactIfJpaTm(
+            () -> tm().loadByKeyIfPresent(Cursor.createVKey(CursorType.RDE_UPLOAD, tld)));
     DateTime cursorTime = getCursorTimeOrStartOfTime(cursor);
     if (isBeforeOrAt(cursorTime, watermark)) {
       throw new NoContentException(
@@ -86,12 +92,17 @@ public final class RdeReportAction implements Runnable, EscrowTask {
                   + "last upload completion was at %s",
               tld, watermark, cursorTime));
     }
-    String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, 0);
+    int revision =
+        RdeRevision.getCurrentRevision(tld, watermark, FULL)
+            .orElseThrow(
+                () -> new IllegalStateException("RdeRevision was not set on generated deposit"));
+    String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
     GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
     verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
     reporter.send(readReportFromGcs(reportFilename));
     response.setContentType(PLAIN_TEXT_UTF_8);
     response.setPayload(String.format("OK %s %s\n", tld, watermark));
+    logger.atInfo().log("Successfully sent report %s.", reportFilename);
   }
 
   /** Reads and decrypts the XML file from cloud storage. */

core/src/main/java/google/registry/rde/RdeStagingReducer.java

@@ -20,8 +20,8 @@ import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGc
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
@@ -210,7 +210,11 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
     tm().transact(
             () -> {
               Registry registry = Registry.get(tld);
-              Cursor cursor = ofy().load().key(Cursor.createKey(key.cursor(), registry)).now();
+              Optional<Cursor> cursor =
+                  transactIfJpaTm(
+                      () ->
+                          tm().loadByKeyIfPresent(
+                                  Cursor.createVKey(key.cursor(), registry.getTldStr())));
               DateTime position = getCursorTimeOrStartOfTime(cursor);
               checkState(key.interval() != null, "Interval must be present");
               DateTime newPosition = key.watermark().plus(key.interval());

core/src/main/java/google/registry/rde/RdeUploadAction.java

@@ -64,6 +64,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
+import java.util.Optional;
 import javax.inject.Inject;
 import javax.inject.Named;
 import org.bouncycastle.openpgp.PGPKeyPair;
@@ -133,7 +134,8 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
   @Override
   public void runWithLock(final DateTime watermark) throws Exception {
     logger.atInfo().log("Verifying readiness to upload the RDE deposit.");
-    Cursor cursor = transactIfJpaTm(() -> tm().loadByKey(Cursor.createVKey(RDE_STAGING, tld)));
+    Optional<Cursor> cursor =
+        transactIfJpaTm(() -> tm().loadByKeyIfPresent(Cursor.createVKey(RDE_STAGING, tld)));
     DateTime stagingCursorTime = getCursorTimeOrStartOfTime(cursor);
     if (isBeforeOrAt(stagingCursorTime, watermark)) {
       throw new NoContentException(
@@ -158,8 +160,10 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
               sftpCursorTime,
               timeSinceLastSftp.getStandardMinutes()));
     }
-    int revision = RdeRevision.getNextRevision(tld, watermark, FULL) - 1;
-    verify(revision >= 0, "RdeRevision was not set on generated deposit");
+    int revision =
+        RdeRevision.getCurrentRevision(tld, watermark, FULL)
+            .orElseThrow(
+                () -> new IllegalStateException("RdeRevision was not set on generated deposit"));
     final String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
     final GcsFilename xmlFilename = new GcsFilename(bucket, name + ".xml.ghostryde");
     final GcsFilename xmlLengthFilename = new GcsFilename(bucket, name + ".xml.length");

core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java

@@ -16,7 +16,6 @@ package google.registry.reporting.icann;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
-import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -107,10 +106,10 @@ public final class IcannReportingUploadAction implements Runnable {
 
           // If cursor time is before now, upload the corresponding report
           cursors.entrySet().stream()
-              .filter(entry -> getCursorTimeOrStartOfTime(entry.getKey()).isBefore(clock.nowUtc()))
+              .filter(entry -> entry.getKey().getCursorTime().isBefore(clock.nowUtc()))
               .forEach(
                   entry -> {
-                    DateTime cursorTime = getCursorTimeOrStartOfTime(entry.getKey());
+                    DateTime cursorTime = entry.getKey().getCursorTime();
                     uploadReport(
                         cursorTime,
                         entry.getKey().getType(),
@@ -278,9 +277,15 @@ public final class IcannReportingUploadAction implements Runnable {
   }
 
   private void emailUploadResults(ImmutableMap<String, Boolean> reportSummary) {
-    String subject = String.format(
-        "ICANN Monthly report upload summary: %d/%d succeeded",
-        reportSummary.values().stream().filter((b) -> b).count(), reportSummary.size());
+    if (reportSummary.size() == 0) {
+      logger.atInfo().log("No uploads were attempted today; skipping notification email.");
+      return;
+    }
+    String subject =
+        String.format(
+            "ICANN Monthly report upload summary: %d/%d succeeded",
+            // This filter() does in fact do something: It counts only the trues.
+            reportSummary.values().stream().filter((b) -> b).count(), reportSummary.size());
     String body =
         String.format(
             "Report Filename - Upload status:\n%s",

core/src/main/java/google/registry/reporting/spec11/Spec11EmailUtils.java

@@ -17,7 +17,9 @@ package google.registry.reporting.spec11;
 import static com.google.common.base.Throwables.getRootCause;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.io.Resources.getResource;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.QueryComposer.Comparator;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -129,17 +131,20 @@ public class Spec11EmailUtils {
   private RegistrarThreatMatches filterOutNonPublishedMatches(
       RegistrarThreatMatches registrarThreatMatches) {
     ImmutableList<ThreatMatch> filteredMatches =
-        registrarThreatMatches.threatMatches().stream()
-            .filter(
-                threatMatch ->
-                    ofy()
-                        .load()
-                        .type(DomainBase.class)
-                        .filter("fullyQualifiedDomainName", threatMatch.fullyQualifiedDomainName())
-                        .first()
-                        .now()
-                        .shouldPublishToDns())
-            .collect(toImmutableList());
+        transactIfJpaTm(
+            () -> {
+              return registrarThreatMatches.threatMatches().stream()
+                  .filter(
+                      threatMatch ->
+                          tm().createQueryComposer(DomainBase.class)
+                              .where(
+                                  "fullyQualifiedDomainName",
+                                  Comparator.EQ,
+                                  threatMatch.fullyQualifiedDomainName())
+                              .getSingleResult()
+                              .shouldPublishToDns())
+                  .collect(toImmutableList());
+            });
     return RegistrarThreatMatches.create(registrarThreatMatches.clientId(), filteredMatches);
   }
 

core/src/main/java/google/registry/schema/replay/ReplaySpecializer.java

@@ -26,21 +26,30 @@ import java.lang.reflect.Method;
  * to invoke special class methods if they are present.
  */
 public class ReplaySpecializer {
+
   public static void beforeSqlDelete(VKey<?> key) {
+    invokeMethod(key.getKind(), "beforeSqlDelete", key);
+  }
+
+  public static void beforeSqlSave(SqlEntity sqlEntity) {
+    invokeMethod(sqlEntity.getClass(), "beforeSqlSave", sqlEntity);
+  }
+
+  private static <T> void invokeMethod(Class<T> clazz, String methodName, Object argument) {
     try {
-      Method method = key.getKind().getMethod("beforeSqlDelete", VKey.class);
-      method.invoke(null, new Object[] {key});
+      Method method = clazz.getMethod(methodName, argument.getClass());
+      method.invoke(null, argument);
     } catch (NoSuchMethodException e) {
       // Ignore, this just means that the class doesn't need this hook.
     } catch (IllegalAccessException e) {
       throw new RuntimeException(
-          "beforeSqlDelete() method is defined for class "
-              + key.getKind().getName()
-              + " but is not public.",
+          String.format(
+              "%s() method is defined for class %s but is not public.",
+              methodName, clazz.getName()),
           e);
     } catch (InvocationTargetException e) {
       throw new RuntimeException(
-          "beforeSqlDelete() method for class " + key.getKind().getName() + " threw an exception.",
+          String.format("%s() method for class %s threw an exception", methodName, clazz.getName()),
           e);
     }
   }

core/src/main/java/google/registry/tools/ComparePremiumListsCommand.java

@@ -0,0 +1,105 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
+import google.registry.model.registry.label.PremiumList;
+import google.registry.model.registry.label.PremiumList.PremiumListEntry;
+import google.registry.model.registry.label.PremiumListDatastoreDao;
+import google.registry.schema.tld.PremiumEntry;
+import google.registry.schema.tld.PremiumListSqlDao;
+import java.util.Optional;
+import org.joda.money.BigMoney;
+
+/** Command to compare all PremiumLists in Datastore to all PremiumLists in Cloud SQL. */
+@Parameters(
+    separators = " =",
+    commandDescription = "Compare all the PremiumLists in Datastore to those in Cloud SQL.")
+final class ComparePremiumListsCommand implements CommandWithRemoteApi {
+
+  @Override
+  public void run() {
+    ImmutableSet<String> datastoreLists =
+        ofyTm().loadAllOf(PremiumList.class).stream()
+            .map(PremiumList::getName)
+            .collect(toImmutableSet());
+
+    ImmutableSet<String> sqlLists =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAllOf(PremiumList.class).stream()
+                        .map(PremiumList::getName)
+                        .collect(toImmutableSet()));
+
+    int listsWithDiffs = 0;
+
+    for (String listName : Sets.difference(datastoreLists, sqlLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "PremiumList '%s' is present in Datastore, but not in Cloud SQL.%n", listName);
+    }
+    for (String listName : Sets.difference(sqlLists, datastoreLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "PremiumList '%s' is present in Cloud SQL, but not in Datastore.%n", listName);
+    }
+
+    for (String listName : Sets.intersection(datastoreLists, sqlLists)) {
+      Optional<PremiumList> sqlList = PremiumListSqlDao.getLatestRevision(listName);
+
+      // Datastore and Cloud SQL use different objects to represent premium list entries
+      // so the best way to compare them is to compare their string representations.
+      ImmutableSet<String> datastoreListStrings =
+          Streams.stream(
+                  PremiumListDatastoreDao.loadPremiumListEntriesUncached(
+                      PremiumListDatastoreDao.getLatestRevision(listName).get()))
+              .map(PremiumListEntry::toString)
+              .collect(toImmutableSet());
+
+      Iterable<PremiumEntry> sqlListEntries =
+          jpaTm().transact(() -> PremiumListSqlDao.loadPremiumListEntriesUncached(sqlList.get()));
+
+      ImmutableSet<String> sqlListStrings =
+          Streams.stream(sqlListEntries)
+              .map(
+                  premiumEntry ->
+                      new PremiumListEntry.Builder()
+                          .setPrice(
+                              BigMoney.of(sqlList.get().getCurrency(), premiumEntry.getPrice())
+                                  .toMoney())
+                          .setLabel(premiumEntry.getDomainLabel())
+                          .build()
+                          .toString())
+              .collect(toImmutableSet());
+
+      // This will only print out the name of the unequal list. GetPremiumListCommand
+      // should be used to determine what the actual differences are.
+      if (!datastoreListStrings.equals(sqlListStrings)) {
+        listsWithDiffs++;
+        System.out.printf("PremiumList '%s' has different entries in each database.%n", listName);
+      }
+    }
+
+    System.out.printf("Found %d unequal list(s).%n", listsWithDiffs);
+  }
+}

core/src/main/java/google/registry/tools/CompareReservedListsCommand.java

@@ -0,0 +1,82 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import google.registry.model.registry.label.ReservedList;
+import google.registry.model.registry.label.ReservedList.ReservedListEntry;
+import google.registry.model.registry.label.ReservedListDatastoreDao;
+import google.registry.model.registry.label.ReservedListSqlDao;
+
+/** Command to compare all ReservedLists in Datastore to all ReservedLists in Cloud SQL. */
+@Parameters(
+    separators = " =",
+    commandDescription = "Compare all the ReservedLists in Datastore to those in Cloud SQL.")
+final class CompareReservedListsCommand implements CommandWithRemoteApi {
+
+  @Override
+  public void run() {
+    ImmutableSet<String> datastoreLists =
+        ofy().load().type(ReservedList.class).ancestor(getCrossTldKey()).list().stream()
+            .map(ReservedList::getName)
+            .collect(toImmutableSet());
+
+    ImmutableSet<String> cloudSqlLists =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAllOf(ReservedList.class).stream()
+                        .map(ReservedList::getName)
+                        .collect(toImmutableSet()));
+
+    int listsWithDiffs = 0;
+
+    for (String listName : Sets.difference(datastoreLists, cloudSqlLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "ReservedList '%s' is present in Datastore, but not in Cloud SQL.%n", listName);
+    }
+    for (String listName : Sets.difference(cloudSqlLists, datastoreLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "ReservedList '%s' is present in Cloud SQL, but not in Datastore.%n", listName);
+    }
+
+    for (String listName : Sets.intersection(datastoreLists, cloudSqlLists)) {
+      ImmutableMap<String, ReservedListEntry> namesInSql =
+          ReservedListSqlDao.getLatestRevision(listName).get().getReservedListEntries();
+
+      ImmutableMap<String, ReservedListEntry> namesInDatastore =
+          ReservedListDatastoreDao.getLatestRevision(listName).get().getReservedListEntries();
+
+      // This will only print out the name of the unequal list. GetReservedListCommand should be
+      // used to determine what the actual differences are.
+      if (!namesInDatastore.equals(namesInSql)) {
+        listsWithDiffs++;
+        System.out.printf("ReservedList '%s' has different entries in each database.%n", listName);
+      }
+    }
+
+    System.out.printf("Found %d unequal list(s).%n", listsWithDiffs);
+  }
+}

core/src/main/java/google/registry/tools/GetKeyringSecretCommand.java

@@ -65,12 +65,6 @@ final class GetKeyringSecretCommand implements CommandWithRemoteApi {
       case BRDA_SIGNING_PUBLIC_KEY:
         out.write(KeySerializer.serializePublicKey(keyring.getBrdaSigningKey().getPublicKey()));
         break;
-      case CLOUD_SQL_PASSWORD:
-        out.write(KeySerializer.serializeString(keyring.getCloudSqlPassword()));
-        break;
-      case TOOLS_CLOUD_SQL_PASSWORD:
-        out.write(KeySerializer.serializeString(keyring.getToolsCloudSqlPassword()));
-        break;
       case ICANN_REPORTING_PASSWORD:
         out.write(KeySerializer.serializeString(keyring.getIcannReportingPassword()));
         break;

core/src/main/java/google/registry/tools/MigrateKmsKeyringCommand.java

@@ -0,0 +1,78 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.beust.jcommander.Parameters;
+import google.registry.keyring.api.Keyring;
+import google.registry.keyring.kms.KmsKeyring;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
+import java.util.Map;
+import javax.inject.Inject;
+
+/** Migrates secrets from the KMS keyring to the Secret Manager. */
+@Parameters(
+    separators = " =",
+    commandDescription = "Migrate values of secrets in KmsKeyring to Secret Manager.")
+public class MigrateKmsKeyringCommand extends ConfirmingCommand implements CommandWithRemoteApi {
+
+  @Inject Keyring keyring;
+
+  @Inject KeyringSecretStore secretStore;
+
+  Map<String, Runnable> migrationTasks;
+
+  @Inject
+  MigrateKmsKeyringCommand() {}
+
+  @Override
+  protected void init() {
+
+    checkState(
+        keyring instanceof KmsKeyring,
+        "Expecting KmsKeyring, found %s",
+        keyring.getClass().getSimpleName());
+
+    migrationTasks = ((KmsKeyring) keyring).migrationPlan();
+  }
+
+  @Override
+  protected boolean dontRunCommand() {
+    return migrationTasks.isEmpty();
+  }
+
+  @Override
+  protected String prompt() {
+    if (migrationTasks.isEmpty()) {
+      return "All keys are up to date.";
+    }
+    return String.format("Migrate %s keys?", migrationTasks.size());
+  }
+
+  @Override
+  protected String execute() {
+    int errors = 0;
+    for (Map.Entry<String, Runnable> entry : migrationTasks.entrySet()) {
+      try {
+        entry.getValue().run();
+      } catch (Exception e) {
+        System.err.printf("Failed to migrate %s: %s", entry.getKey(), e.getMessage());
+        errors++;
+      }
+    }
+    return errors == 0 ? "Success!" : "Failed to migrate " + errors + "keys.";
+  }
+}

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -17,6 +17,7 @@ package google.registry.tools;
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
 import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
+import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 
@@ -37,6 +38,8 @@ public final class RegistryTool {
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
+          .put("compare_premium_lists", ComparePremiumListsCommand.class)
+          .put("compare_reserved_lists", CompareReservedListsCommand.class)
           .put("convert_idn", ConvertIdnCommand.class)
           .put("count_domains", CountDomainsCommand.class)
           .put("create_anchor_tenant", CreateAnchorTenantCommand.class)
@@ -53,6 +56,7 @@ public final class RegistryTool {
           .put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class)
           .put("dedupe_recurring_billing_event_ids", DedupeRecurringBillingEventIdsCommand.class)
           .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class)
+          .put("delete_contact_by_roid", DeleteContactByRoidCommand.class)
           .put("delete_domain", DeleteDomainCommand.class)
           .put("delete_host", DeleteHostCommand.class)
           .put("delete_premium_list", DeletePremiumListCommand.class)
@@ -101,6 +105,7 @@ public final class RegistryTool {
           .put("lock_domain", LockDomainCommand.class)
           .put("login", LoginCommand.class)
           .put("logout", LogoutCommand.class)
+          .put("migrate_kms_keyring", MigrateKmsKeyringCommand.class)
           .put("pending_escrow", PendingEscrowCommand.class)
           .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -42,6 +42,7 @@ import google.registry.request.Modules.URLFetchServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
+import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.util.UtilsModule;
 import google.registry.whois.NonCachingWhoisModule;
 import javax.annotation.Nullable;
@@ -104,6 +105,8 @@ interface RegistryToolComponent {
 
   void inject(CreateTldCommand command);
 
+  void inject(DeleteContactByRoidCommand command);
+
   void inject(DeployInvoicingPipelineCommand command);
 
   void inject(DeploySpec11PipelineCommand command);
@@ -138,6 +141,8 @@ interface RegistryToolComponent {
 
   void inject(LogoutCommand command);
 
+  void inject(MigrateKmsKeyringCommand command);
+
   void inject(PendingEscrowCommand command);
 
   void inject(RenewDomainCommand command);

core/src/main/java/google/registry/tools/SendEscrowReportToIcannCommand.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
+import dagger.Lazy;
 import google.registry.rde.RdeReporter;
 import google.registry.tools.params.PathParameter;
 import java.nio.file.Files;
@@ -33,13 +34,12 @@ final class SendEscrowReportToIcannCommand implements CommandWithRemoteApi {
       required = true)
   private List<Path> files;
 
-  @Inject
-  RdeReporter rdeReporter;
+  @Inject Lazy<RdeReporter> rdeReporter;
 
   @Override
   public void run() throws Exception {
     for (Path file : files) {
-      rdeReporter.send(Files.readAllBytes(file));
+      rdeReporter.get().send(Files.readAllBytes(file));
       System.out.printf("Uploaded: %s\n", file);
     }
   }

core/src/main/java/google/registry/tools/SetDatabaseTransitionScheduleCommand.java

@@ -14,6 +14,8 @@
 
 package google.registry.tools;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.collect.ImmutableSortedMap;
@@ -23,14 +25,13 @@ import google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabaseTr
 import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
 import google.registry.model.common.TimedTransitionProperty;
 import google.registry.tools.params.TransitionListParameter.PrimaryDatabaseTransitions;
-import java.util.Optional;
 import org.joda.time.DateTime;
 
 /** Command to update {@link DatabaseTransitionSchedule}. */
 @Parameters(
     separators = " =",
     commandDescription = "Set the database transition schedule for transition id.")
-public class SetDatabaseTransitionScheduleCommand extends MutatingCommand {
+public class SetDatabaseTransitionScheduleCommand extends ConfirmingCommand {
 
   @Parameter(
       names = "--transition_schedule",
@@ -43,20 +44,25 @@ public class SetDatabaseTransitionScheduleCommand extends MutatingCommand {
 
   @Parameter(
       names = "--transition_id",
+      required = true,
       description = "Transition id string for the schedule being updated")
   private TransitionId transitionId;
 
   @Override
-  protected void init() {
-    Optional<DatabaseTransitionSchedule> currentSchedule =
-        DatabaseTransitionSchedule.get(transitionId);
+  protected String prompt() {
+    return String.format(
+        "Insert new schedule %s for transition ID %s?", transitionSchedule, transitionId);
+  }
 
+  @Override
+  protected String execute() {
     DatabaseTransitionSchedule newSchedule =
         DatabaseTransitionSchedule.create(
             transitionId,
             TimedTransitionProperty.fromValueMap(
                 transitionSchedule, PrimaryDatabaseTransition.class));
-
-    stageEntityChange(currentSchedule.orElse(null), newSchedule);
+    ofyTm().transact(() -> ofyTm().put(newSchedule));
+    return String.format(
+        "Inserted new schedule %s for transition ID %s.", transitionSchedule, transitionId);
   }
 }

core/src/main/java/google/registry/tools/UnrenewDomainCommand.java

@@ -20,8 +20,8 @@ import static google.registry.flows.domain.DomainFlowUtils.newAutorenewBillingEv
 import static google.registry.flows.domain.DomainFlowUtils.newAutorenewPollMessage;
 import static google.registry.flows.domain.DomainFlowUtils.updateAutorenewRecurrenceEndTime;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.leapSafeSubtractYears;
 
@@ -33,12 +33,12 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.Period.Unit;
 import google.registry.model.eppcommon.StatusValue;
-import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
+import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.poll.PollMessage;
-import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.HistoryEntry.Type;
 import google.registry.util.Clock;
 import google.registry.util.NonFinalForTesting;
@@ -87,7 +87,7 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
         new ImmutableMap.Builder<>();
 
     for (String domainName : mainParameters) {
-      if (ofy().load().type(ForeignKeyDomainIndex.class).id(domainName).now() == null) {
+      if (ForeignKeyIndex.load(DomainBase.class, domainName, START_OF_TIME) == null) {
         domainsNonexistentBuilder.add(domainName);
         continue;
       }
@@ -183,8 +183,8 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
 
     DateTime newExpirationTime =
         leapSafeSubtractYears(domain.getRegistrationExpirationTime(), period);
-    HistoryEntry historyEntry =
-        new HistoryEntry.Builder()
+    DomainHistory domainHistory =
+        new DomainHistory.Builder()
             .setParent(domain)
             .setModificationTime(now)
             .setBySuperuser(true)
@@ -201,19 +201,19 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
                 String.format(
                     "Domain %s was unrenewed by %d years; now expires at %s.",
                     domainName, period, newExpirationTime))
-            .setParent(historyEntry)
+            .setParent(domainHistory)
             .setEventTime(now)
             .build();
     // Create a new autorenew billing event and poll message starting at the new expiration time.
     BillingEvent.Recurring newAutorenewEvent =
         newAutorenewBillingEvent(domain)
             .setEventTime(newExpirationTime)
-            .setParent(historyEntry)
+            .setParent(domainHistory)
             .build();
     PollMessage.Autorenew newAutorenewPollMessage =
         newAutorenewPollMessage(domain)
             .setEventTime(newExpirationTime)
-            .setParent(historyEntry)
+            .setParent(domainHistory)
             .build();
     // End the old autorenew billing event and poll message now.
     updateAutorenewRecurrenceEndTime(domain, now);
@@ -229,11 +229,9 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
     // In order to do it'll need to write out a new HistoryEntry (likely of type SYNTHETIC), a new
     // autorenew billing event and poll message, and a new one time poll message at the present time
     // informing the registrar of this out-of-band change.
-    ofy()
-        .save()
-        .entities(
+    tm().putAll(
             newDomain,
-            historyEntry,
+            domainHistory,
             oneTimePollMessage,
             newAutorenewEvent,
             newAutorenewPollMessage);

core/src/main/java/google/registry/tools/UpdateKmsKeyringCommand.java

@@ -65,12 +65,6 @@ final class UpdateKmsKeyringCommand implements CommandWithRemoteApi {
         throw new IllegalArgumentException(
             "Can't update BRDA_SIGNING_PUBLIC_KEY directly."
             + " Must update public and private keys together using BRDA_SIGNING_KEY_PAIR.");
-      case CLOUD_SQL_PASSWORD:
-        kmsUpdater.setCloudSqlPassword(deserializeString(input));
-        break;
-      case TOOLS_CLOUD_SQL_PASSWORD:
-        kmsUpdater.setToolsCloudSqlPassword(deserializeString(input));
-        break;
       case ICANN_REPORTING_PASSWORD:
         kmsUpdater.setIcannReportingPassword(deserializeString(input));
         break;

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -19,6 +19,7 @@ import static com.google.common.collect.ImmutableListMultimap.flatteningToImmuta
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -135,12 +136,7 @@ public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
             flatteningToImmutableListMultimap(
                 Function.identity(),
                 (domainName) -> {
-                  List<DomainBase> domains =
-                      ofy()
-                          .load()
-                          .type(DomainBase.class)
-                          .filter("fullyQualifiedDomainName", domainName)
-                          .list();
+                  List<DomainBase> domains = loadDomainsForFqdn(domainName);
                   domains.sort(Comparator.comparing(DomainBase::getCreationTime).reversed());
                   checkState(
                       !domains.isEmpty(),
@@ -150,6 +146,25 @@ public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
                 }));
   }
 
+  /** Loads in all {@link DomainBase} objects for a given FQDN. */
+  private List<DomainBase> loadDomainsForFqdn(String fullyQualifiedDomainName) {
+    if (tm().isOfy()) {
+      return ofy()
+          .load()
+          .type(DomainBase.class)
+          .filter("fullyQualifiedDomainName", fullyQualifiedDomainName)
+          .list();
+    } else {
+      return jpaTm()
+          .transact(
+              () ->
+                  jpaTm()
+                      .query("FROM Domain WHERE fullyQualifiedDomainName = :fqdn", DomainBase.class)
+                      .setParameter("fqdn", fullyQualifiedDomainName)
+                      .getResultList());
+    }
+  }
+
   /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
   private Spec11ThreatMatch threatMatchToCloudSqlObject(
       ThreatMatch threatMatch,

core/src/main/java/google/registry/tools/javascrap/DeleteContactByRoidCommand.java

@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.eppcommon.StatusValue;
+import google.registry.model.index.EppResourceIndex;
+import google.registry.model.index.ForeignKeyIndex;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.SystemClock;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Deletes a {@link google.registry.model.contact.ContactResource} by its ROID.
+ *
+ * <p>This is a short-term tool for race condition clean up while the bug is being fixed.
+ */
+@Parameters(separators = " =", commandDescription = "Delete a contact by its ROID.")
+public class DeleteContactByRoidCommand extends ConfirmingCommand implements CommandWithRemoteApi {
+
+  @Parameter(names = "--roid", description = "The roid of the contact to be deleted.")
+  String roid;
+
+  @Parameter(
+      names = "--contact_id",
+      description = "The user provided contactId, for verification purpose.")
+  String contactId;
+
+  ImmutableList<Key<?>> toDelete;
+
+  @Override
+  protected void init() throws Exception {
+    System.out.printf("Deleting %s, which refers to %s.\n", roid, contactId);
+    tm().transact(
+            () -> {
+              Key<ContactResource> targetKey = Key.create(ContactResource.class, roid);
+              ContactResource targetContact = ofy().load().key(targetKey).now();
+              verify(
+                  Objects.equals(targetContact.getContactId(), contactId),
+                  "contactId does not match.");
+              verify(
+                  Objects.equals(targetContact.getStatusValues(), ImmutableSet.of(StatusValue.OK)));
+              System.out.println("Target contact has the expected contactId");
+              String canonicalResource =
+                  ForeignKeyIndex.load(ContactResource.class, contactId, new SystemClock().nowUtc())
+                      .getResourceKey()
+                      .getOfyKey()
+                      .getName();
+              verify(!Objects.equals(canonicalResource, roid), "Contact still in ForeignKeyIndex.");
+              System.out.printf(
+                  "It is safe to delete %s, since the contactId is mapped to a different entry in"
+                      + " the Foreign key index (%s).\n\n",
+                  roid, canonicalResource);
+
+              List<Object> ancestors =
+                  ofy().load().ancestor(Key.create(ContactResource.class, roid)).list();
+
+              System.out.println("Ancestor query returns: ");
+              for (Object entity : ancestors) {
+                System.out.println(Key.create(entity));
+              }
+
+              ImmutableSet<String> deletetableKinds =
+                  ImmutableSet.of("HistoryEntry", "ContactResource");
+              toDelete =
+                  ancestors.stream()
+                      .map(Key::create)
+                      .filter(key -> deletetableKinds.contains(key.getKind()))
+                      .collect(ImmutableList.toImmutableList());
+
+              EppResourceIndex eppResourceIndex =
+                  ofy().load().entity(EppResourceIndex.create(targetKey)).now();
+              verify(eppResourceIndex.getKey().equals(targetKey), "Wrong EppResource Index loaded");
+              System.out.printf("\n\nEppResourceIndex found (%s).\n", Key.create(eppResourceIndex));
+
+              toDelete =
+                  new ImmutableList.Builder<Key<?>>()
+                      .addAll(toDelete)
+                      .add(Key.create(eppResourceIndex))
+                      .build();
+
+              System.out.printf("\n\nAbout to delete %s entities:\n", toDelete.size());
+              toDelete.forEach(key -> System.out.println(key));
+            });
+  }
+
+  @Override
+  protected String execute() {
+    tm().transact(() -> ofy().delete().keys(toDelete).now());
+    return "Done";
+  }
+}

core/src/main/java/google/registry/tools/javascrap/PopulateNullRegistrarFieldsCommand.java

@@ -15,8 +15,6 @@
 package google.registry.tools.javascrap;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
-import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 
 import com.beust.jcommander.Parameters;
 import com.google.common.collect.ImmutableList;
@@ -38,7 +36,7 @@ public class PopulateNullRegistrarFieldsCommand extends MutatingCommand {
 
   @Override
   protected void init() {
-    for (Registrar registrar : ofy().load().type(Registrar.class).ancestor(getCrossTldKey())) {
+    for (Registrar registrar : Registrar.loadAll()) {
       Registrar.Builder changeBuilder = registrar.asBuilder();
       changeBuilder.setRegistrarName(
           firstNonNull(registrar.getRegistrarName(), registrar.getClientId()));

core/src/main/java/google/registry/tools/javascrap/RemoveIpAddressCommand.java

@@ -14,13 +14,15 @@
 
 package google.registry.tools.javascrap;
 
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.template.soy.data.SoyMapData;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.tools.MutatingEppToolCommand;
 import google.registry.tools.params.PathParameter;
 import google.registry.tools.soy.RemoveIpAddressSoyInfo;
@@ -30,6 +32,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Command to remove external IP Addresses from HostResources identified by text file listing
@@ -56,14 +59,15 @@ public class RemoveIpAddressCommand extends MutatingEppToolCommand {
 
     for (String roid : roids) {
       // Look up the HostResource from its roid.
-      HostResource host = ofy().load().type(HostResource.class).id(roid).now();
-      if (host == null) {
+      Optional<HostResource> host =
+          transactIfJpaTm(() -> tm().loadByKeyIfPresent(VKey.create(HostResource.class, roid)));
+      if (!host.isPresent()) {
         System.err.printf("Record for %s not found.\n", roid);
         continue;
       }
 
       ArrayList<SoyMapData> ipAddresses = new ArrayList<>();
-      for (InetAddress address : host.getInetAddresses()) {
+      for (InetAddress address : host.get().getInetAddresses()) {
         SoyMapData dataMap = new SoyMapData(
             "address", address.getHostAddress(),
             "version", address instanceof Inet6Address ? "v6" : "v4");
@@ -76,7 +80,7 @@ public class RemoveIpAddressCommand extends MutatingEppToolCommand {
       addSoyRecord(
           registrarId,
           new SoyMapData(
-              "name", host.getHostName(),
+              "name", host.get().getHostName(),
               "ipAddresses", ipAddresses,
               "requestedByRegistrar", registrarId));
     }

core/src/main/java/google/registry/tools/params/KeyringKeyName.java

@@ -24,7 +24,6 @@ public enum KeyringKeyName {
   BRDA_RECEIVER_PUBLIC_KEY,
   BRDA_SIGNING_KEY_PAIR,
   BRDA_SIGNING_PUBLIC_KEY,
-  CLOUD_SQL_PASSWORD,
   ICANN_REPORTING_PASSWORD,
   JSON_CREDENTIAL,
   MARKSDB_DNL_LOGIN_AND_PASSWORD,
@@ -37,6 +36,5 @@ public enum KeyringKeyName {
   RDE_SSH_CLIENT_PUBLIC_KEY,
   RDE_STAGING_KEY_PAIR,
   RDE_STAGING_PUBLIC_KEY,
-  SAFE_BROWSING_API_KEY,
-  TOOLS_CLOUD_SQL_PASSWORD,
+  SAFE_BROWSING_API_KEY
 }

core/src/main/java/google/registry/whois/NameserverLookupByIpCommand.java

@@ -16,12 +16,15 @@ package google.registry.whois;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.model.EppResourceUtils.queryNotDeleted;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
+import com.google.common.net.InetAddresses;
 import com.google.common.net.InternetDomainName;
 import google.registry.model.host.HostResource;
 import google.registry.model.registry.Registries;
@@ -46,9 +49,34 @@ final class NameserverLookupByIpCommand implements WhoisCommand {
   }
 
   @Override
+  @SuppressWarnings("unchecked")
   public WhoisResponse executeQuery(DateTime now) throws WhoisException {
+    Iterable<HostResource> hostsFromDb;
+    if (tm().isOfy()) {
+      hostsFromDb =
+          ofy()
+              .load()
+              .type(HostResource.class)
+              .filter("inetAddresses", ipAddress)
+              .filter("deletionTime >", now.toDate());
+    } else {
+      hostsFromDb =
+          jpaTm()
+              .transact(
+                  () ->
+                      // We cannot query @Convert-ed fields in HQL so we must use native Postgres
+                      jpaTm()
+                          .getEntityManager()
+                          .createNativeQuery(
+                              "SELECT * From \"Host\" WHERE :address = ANY(inet_addresses) AND "
+                                  + "deletion_time > CAST(:now AS timestamptz)",
+                              HostResource.class)
+                          .setParameter("address", InetAddresses.toAddrString(ipAddress))
+                          .setParameter("now", now.toString())
+                          .getResultList());
+    }
     ImmutableList<HostResource> hosts =
-        Streams.stream(queryNotDeleted(HostResource.class, now, "inetAddresses", ipAddress))
+        Streams.stream(hostsFromDb)
             .filter(
                 host ->
                     Registries.findTldForName(InternetDomainName.from(host.getHostName()))

core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java

@@ -125,6 +125,7 @@ public class ReplayCommitLogsToSqlActionTest {
     action.diffLister.gcsBucket = GCS_BUCKET;
     action.diffLister.executor = newDirectExecutorService();
     RegistryConfig.overrideCloudSqlReplayCommitLogs(true);
+    TestObject.beforeSqlSaveCallCount = 0;
     TestObject.beforeSqlDeleteCallCount = 0;
   }
 
@@ -442,6 +443,21 @@ public class ReplayCommitLogsToSqlActionTest {
         .isEqualTo("Can't acquire SQL commit log replay lock, aborting.");
   }
 
+  @Test
+  void testSuccess_beforeSqlSaveCallback() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    Key<CommitLogBucket> bucketKey = getBucketKey(1);
+    Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(bucketKey, now, null),
+        CommitLogMutation.create(manifestKey, TestObject.create("a")));
+    runAndAssertSuccess(now.minusMinutes(1));
+    assertThat(TestObject.beforeSqlSaveCallCount).isEqualTo(1);
+  }
+
   @Test
   void testSuccess_deleteSqlCallback() throws Exception {
     DateTime now = fakeClock.nowUtc();

core/src/test/java/google/registry/batch/DeleteExpiredDomainsActionTest.java

@@ -16,6 +16,7 @@ package google.registry.batch;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
+import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.HistoryEntry.Type.DOMAIN_CREATE;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
@@ -121,14 +122,30 @@ class DeleteExpiredDomainsActionTest {
   }
 
   @Test
-  void test_deletesThreeDomainsInOneRun() {
+  void test_deletesThreeDomainsInOneRun() throws Exception {
     DomainBase domain1 = persistNonAutorenewingDomain("ecck1.tld");
     DomainBase domain2 = persistNonAutorenewingDomain("veee2.tld");
     DomainBase domain3 = persistNonAutorenewingDomain("tarm3.tld");
 
-    // action.run() executes a non-transactional query by design but makes this test flaky.
-    // Executing a transaction here seems to force the test Datastore to become up to date.
-    assertThat(tm().loadByEntity(domain3).getStatusValues()).doesNotContain(PENDING_DELETE);
+    // action.run() executes an ancestor-less query which is subject to eventual consistency (it
+    // uses an index that is updated asynchronously). For a deterministic test outcome, we busy
+    // wait here until all domains above are returned by the query.
+    for (int attempts = 0; attempts < 3; attempts++) {
+      ImmutableSet<String> matchingDomains =
+          ofy()
+              .load()
+              .type(DomainBase.class)
+              .filter("autorenewEndTime <=", clock.nowUtc())
+              .list()
+              .stream()
+              .map(DomainBase::getDomainName)
+              .collect(ImmutableSet.toImmutableSet());
+      if (matchingDomains.containsAll(ImmutableSet.of("ecck1.tld", "veee2.tld", "tarm3.tld"))) {
+        break;
+      }
+      Thread.sleep(100);
+    }
+
     action.run();
 
     assertThat(tm().loadByEntity(domain1).getStatusValues()).contains(PENDING_DELETE);

core/src/test/java/google/registry/batch/RefreshDnsOnHostRenameActionTest.java

@@ -19,7 +19,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_HOST_RENAME;
 import static google.registry.batch.AsyncTaskMetrics.OperationType.DNS_REFRESH;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.newHostResource;
@@ -47,11 +47,14 @@ import google.registry.batch.AsyncTaskMetrics.OperationResult;
 import google.registry.batch.RefreshDnsOnHostRenameAction.RefreshDnsOnHostRenameReducer;
 import google.registry.model.host.HostResource;
 import google.registry.model.server.Lock;
+import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import google.registry.testing.mapreduce.MapreduceTestCase;
 import google.registry.util.AppEngineServiceUtils;
 import google.registry.util.RequestStatusChecker;
@@ -62,11 +65,11 @@ import java.util.Optional;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockito.Mock;
 
 /** Unit tests for {@link RefreshDnsOnHostRenameAction}. */
+@DualDatabaseTest
 public class RefreshDnsOnHostRenameActionTest
     extends MapreduceTestCase<RefreshDnsOnHostRenameAction> {
 
@@ -109,7 +112,7 @@ public class RefreshDnsOnHostRenameActionTest
     executeTasksUntilEmpty("mapreduce", clock);
     sleeper.sleep(millis(50));
     clock.advanceBy(standardSeconds(5));
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
   }
 
   /** Kicks off, but does not run, the mapreduce tasks. Useful for testing validation/setup. */
@@ -117,10 +120,13 @@ public class RefreshDnsOnHostRenameActionTest
     clock.advanceOneMilli();
     action.run();
     clock.advanceBy(standardSeconds(5));
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
   }
 
-  @Test
+  // TODO(b/181662306) None of the map reduce tests work with SQL since our map-reduce setup is
+  // inherently Datastore oriented, but this is a bigger task.
+
+  @TestOfyOnly
   void testSuccess_dnsUpdateEnqueued() throws Exception {
     HostResource host = persistActiveHost("ns1.example.tld");
     persistResource(newDomainBase("example.tld", host));
@@ -137,7 +143,7 @@ public class RefreshDnsOnHostRenameActionTest
     verifyNoMoreInteractions(action.asyncTaskMetrics);
   }
 
-  @Test
+  @TestOfyOnly
   void testSuccess_multipleHostsProcessedInBatch() throws Exception {
     HostResource host1 = persistActiveHost("ns1.example.tld");
     HostResource host2 = persistActiveHost("ns2.example.tld");
@@ -161,7 +167,7 @@ public class RefreshDnsOnHostRenameActionTest
     verifyNoMoreInteractions(action.asyncTaskMetrics);
   }
 
-  @Test
+  @TestOfyOnly
   void testSuccess_deletedHost_doesntTriggerDnsRefresh() throws Exception {
     HostResource host = persistDeletedHost("ns11.fakesss.tld", clock.nowUtc().minusDays(4));
     persistResource(newDomainBase("example1.tld", host));
@@ -176,7 +182,7 @@ public class RefreshDnsOnHostRenameActionTest
     verifyNoMoreInteractions(action.asyncTaskMetrics);
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_noDnsTasksForDeletedDomain() throws Exception {
     HostResource renamedHost = persistActiveHost("ns1.example.tld");
     persistResource(
@@ -190,7 +196,7 @@ public class RefreshDnsOnHostRenameActionTest
     assertNoTasksEnqueued(QUEUE_ASYNC_HOST_RENAME);
   }
 
-  @Test
+  @TestOfyAndSql
   void testRun_hostDoesntExist_delaysTask() {
     HostResource host = newHostResource("ns1.example.tld");
     enqueuer.enqueueAsyncDnsRefresh(host, clock.nowUtc());
@@ -204,7 +210,7 @@ public class RefreshDnsOnHostRenameActionTest
     assertThat(acquireLock()).isPresent();
   }
 
-  @Test
+  @TestOfyAndSql
   void test_cannotAcquireLock() {
     // Make lock acquisition fail.
     acquireLock();
@@ -213,7 +219,7 @@ public class RefreshDnsOnHostRenameActionTest
     assertNoDnsTasksEnqueued();
   }
 
-  @Test
+  @TestOfyAndSql
   void test_mapreduceHasWorkToDo_lockIsAcquired() {
     HostResource host = persistActiveHost("ns1.example.tld");
     enqueuer.enqueueAsyncDnsRefresh(host, clock.nowUtc());
@@ -221,7 +227,7 @@ public class RefreshDnsOnHostRenameActionTest
     assertThat(acquireLock()).isEmpty();
   }
 
-  @Test
+  @TestOfyAndSql
   void test_noTasksToLease_releasesLockImmediately() {
     enqueueMapreduceOnly();
     assertNoDnsTasksEnqueued();

core/src/test/java/google/registry/batch/WipeOutDatastoreActionTest.java

@@ -0,0 +1,98 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.batch;
+
+import static com.google.common.truth.Truth.assertThat;
+import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
+import static org.apache.http.HttpStatus.SC_OK;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import com.google.api.services.dataflow.Dataflow;
+import com.google.api.services.dataflow.model.Job;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateRequest;
+import com.google.api.services.dataflow.model.LaunchFlexTemplateResponse;
+import google.registry.testing.FakeResponse;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/** Unit tests for {@link WipeoutDatastoreAction}. */
+@ExtendWith(MockitoExtension.class)
+class WipeOutDatastoreActionTest {
+
+  @Mock private Dataflow dataflow;
+  @Mock private Dataflow.Projects projects;
+  @Mock private Dataflow.Projects.Locations locations;
+  @Mock private Dataflow.Projects.Locations.FlexTemplates flexTemplates;
+  @Mock private Dataflow.Projects.Locations.FlexTemplates.Launch launch;
+  private LaunchFlexTemplateResponse launchResponse =
+      new LaunchFlexTemplateResponse().setJob(new Job());
+
+  private FakeResponse response = new FakeResponse();
+
+  @BeforeEach
+  void beforeEach() throws Exception {
+    lenient().when(dataflow.projects()).thenReturn(projects);
+    lenient().when(projects.locations()).thenReturn(locations);
+    lenient().when(locations.flexTemplates()).thenReturn(flexTemplates);
+    lenient()
+        .when(flexTemplates.launch(anyString(), anyString(), any(LaunchFlexTemplateRequest.class)))
+        .thenReturn(launch);
+    lenient().when(launch.execute()).thenReturn(launchResponse);
+  }
+
+  @Test
+  void run_projectNotAllowed() {
+    WipeoutDatastoreAction action =
+        new WipeoutDatastoreAction(
+            "domain-registry", "us-central1", "gs://some-bucket", response, dataflow);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_FORBIDDEN);
+    verifyNoInteractions(dataflow);
+  }
+
+  @Test
+  void run_projectAllowed() throws Exception {
+    WipeoutDatastoreAction action =
+        new WipeoutDatastoreAction(
+            "domain-registry-qa", "us-central1", "gs://some-bucket", response, dataflow);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    verify(launch, times(1)).execute();
+    verifyNoMoreInteractions(launch);
+  }
+
+  @Test
+  void run_failure() throws Exception {
+    when(launch.execute()).thenThrow(new RuntimeException());
+    WipeoutDatastoreAction action =
+        new WipeoutDatastoreAction(
+            "domain-registry-qa", "us-central1", "gs://some-bucket", response, dataflow);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_INTERNAL_SERVER_ERROR);
+    verify(launch, times(1)).execute();
+    verifyNoMoreInteractions(launch);
+  }
+}

core/src/test/java/google/registry/flows/FlowTestCase.java

@@ -42,7 +42,6 @@ import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppoutput.EppOutput;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.EppLoader;
@@ -112,8 +111,7 @@ public abstract class FlowTestCase<F extends Flow> {
     sessionMetadata = new HttpSessionMetadata(new FakeHttpSession());
     sessionMetadata.setClientId("TheRegistrar");
     sessionMetadata.setServiceExtensionUris(ProtocolDefinition.getVisibleServiceExtensionUris());
-    ofy().saveWithoutBackup().entity(new ClaimsListSingleton()).now();
- }
+  }
 
   protected void removeServiceExtensionUri(String uri) {
     sessionMetadata.setServiceExtensionUris(

core/src/test/java/google/registry/keyring/kms/KmsKeyringTest.java

@@ -20,6 +20,8 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import google.registry.keyring.api.KeySerializer;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
+import google.registry.privileges.secretmanager.FakeSecretManagerClient;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.DualDatabaseTest;
@@ -45,21 +47,9 @@ class KmsKeyringTest {
 
   @BeforeEach
   void beforeEach() {
-    keyring = new KmsKeyring(new FakeKmsConnection());
-  }
-
-  @TestOfyAndSql
-  void test_getCloudSqlPassword() {
-    saveCleartextSecret("cloud-sql-password-string");
-    String cloudSqlPassword = keyring.getCloudSqlPassword();
-    assertThat(cloudSqlPassword).isEqualTo("cloud-sql-password-stringmoo");
-  }
-
-  @TestOfyAndSql
-  void test_getToolsCloudSqlPassword() {
-    saveCleartextSecret("tools-cloud-sql-password-string");
-    String toolsCloudSqlPassword = keyring.getToolsCloudSqlPassword();
-    assertThat(toolsCloudSqlPassword).isEqualTo("tools-cloud-sql-password-stringmoo");
+    keyring =
+        new KmsKeyring(
+            new FakeKmsConnection(), new KeyringSecretStore(new FakeSecretManagerClient()));
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/keyring/kms/KmsUpdaterTest.java

@@ -24,6 +24,8 @@ import google.registry.keyring.api.KeySerializer;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
 import google.registry.model.server.KmsSecretRevisionSqlDao;
+import google.registry.privileges.secretmanager.FakeSecretManagerClient;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.DualDatabaseTest;
@@ -49,7 +51,9 @@ public class KmsUpdaterTest {
 
   @BeforeEach
   void beforeEach() {
-    updater = new KmsUpdater(new FakeKmsConnection());
+    updater =
+        new KmsUpdater(
+            new FakeKmsConnection(), new KeyringSecretStore(new FakeSecretManagerClient()));
   }
 
   @TestOfyAndSql
@@ -96,24 +100,6 @@ public class KmsUpdaterTest {
         getCiphertext(KmsTestHelper.getPublicKey()));
   }
 
-  @TestOfyAndSql
-  void test_setCloudSqlPassword() {
-    updater.setCloudSqlPassword("value1").update();
-
-    verifySecretAndSecretRevisionWritten(
-        "cloud-sql-password-string", "cloud-sql-password-string/foo", getCiphertext("value1"));
-  }
-
-  @TestOfyAndSql
-  void test_setToolsCloudSqlPassword() {
-    updater.setToolsCloudSqlPassword("value1").update();
-
-    verifySecretAndSecretRevisionWritten(
-        "tools-cloud-sql-password-string",
-        "tools-cloud-sql-password-string/foo",
-        getCiphertext("value1"));
-  }
-
   @TestOfyAndSql
   void test_setIcannReportingPassword() {
     updater.setIcannReportingPassword("value1").update();

core/src/test/java/google/registry/model/OteAccountBuilderTest.java

@@ -34,6 +34,8 @@ import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import google.registry.util.CidrAddressBlock;
 import google.registry.util.SystemClock;
 import org.joda.money.Money;
@@ -41,16 +43,16 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
+@DualDatabaseTest
 public final class OteAccountBuilderTest {
 
   @RegisterExtension
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder().withDatastoreAndCloudSql().build();
 
-  @Test
+  @TestOfyAndSql
   void testGetRegistrarToTldMap() {
     assertThat(OteAccountBuilder.forClientId("myclientid").getClientIdToTldMap())
         .containsExactly(
@@ -102,7 +104,7 @@ public final class OteAccountBuilderTest {
     assertThat(contact.getGaeUserId()).isNotEmpty();
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_success() {
     OteAccountBuilder.forClientId("myclientid").addContact("email@example.com").buildAndPersist();
 
@@ -119,7 +121,7 @@ public final class OteAccountBuilderTest {
     assertContactExists("myclientid-5", "email@example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_multipleContacts_success() {
     OteAccountBuilder.forClientId("myclientid")
         .addContact("email@example.com")
@@ -148,7 +150,7 @@ public final class OteAccountBuilderTest {
     assertContactExists("myclientid-5", "someone@example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_setPassword() {
     OteAccountBuilder.forClientId("myclientid").setPassword("myPassword").buildAndPersist();
 
@@ -156,7 +158,7 @@ public final class OteAccountBuilderTest {
         .isTrue();
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_setCertificate() {
     OteAccountBuilder.forClientId("myclientid")
         .setCertificate(SAMPLE_CERT, new SystemClock().nowUtc())
@@ -168,7 +170,7 @@ public final class OteAccountBuilderTest {
         .hasValue(SAMPLE_CERT);
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_setIpAllowList() {
     OteAccountBuilder.forClientId("myclientid")
         .setIpAllowList(ImmutableList.of("1.1.1.0/24"))
@@ -178,7 +180,7 @@ public final class OteAccountBuilderTest {
         .containsExactly(CidrAddressBlock.create("1.1.1.0/24"));
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_invalidClientId_fails() {
     assertThat(
             assertThrows(
@@ -187,7 +189,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: 3blo-bio");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_clientIdTooShort_fails() {
     assertThat(
             assertThrows(IllegalArgumentException.class, () -> OteAccountBuilder.forClientId("bl")))
@@ -195,7 +197,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: bl");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_clientIdTooLong_fails() {
     assertThat(
             assertThrows(
@@ -205,7 +207,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: blobiotoooolong");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_clientIdBadCharacter_fails() {
     assertThat(
             assertThrows(
@@ -214,7 +216,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: blo#bio");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_registrarExists_failsWhenNotReplaceExisting() {
     persistSimpleResource(makeRegistrar1().asBuilder().setClientId("myclientid-1").build());
 
@@ -225,7 +227,7 @@ public final class OteAccountBuilderTest {
         .contains("Found existing object(s) conflicting with OT&E objects");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_tldExists_failsWhenNotReplaceExisting() {
     createTld("myclientid-ga", START_DATE_SUNRISE);
 
@@ -236,7 +238,7 @@ public final class OteAccountBuilderTest {
         .contains("Found existing object(s) conflicting with OT&E objects");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_entitiesExist_succeedsWhenReplaceExisting() {
     persistSimpleResource(makeRegistrar1().asBuilder().setClientId("myclientid-1").build());
     // we intentionally create the -ga TLD with the wrong state, to make sure it's overwritten.
@@ -251,7 +253,7 @@ public final class OteAccountBuilderTest {
     assertRegistrarExists("myclientid-3", "myclientid-ga");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_doubleCreation_actuallyReplaces() {
     OteAccountBuilder.forClientId("myclientid")
         .setPassword("oldPassword")
@@ -273,7 +275,7 @@ public final class OteAccountBuilderTest {
         .isTrue();
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_doubleCreation_keepsOldContacts() {
     OteAccountBuilder.forClientId("myclientid").addContact("email@example.com").buildAndPersist();
 
@@ -288,7 +290,7 @@ public final class OteAccountBuilderTest {
     assertContactExists("myclientid-3", "email@example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateClientIdToTldMap_validEntries() {
     assertThat(OteAccountBuilder.createClientIdToTldMap("myclientid"))
         .containsExactly(
@@ -298,7 +300,7 @@ public final class OteAccountBuilderTest {
             "myclientid-5", "myclientid-eap");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateClientIdToTldMap_invalidId() {
     IllegalArgumentException exception =
         assertThrows(
@@ -306,12 +308,12 @@ public final class OteAccountBuilderTest {
     assertThat(exception).hasMessageThat().isEqualTo("Invalid registrar name: a");
   }
 
-  @Test
+  @TestOfyAndSql
   void testGetBaseClientId_validOteId() {
     assertThat(OteAccountBuilder.getBaseClientId("myclientid-4")).isEqualTo("myclientid");
   }
 
-  @Test
+  @TestOfyAndSql
   void testGetBaseClientId_invalidInput_malformed() {
     assertThat(
             assertThrows(
@@ -321,7 +323,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid OT&E client ID: myclientid");
   }
 
-  @Test
+  @TestOfyAndSql
   void testGetBaseClientId_invalidInput_wrongForBase() {
     assertThat(
             assertThrows(

core/src/test/java/google/registry/model/ofy/EntityWritePrioritiesTest.java

@@ -40,7 +40,7 @@ class EntityWritePrioritiesTest {
             Key.create(HistoryEntry.class, 200), "fake history entry",
             Key.create(Registrar.class, 300), "fake registrar");
     ImmutableMap<Long, Integer> expectedValues =
-        ImmutableMap.of(100L, EntityWritePriorities.DELETE_RANGE + 10, 200L, -10, 300L, 0);
+        ImmutableMap.of(100L, EntityWritePriorities.DELETE_RANGE - 20, 200L, 20, 300L, 0);
 
     for (ImmutableMap.Entry<Key<?>, Object> entry : actions.entrySet()) {
       assertThat(

core/src/test/java/google/registry/model/registry/label/PremiumListDualDaoTest.java

@@ -15,35 +15,41 @@
 package google.registry.model.registry.label;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.newRegistry;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.time.Duration.standardDays;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.truth.Truth8;
 import google.registry.dns.writer.VoidDnsWriter;
+import google.registry.model.EntityTestCase;
+import google.registry.model.common.DatabaseTransitionSchedule;
+import google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabase;
+import google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabaseTransition;
+import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
+import google.registry.model.common.TimedTransitionProperty;
 import google.registry.model.pricing.StaticPremiumListPricingEngine;
 import google.registry.model.registry.Registry;
 import google.registry.schema.tld.PremiumListSqlDao;
-import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.TestCacheExtension;
 import google.registry.testing.TestOfyAndSql;
-import google.registry.testing.TestOfyOnly;
-import google.registry.testing.TestSqlOnly;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link PremiumListDualDao}. */
 @DualDatabaseTest
-public class PremiumListDualDaoTest {
-
-  @RegisterExtension
-  public final AppEngineExtension appEngine =
-      AppEngineExtension.builder().withDatastoreAndCloudSql().build();
+public class PremiumListDualDaoTest extends EntityTestCase {
 
   // Set long persist times on caches so they can be tested (cache times default to 0 in tests).
   @RegisterExtension
@@ -56,9 +62,28 @@ public class PremiumListDualDaoTest {
   @BeforeEach
   void before() {
     createTld("tld");
+    fakeClock.setAutoIncrementStep(Duration.millis(1));
+    fakeClock.setTo(DateTime.parse("1984-12-21T00:00:00.000Z"));
+    DatabaseTransitionSchedule schedule =
+        DatabaseTransitionSchedule.create(
+            TransitionId.DOMAIN_LABEL_LISTS,
+            TimedTransitionProperty.fromValueMap(
+                ImmutableSortedMap.of(
+                    START_OF_TIME,
+                    PrimaryDatabase.DATASTORE,
+                    fakeClock.nowUtc().plusDays(1),
+                    PrimaryDatabase.CLOUD_SQL),
+                PrimaryDatabaseTransition.class));
+
+    tm().transactNew(() -> ofyTm().putWithoutBackup(schedule));
   }
 
-  @TestOfyOnly
+  @AfterEach
+  void after() {
+    fakeClock.setAutoIncrementStep(Duration.ZERO);
+  }
+
+  @TestOfyAndSql
   void testGetPremiumPrice_secondaryLoadMissingSql() {
     PremiumListSqlDao.delete(PremiumListSqlDao.getLatestRevision("tld").get());
     assertThat(
@@ -71,8 +96,9 @@ public class PremiumListDualDaoTest {
                 + "(Optional[USD 20.00]) and secondary SQL db (Optional.empty).");
   }
 
-  @TestSqlOnly
+  @TestOfyAndSql
   void testGetPremiumPrice_secondaryLoadMissingOfy() {
+    fakeClock.advanceBy(Duration.standardDays(5));
     PremiumList premiumList = PremiumListDatastoreDao.getLatestRevision("tld").get();
     PremiumListDatastoreDao.delete(premiumList);
     assertThat(
@@ -85,7 +111,7 @@ public class PremiumListDualDaoTest {
                 + "and secondary Datastore db (Optional.empty).");
   }
 
-  @TestOfyOnly
+  @TestOfyAndSql
   void testGetPremiumPrice_secondaryDifferentSql() {
     PremiumListSqlDao.save("tld", ImmutableList.of("brass,USD 50"));
     assertThat(
@@ -98,8 +124,9 @@ public class PremiumListDualDaoTest {
                 + "(Optional[USD 20.00]) and secondary SQL db (Optional[USD 50.00]).");
   }
 
-  @TestSqlOnly
+  @TestOfyAndSql
   void testGetPremiumPrice_secondaryDifferentOfy() {
+    fakeClock.advanceBy(Duration.standardDays(5));
     PremiumListDatastoreDao.save("tld", ImmutableList.of("brass,USD 50"));
     assertThat(
             assertThrows(

core/src/test/java/google/registry/model/server/ServerSecretTest.java

@@ -15,16 +15,19 @@
 package google.registry.model.server;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.loadByEntity;
+import static google.registry.testing.DatabaseHelper.persistResource;
 
 import google.registry.model.EntityTestCase;
 import google.registry.model.ofy.RequestCapturingAsyncDatastoreService;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.UUID;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link ServerSecret}. */
+@DualDatabaseTest
 public class ServerSecretTest extends EntityTestCase {
 
   ServerSecretTest() {
@@ -36,24 +39,22 @@ public class ServerSecretTest extends EntityTestCase {
     ServerSecret.resetCache();
   }
 
-  @Test
+  @TestOfyAndSql
   void testGet_bootstrapping_savesSecretToDatastore() {
     ServerSecret secret = ServerSecret.get();
     assertThat(secret).isNotNull();
-    assertThat(ofy().load().entity(new ServerSecret()).now()).isEqualTo(secret);
-    assertThat(loadFromSql()).isEqualTo(secret);
+    assertThat(loadByEntity(new ServerSecret())).isEqualTo(secret);
   }
 
-  @Test
+  @TestOfyAndSql
   void testGet_existingSecret_returned() {
     ServerSecret secret = ServerSecret.create(new UUID(123, 456));
-    ofy().saveWithoutBackup().entity(secret).now();
+    persistResource(secret);
     assertThat(ServerSecret.get()).isEqualTo(secret);
-    assertThat(ofy().load().entity(new ServerSecret()).now()).isEqualTo(secret);
-    assertThat(loadFromSql()).isEqualTo(secret);
+    assertThat(loadByEntity(new ServerSecret())).isEqualTo(secret);
   }
 
-  @Test
+  @TestOfyOnly // relies on request-capturing datastore
   void testGet_cachedSecret() {
     int numInitialReads = RequestCapturingAsyncDatastoreService.getReads().size();
     ServerSecret secret = ServerSecret.get();
@@ -63,21 +64,9 @@ public class ServerSecretTest extends EntityTestCase {
     assertThat(RequestCapturingAsyncDatastoreService.getReads()).hasSize(numReads);
   }
 
-  @Test
+  @TestOfyAndSql
   void testAsBytes() {
     byte[] bytes = ServerSecret.create(new UUID(123, 0x456)).asBytes();
     assertThat(bytes).isEqualTo(new byte[] {0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0x4, 0x56});
   }
-
-  private static ServerSecret loadFromSql() {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .query("FROM ServerSecret", ServerSecret.class)
-                    .setMaxResults(1)
-                    .getResultStream()
-                    .findFirst()
-                    .get());
-  }
 }

core/src/test/java/google/registry/model/tmch/TmchCrlTest.java

@@ -15,61 +15,43 @@
 package google.registry.model.tmch;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.loadByEntity;
 
 import google.registry.model.EntityTestCase;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import java.util.Optional;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link TmchCrl}. */
+@DualDatabaseTest
 public class TmchCrlTest extends EntityTestCase {
 
   TmchCrlTest() {
     super(JpaEntityCoverageCheck.ENABLED);
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess() {
     assertThat(TmchCrl.get()).isEqualTo(Optional.empty());
     TmchCrl.set("lolcat", "https://lol.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("lolcat");
   }
 
-  @Test
+  @TestOfyAndSql
   void testDualWrite() {
     TmchCrl expected = new TmchCrl();
     expected.crl = "lolcat";
     expected.url = "https://lol.cat";
     expected.updated = fakeClock.nowUtc();
     TmchCrl.set("lolcat", "https://lol.cat");
-    assertThat(ofy().load().entity(new TmchCrl()).now()).isEqualTo(expected);
-    assertThat(loadFromSql()).isEqualTo(expected);
+    assertThat(loadByEntity(new TmchCrl())).isEqualTo(expected);
   }
 
-  @Test
+  @TestOfyAndSql
   void testMultipleWrites() {
     TmchCrl.set("first", "https://first.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("first");
     TmchCrl.set("second", "https://second.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("second");
-    jpaTm()
-        .transact(
-            () ->
-                assertThat(
-                        jpaTm().query("SELECT COUNT(*) FROM TmchCrl", Long.class).getSingleResult())
-                    .isEqualTo(1L));
-  }
-
-  private static TmchCrl loadFromSql() {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .query("FROM TmchCrl", TmchCrl.class)
-                    .setMaxResults(1)
-                    .getResultStream()
-                    .findFirst()
-                    .get());
   }
 }

core/src/test/java/google/registry/persistence/transaction/QueryComposerTest.java

@@ -0,0 +1,269 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.QueryComposer.Comparator;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Index;
+import google.registry.model.ImmutableObject;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.FakeClock;
+import google.registry.testing.TestOfyAndSql;
+import java.util.Optional;
+import javax.persistence.Column;
+import javax.persistence.NoResultException;
+import javax.persistence.NonUniqueResultException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@DualDatabaseTest
+public class QueryComposerTest {
+
+  private final FakeClock fakeClock = new FakeClock();
+
+  TestEntity alpha = new TestEntity("alpha", 3);
+  TestEntity bravo = new TestEntity("bravo", 2);
+  TestEntity charlie = new TestEntity("charlie", 1);
+
+  @RegisterExtension
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withClock(fakeClock)
+          .withDatastoreAndCloudSql()
+          .withOfyTestEntities(TestEntity.class)
+          .withJpaUnitTestEntities(TestEntity.class)
+          .build();
+
+  public QueryComposerTest() {}
+
+  @BeforeEach
+  void setUp() {
+    tm().transact(
+            () -> {
+              tm().insert(alpha);
+              tm().insert(bravo);
+              tm().insert(charlie);
+            });
+  }
+
+  @TestOfyAndSql
+  public void testFirstQueries() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "bravo")
+                        .first()
+                        .get()))
+        .isEqualTo(bravo);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "bravo")
+                        .first()
+                        .get()))
+        .isEqualTo(charlie);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GTE, "charlie")
+                        .first()
+                        .get()))
+        .isEqualTo(charlie);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LT, "bravo")
+                        .first()
+                        .get()))
+        .isEqualTo(alpha);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LTE, "alpha")
+                        .first()
+                        .get()))
+        .isEqualTo(alpha);
+  }
+
+  @TestOfyAndSql
+  public void testGetSingleResult() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "alpha")
+                        .getSingleResult()))
+        .isEqualTo(alpha);
+  }
+
+  @TestOfyAndSql
+  public void testGetSingleResult_noResults() {
+    assertThrows(
+        NoResultException.class,
+        () ->
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "ziggy")
+                        .getSingleResult()));
+  }
+
+  @TestOfyAndSql
+  public void testGetSingleResult_nonUniqueResult() {
+    assertThrows(
+        NonUniqueResultException.class,
+        () ->
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "alpha")
+                        .getSingleResult()));
+  }
+
+  @TestOfyAndSql
+  public void testStreamQueries() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "alpha")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(alpha));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "alpha")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(bravo, charlie));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GTE, "bravo")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(bravo, charlie));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LT, "charlie")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(alpha, bravo));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LTE, "bravo")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(alpha, bravo));
+  }
+
+  @TestOfyAndSql
+  public void testNonPrimaryKey() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("val", Comparator.EQ, 2)
+                        .first()
+                        .get()))
+        .isEqualTo(bravo);
+  }
+
+  @TestOfyAndSql
+  public void testOrderBy() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("val", Comparator.GT, 1)
+                        .orderBy("val")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(bravo, alpha));
+  }
+
+  @TestOfyAndSql
+  public void testEmptyQueries() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "foxtrot")
+                        .first()))
+        .isEqualTo(Optional.empty());
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "foxtrot")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of());
+  }
+
+  @javax.persistence.Entity
+  @Entity(name = "QueryComposerTestEntity")
+  private static class TestEntity extends ImmutableObject {
+    @javax.persistence.Id @Id private String name;
+
+    @Index
+    // Renaming this implicitly verifies that property names work for hibernate queries.
+    @Column(name = "some_value")
+    private int val;
+
+    public TestEntity() {}
+
+    public TestEntity(String name, int val) {
+      this.name = name;
+      this.val = val;
+    }
+
+    public int getVal() {
+      return val;
+    }
+
+    public String getName() {
+      return name;
+    }
+  }
+}

core/src/test/java/google/registry/privileges/secretmanager/FakeSecretManagerClient.java

@@ -30,7 +30,7 @@ public class FakeSecretManagerClient implements SecretManagerClient {
   private final HashMap<String, SecretEntry> secrets = new HashMap<>();
 
   @Inject
-  FakeSecretManagerClient() {}
+  public FakeSecretManagerClient() {}
 
   @Override
   public String getProject() {

core/src/test/java/google/registry/privileges/secretmanager/KeyringSecretStoreTest.java

@@ -0,0 +1,53 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link KeyringSecretStore}. */
+public class KeyringSecretStoreTest {
+
+  private final SecretManagerClient csmClient = new FakeSecretManagerClient();
+  private final KeyringSecretStore secretStore = new KeyringSecretStore(csmClient);
+  private final byte[] data = {1, 2, 3, 0};
+
+  @Test
+  void createSecret() {
+    secretStore.createOrUpdateSecret("a", data);
+    byte[] persistedData = secretStore.getSecret("a");
+    assertThat(persistedData).isEqualTo(data);
+  }
+
+  @Test
+  void createSecret_underTheHood() {
+    secretStore.createOrUpdateSecret("a", data);
+    byte[] persistedData =
+        csmClient.getSecretData("keyring-a", Optional.empty()).getBytes(StandardCharsets.UTF_8);
+    assertThat(persistedData).isEqualTo(data);
+  }
+
+  @Test
+  void updateSecret() {
+    secretStore.createOrUpdateSecret("a", data);
+    byte[] newData = {0, 1, 2, 3};
+    secretStore.createOrUpdateSecret("a", newData);
+    byte[] persistedData = secretStore.getSecret("a");
+    assertThat(persistedData).isEqualTo(newData);
+  }
+}

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -19,9 +19,12 @@ import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
 import static google.registry.model.common.Cursor.CursorType.RDE_UPLOAD;
+import static google.registry.model.rde.RdeMode.FULL;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadByKey;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.GcsTestingUtils.deleteGcsFile;
 import static google.registry.testing.GcsTestingUtils.writeGcsFile;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -46,6 +49,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSource;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.common.Cursor;
+import google.registry.model.rde.RdeRevision;
 import google.registry.model.registry.Registry;
 import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.request.HttpException.NoContentException;
@@ -123,6 +127,7 @@ public class RdeReportActionTest {
     persistResource(
         Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
     writeGcsFile(gcsService, reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
+    tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
   }
 
   @TestOfyAndSql
@@ -162,6 +167,33 @@ public class RdeReportActionTest {
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
   }
 
+  @TestOfyAndSql
+  void testRunWithLock_regeneratedReport() throws Exception {
+    deleteGcsFile(gcsService, reportFile);
+    GcsFilename newReport =
+        new GcsFilename("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde");
+    PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
+    writeGcsFile(gcsService, newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
+    tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
+    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    createAction().runWithLock(loadRdeReportCursor());
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  void testRunWithLock_nonexistentCursor_throws204() {
+    tm().transact(() -> tm().delete(Cursor.createVKey(RDE_UPLOAD, "test")));
+    NoContentException thrown =
+        assertThrows(
+            NoContentException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Waiting on RdeUploadAction for TLD test to send 2006-06-06T00:00:00.000Z report; last"
+                + " upload completion was at 1970-01-01T00:00:00.000Z");
+  }
+
   @TestOfyAndSql
   void testRunWithLock_uploadNotFinished_throws204() {
     persistResource(

core/src/test/java/google/registry/rde/RdeUploadActionTest.java

@@ -318,6 +318,23 @@ public class RdeUploadActionTest {
     assertThat(stderr).contains("rde-unittest@registry.test");
   }
 
+  @TestOfyAndSql
+  void testRunWithLock_nonexistentCursor_throws204() throws Exception {
+    int port = sftpd.serve("user", "password", folder);
+    URI uploadUrl = URI.create(String.format("sftp://user:password@localhost:%d/", port));
+    DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
+    RdeUploadAction action = createAction(uploadUrl);
+    NoContentException thrown =
+        assertThrows(NoContentException.class, () -> action.runWithLock(uploadCursor));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Waiting on RdeStagingAction for TLD tld to send 2010-10-17T00:00:00.000Z upload; last"
+                + " RDE staging completion was at 1970-01-01T00:00:00.000Z");
+    assertNoTasksEnqueued("rde-upload");
+    assertThat(folder.list()).isEmpty();
+  }
+
   @TestOfyAndSql
   void testRunWithLock_stagingNotFinished_throws204() {
     URI url = URI.create("sftp://user:password@localhost:32323/");

core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java

@@ -210,13 +210,7 @@ class IcannReportingUploadActionTest {
     action.run();
     tm().clearSessionCache();
     verifyNoMoreInteractions(mockReporter);
-    verify(emailService)
-        .sendEmail(
-            EmailMessage.create(
-                "ICANN Monthly report upload summary: 0/0 succeeded",
-                "Report Filename - Upload status:\n",
-                new InternetAddress("recipient@example.com"),
-                new InternetAddress("sender@example.com")));
+    verifyNoMoreInteractions(emailService);
   }
 
   @TestOfyAndSql