Upload latest version of RDE report to icann (#1089)

* Upload latest version of RDE report to icann

Currently the RdeReportAction is hard coded to load the initial version
of a report. This is wrong when reports have been regenerated.

Changed lines are copied from RdeUploadAction.

build.gradle

@@ -24,7 +24,7 @@ buildscript {
   }
 
   dependencies {
-    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.0.1'
+    classpath 'com.google.cloud.tools:appengine-gradle-plugin:2.4.1'
     classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.6.1'
     classpath 'org.sonatype.aether:aether-api:1.13.1'
     classpath 'org.sonatype.aether:aether-impl:1.13.1'

common/src/testing/java/google/registry/testing/FakeClock.java

@@ -35,6 +35,8 @@ public final class FakeClock implements Clock {
   // threads should see a consistent flow.
   private final AtomicLong currentTimeMillis = new AtomicLong();
 
+  private volatile long autoIncrementStepMs;
+
   /** Creates a FakeClock that starts at START_OF_TIME. */
   public FakeClock() {
     this(START_OF_TIME);
@@ -48,7 +50,21 @@ public final class FakeClock implements Clock {
   /** Returns the current time. */
   @Override
   public DateTime nowUtc() {
-    return new DateTime(currentTimeMillis.get(), UTC);
+    return new DateTime(currentTimeMillis.addAndGet(autoIncrementStepMs), UTC);
+  }
+
+  /**
+   * Sets the increment applied to the clock whenever it is queried. The increment is zero by
+   * default: the clock is left unchanged when queried.
+   *
+   * <p>Passing a duration of zero to this method effectively unsets the auto increment mode.
+   *
+   * @param autoIncrementStep the new auto increment duration
+   * @return this
+   */
+  public FakeClock setAutoIncrementStep(ReadableDuration autoIncrementStep) {
+    this.autoIncrementStepMs = autoIncrementStep.getMillis();
+    return this;
   }
 
   /** Advances clock by one millisecond. */

core/build.gradle

@@ -423,7 +423,7 @@ task jaxbToJava {
       }
     }
     execInBash(
-        'find . -name *.java -exec sed -i /\\*\\ \\<p\\>\\$/d {} +',
+        "find . -name *.java -exec sed -i -e '/" + /\* <p>$/ + "/d' {} +",
         generatedDir)
   }
 }

core/gradle/dependency-locks/compile.lockfile

@@ -253,10 +253,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/compileClasspath.lockfile

@@ -246,10 +246,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/default.lockfile

@@ -267,10 +267,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/deploy_jar.lockfile

@@ -266,10 +266,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodCompile.lockfile

@@ -253,10 +253,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodCompileClasspath.lockfile

@@ -247,10 +247,10 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodRuntime.lockfile

@@ -265,10 +265,10 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/nonprodRuntimeClasspath.lockfile

@@ -265,10 +265,10 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/runtime.lockfile

@@ -265,10 +265,10 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/runtimeClasspath.lockfile

@@ -266,10 +266,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testCompile.lockfile

@@ -300,12 +300,12 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -294,12 +294,12 @@ org.seleniumhq.selenium:selenium-remote-driver:3.141.59
 org.seleniumhq.selenium:selenium-safari-driver:3.141.59
 org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testRuntime.lockfile

@@ -313,12 +313,12 @@ org.seleniumhq.selenium:selenium-support:3.141.59
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -314,12 +314,12 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:selenium:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:selenium:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -154,7 +154,13 @@ public class ReplayCommitLogsToSqlAction implements Runnable {
     Object ofyPojo = ofy().toPojo(entity);
     if (ofyPojo instanceof DatastoreEntity) {
       DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
-      datastoreEntity.toSqlEntity().ifPresent(jpaTm()::put);
+      datastoreEntity
+          .toSqlEntity()
+          .ifPresent(
+              sqlEntity -> {
+                ReplaySpecializer.beforeSqlSave(sqlEntity);
+                jpaTm().put(sqlEntity);
+              });
     } else {
       // this should never happen, but we shouldn't fail on it
       logger.atSevere().log(

core/src/main/java/google/registry/keyring/kms/KmsKeyring.java

@@ -21,6 +21,10 @@ import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Streams;
+import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeySerializer;
@@ -29,8 +33,11 @@ import google.registry.keyring.api.KeyringException;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
 import google.registry.model.server.KmsSecretRevisionSqlDao;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Optional;
+import java.util.stream.Stream;
 import javax.inject.Inject;
 import org.bouncycastle.openpgp.PGPException;
 import org.bouncycastle.openpgp.PGPKeyPair;
@@ -46,6 +53,8 @@ import org.bouncycastle.openpgp.PGPPublicKey;
  */
 public class KmsKeyring implements Keyring {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   /** Key labels for private key secrets. */
   enum PrivateKeyLabel {
     BRDA_SIGNING_PRIVATE,
@@ -87,10 +96,13 @@ public class KmsKeyring implements Keyring {
   }
 
   private final KmsConnection kmsConnection;
+  private final KeyringSecretStore secretStore;
 
   @Inject
-  KmsKeyring(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
+  KmsKeyring(
+      @Config("defaultKmsConnection") KmsConnection kmsConnection, KeyringSecretStore secretStore) {
     this.kmsConnection = kmsConnection;
+    this.secretStore = secretStore;
   }
 
   @Override
@@ -192,7 +204,7 @@ public class KmsKeyring implements Keyring {
     return getKeyPair(keyLabel).getPrivateKey();
   }
 
-  private byte[] getDecryptedData(String keyName) {
+  private byte[] getDecryptedDataFromDatastore(String keyName) {
     String encryptedData;
     if (tm().isOfy()) {
       KmsSecret secret =
@@ -213,4 +225,56 @@ public class KmsKeyring implements Keyring {
           String.format("CloudKMS decrypt operation failed for secret %s", keyName), e);
     }
   }
+
+  private byte[] getDataFromSecretStore(String keyName) {
+    try {
+      return secretStore.getSecret(keyName);
+    } catch (Exception e) {
+      return new byte[0];
+    }
+  }
+
+  private byte[] getDecryptedData(String keyName) {
+    byte[] dsData = getDecryptedDataFromDatastore(keyName);
+    byte[] secretStoreData = getDataFromSecretStore(keyName);
+
+    if (Arrays.equals(dsData, secretStoreData)) {
+      logger.atInfo().log("Values for %s in Datastore and Secret Manager match.", keyName);
+      return secretStoreData;
+    }
+    logger.atWarning().log("Values for %s in Datastore and Secret Manager do not match.", keyName);
+    return dsData;
+  }
+
+  /**
+   * Generates the tasks to migrate secrets from Datastore to Secret Manager.
+   *
+   * <p>The keys in the returned {@link ImmutableMap} are the names of the secrets that need
+   * migration. The values in the map are {@link Runnable Runnables} that copy secret data from
+   * Datastore to Secret Manager for their corresponding keys. Only secrets that are absent in
+   * Secret Manager or have inconsistent values are included in the returned map.
+   */
+  public ImmutableMap<String, Runnable> migrationPlan() {
+
+    ImmutableMap.Builder<String, Runnable> tasks = new ImmutableMap.Builder<>();
+
+    ImmutableList<String> labels =
+        Streams.concat(
+                Stream.of(PrivateKeyLabel.values()).map(PrivateKeyLabel::getLabel),
+                Stream.of(PublicKeyLabel.values()).map(PublicKeyLabel::getLabel),
+                Stream.of(StringKeyLabel.values()).map(StringKeyLabel::getLabel))
+            .collect(ImmutableList.toImmutableList());
+
+    for (String keyName : labels) {
+      byte[] dsData = getDecryptedDataFromDatastore(keyName);
+      byte[] secretStoreData = getDataFromSecretStore(keyName);
+      if (Arrays.equals(dsData, secretStoreData)) {
+        logger.atInfo().log("%s is already up to date.\n", keyName);
+        continue;
+      }
+      logger.atInfo().log("%s needs to be migrated.\n", keyName);
+      tasks.put(keyName, () -> secretStore.createOrUpdateSecret(keyName, dsData));
+    }
+    return tasks.build();
+  }
 }

core/src/main/java/google/registry/keyring/kms/KmsUpdater.java

@@ -43,6 +43,7 @@ import google.registry.keyring.kms.KmsKeyring.PublicKeyLabel;
 import google.registry.keyring.kms.KmsKeyring.StringKeyLabel;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -59,11 +60,14 @@ import org.bouncycastle.openpgp.PGPPublicKey;
 public final class KmsUpdater {
 
   private final KmsConnection kmsConnection;
+  private final KeyringSecretStore secretStore;
   private final HashMap<String, byte[]> secretValues;
 
   @Inject
-  public KmsUpdater(@Config("defaultKmsConnection") KmsConnection kmsConnection) {
+  public KmsUpdater(
+      @Config("defaultKmsConnection") KmsConnection kmsConnection, KeyringSecretStore secretStore) {
     this.kmsConnection = kmsConnection;
+    this.secretStore = secretStore;
 
     // Use LinkedHashMap to preserve insertion order on update() to simplify testing and debugging
     this.secretValues = new LinkedHashMap<>();
@@ -132,6 +136,19 @@ public final class KmsUpdater {
     checkState(!secretValues.isEmpty(), "At least one Keyring value must be persisted");
 
     persistEncryptedValues(encryptValues(secretValues));
+
+    // Errors when writing to secret store can be thrown to the top, since writes are always
+    // executed by a human user using the UpdateKmsKeyringCommand.
+    try {
+      secretValues
+          .entrySet()
+          .forEach(e -> secretStore.createOrUpdateSecret(e.getKey(), e.getValue()));
+    } catch (RuntimeException e) {
+      throw new RuntimeException(
+          "Failed to persist secrets to Secret Manager. "
+              + "Please check the status of Secret Manager and re-run the command.",
+          e);
+    }
   }
 
   /**

core/src/main/java/google/registry/model/ImmutableObject.java

@@ -17,6 +17,7 @@ package google.registry.model;
 import static com.google.common.collect.Iterables.transform;
 import static com.google.common.collect.Maps.transformValues;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static java.lang.annotation.ElementType.FIELD;
 import static java.lang.annotation.RetentionPolicy.RUNTIME;
 import static java.util.stream.Collectors.toCollection;
@@ -187,7 +188,10 @@ public abstract class ImmutableObject implements Cloneable {
   /** Helper function to recursively hydrate an ImmutableObject. */
   private static Object hydrate(Object value) {
     if (value instanceof Key) {
-      return hydrate(ofy().load().key((Key<?>) value).now());
+      if (tm().isOfy()) {
+        return hydrate(ofy().load().key((Key<?>) value).now());
+      }
+      return value;
     } else if (value instanceof Map) {
       return transformValues((Map<?, ?>) value, ImmutableObject::hydrate);
     } else if (value instanceof Collection) {

core/src/main/java/google/registry/model/ResourceTransferUtils.java

@@ -17,7 +17,6 @@ package google.registry.model;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
@@ -107,14 +106,14 @@ public final class ResourceTransferUtils {
   /** Update the relevant {@link ForeignKeyIndex} to cache the new deletion time. */
   public static <R extends EppResource> void updateForeignKeyIndexDeletionTime(R resource) {
     if (resource instanceof ForeignKeyedEppResource) {
-      ofy().save().entity(ForeignKeyIndex.create(resource, resource.getDeletionTime()));
+      tm().insert(ForeignKeyIndex.create(resource, resource.getDeletionTime()));
     }
   }
 
   /** If there is a transfer out, delete the server-approve entities and enqueue a poll message. */
   public static <R extends EppResource & ResourceWithTransferData>
       void handlePendingTransferOnDelete(
-            R resource, R newResource, DateTime now, HistoryEntry historyEntry) {
+          R resource, R newResource, DateTime now, HistoryEntry historyEntry) {
     if (resource.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) {
       TransferData oldTransferData = resource.getTransferData();
       tm().delete(oldTransferData.getServerApproveEntities());

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -18,7 +18,8 @@ import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.CollectionUtils.forceEmptyToNull;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
@@ -772,10 +773,10 @@ public abstract class BillingEvent extends ImmutableObject
         Modification instance = getInstance();
         checkNotNull(instance.reason);
         checkNotNull(instance.eventRef);
-        BillingEvent.OneTime billingEvent = ofy().load().key(instance.eventRef).now();
-        checkArgument(Objects.equals(
-            instance.cost.getCurrencyUnit(),
-            billingEvent.cost.getCurrencyUnit()),
+        BillingEvent.OneTime billingEvent =
+            transactIfJpaTm(() -> tm().loadByKey(VKey.from(instance.eventRef)));
+        checkArgument(
+            Objects.equals(instance.cost.getCurrencyUnit(), billingEvent.cost.getCurrencyUnit()),
             "Referenced billing event is in a different currency");
         return super.build();
       }

core/src/main/java/google/registry/model/common/CrossTldSingleton.java

@@ -31,7 +31,7 @@ public abstract class CrossTldSingleton extends ImmutableObject {
 
   public static final long SINGLETON_ID = 1; // There is always exactly one of these.
 
-  @Id @Transient long id = SINGLETON_ID;
+  @Id @javax.persistence.Id long id = SINGLETON_ID;
 
   @Transient @Parent Key<EntityGroupRoot> parent = getCrossTldKey();
 }

core/src/main/java/google/registry/model/common/Cursor.java

@@ -36,6 +36,7 @@ import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import java.io.Serializable;
 import java.util.List;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
@@ -280,8 +281,8 @@ public class Cursor extends ImmutableObject implements DatastoreAndSqlEntity {
   /**
    * Returns the current time for a given cursor, or {@code START_OF_TIME} if the cursor is null.
    */
-  public static DateTime getCursorTimeOrStartOfTime(Cursor cursor) {
-    return cursor != null ? cursor.getCursorTime() : START_OF_TIME;
+  public static DateTime getCursorTimeOrStartOfTime(Optional<Cursor> cursor) {
+    return cursor.map(Cursor::getCursorTime).orElse(START_OF_TIME);
   }
 
   public DateTime getCursorTime() {

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.contact;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -114,6 +116,12 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the contactBase field during asynchronous replay
+  public static void beforeSqlSave(ContactHistory contactHistory) {
+    contactHistory.contactBase =
+        jpaTm().loadByKey(VKey.createSql(ContactResource.class, contactHistory.getContactRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link ContactHistory} entity. */
   public static class ContactHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -15,6 +15,7 @@
 package google.registry.model.domain;
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.collect.ImmutableSet;
@@ -263,6 +264,12 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the domainContent field during asynchronous replay
+  public static void beforeSqlSave(DomainHistory domainHistory) {
+    domainHistory.domainContent =
+        jpaTm().loadByKey(VKey.createSql(DomainBase.class, domainHistory.getDomainRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link DomainHistory} entity. */
   public static class DomainHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.host;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -115,6 +117,12 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
     return Optional.of(asHistoryEntry());
   }
 
+  // Used to fill out the hostBase field during asynchronous replay
+  public static void beforeSqlSave(HostHistory hostHistory) {
+    hostHistory.hostBase =
+        jpaTm().loadByKey(VKey.createSql(HostResource.class, hostHistory.getHostRepoId()));
+  }
+
   /** Class to represent the composite primary key of {@link HostHistory} entity. */
   public static class HostHistoryId extends ImmutableObject implements Serializable {
 

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -196,7 +196,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
       Class<E> clazz, Collection<String> foreignKeys, final DateTime now) {
-    return loadIndexesFromStore(clazz, foreignKeys).entrySet().stream()
+    return loadIndexesFromStore(clazz, foreignKeys, true).entrySet().stream()
         .filter(e -> now.isBefore(e.getValue().getDeletionTime()))
         .collect(entriesToImmutableMap());
   }
@@ -206,13 +206,22 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    * keys, regardless of whether or not they have been soft-deleted.
    *
    * <p>Used by both the cached (w/o deletion check) and the non-cached (with deletion check) calls.
+   *
+   * <p>Note that in the cached case, we wish to run this outside of any transaction because we may
+   * be loading many entities, going over the Datastore limit on the number of enrolled entity
+   * groups per transaction (25). If we require consistency, however, we must use a transaction.
+   *
+   * @param inTransaction whether or not to use an Objectify transaction
    */
   private static <E extends EppResource>
       ImmutableMap<String, ForeignKeyIndex<E>> loadIndexesFromStore(
-          Class<E> clazz, Collection<String> foreignKeys) {
+          Class<E> clazz, Collection<String> foreignKeys, boolean inTransaction) {
     if (tm().isOfy()) {
+      Class<ForeignKeyIndex<E>> fkiClass = mapToFkiClass(clazz);
       return ImmutableMap.copyOf(
-          tm().doTransactionless(() -> ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys)));
+          inTransaction
+              ? ofy().load().type(fkiClass).ids(foreignKeys)
+              : tm().doTransactionless(() -> ofy().load().type(fkiClass).ids(foreignKeys)));
     } else {
       String property = RESOURCE_CLASS_TO_FKI_PROPERTY.get(clazz);
       ImmutableList<ForeignKeyIndex<E>> indexes =
@@ -249,7 +258,8 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
           return Optional.ofNullable(
               loadIndexesFromStore(
                       RESOURCE_CLASS_TO_FKI_CLASS.inverse().get(key.getKind()),
-                      ImmutableSet.of(foreignKey))
+                      ImmutableSet.of(foreignKey),
+                      false)
                   .get(foreignKey));
         }
 
@@ -265,7 +275,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               Streams.stream(keys).map(v -> v.getSqlKey().toString()).collect(toImmutableSet());
           ImmutableSet<VKey<ForeignKeyIndex<?>>> typedKeys = ImmutableSet.copyOf(keys);
           ImmutableMap<String, ? extends ForeignKeyIndex<? extends EppResource>> existingFkis =
-              loadIndexesFromStore(resourceClass, foreignKeys);
+              loadIndexesFromStore(resourceClass, foreignKeys, false);
           // ofy() omits keys that don't have values in Datastore, so re-add them in
           // here with Optional.empty() values.
           return Maps.asMap(

core/src/main/java/google/registry/model/ofy/DatastoreTransactionManager.java

@@ -37,12 +37,17 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.host.HostHistory;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.QueryComposer;
 import google.registry.persistence.transaction.TransactionManager;
+import java.util.List;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import javax.annotation.Nullable;
+import javax.persistence.NoResultException;
+import javax.persistence.NonUniqueResultException;
 import org.joda.time.DateTime;
 
 /** Datastore implementation of {@link TransactionManager}. */
@@ -302,6 +307,11 @@ public class DatastoreTransactionManager implements TransactionManager {
     syncIfTransactionless(getOfy().deleteWithoutBackup().entity(entity));
   }
 
+  @Override
+  public <T> QueryComposer<T> createQueryComposer(Class<T> entity) {
+    return new DatastoreQueryComposerImpl(entity);
+  }
+
   @Override
   public void clearSessionCache() {
     getOfy().clearSessionCache();
@@ -363,4 +373,45 @@ public class DatastoreTransactionManager implements TransactionManager {
     }
     return obj;
   }
+
+  private static class DatastoreQueryComposerImpl<T> extends QueryComposer<T> {
+    DatastoreQueryComposerImpl(Class<T> entityClass) {
+      super(entityClass);
+    }
+
+    Query<T> buildQuery() {
+      Query<T> result = ofy().load().type(entityClass);
+      for (WhereClause pred : predicates) {
+        result = result.filter(pred.fieldName + pred.comparator.getDatastoreString(), pred.value);
+      }
+
+      if (orderBy != null) {
+        result = result.order(orderBy);
+      }
+
+      return result;
+    }
+
+    @Override
+    public Optional<T> first() {
+      return Optional.ofNullable(buildQuery().first().now());
+    }
+
+    @Override
+    public T getSingleResult() {
+      List<T> results = buildQuery().limit(2).list();
+      if (results.size() == 0) {
+        // The exception text here is the same as what we get for JPA queries.
+        throw new NoResultException("No entity found for query");
+      } else if (results.size() > 1) {
+        throw new NonUniqueResultException("More than one result found for getSingleResult query");
+      }
+      return results.get(0);
+    }
+
+    @Override
+    public Stream<T> stream() {
+      return Streams.stream(buildQuery());
+    }
+  }
 }

core/src/main/java/google/registry/model/ofy/EntityWritePriorities.java

@@ -41,10 +41,11 @@ public class EntityWritePriorities {
    */
   static final ImmutableMap<String, Integer> CLASS_PRIORITIES =
       ImmutableMap.of(
-          "ContactResource", -15,
-          "HistoryEntry", -10,
           "AllocationToken", -9,
-          "DomainBase", 10);
+          "ContactResource", 8,
+          "HostResource", 9,
+          "DomainBase", 10,
+          "HistoryEntry", 20);
 
   // The beginning of the range of priority numbers reserved for delete.  This must be greater than
   // any of the values in CLASS_PRIORITIES by enough overhead to accommodate  any negative values in

core/src/main/java/google/registry/model/rde/RdeRevision.java

@@ -103,6 +103,15 @@ public final class RdeRevision extends BackupGroupRoot implements NonReplicatedE
     return revisionOptional.map(rdeRevision -> rdeRevision.revision + 1).orElse(0);
   }
 
+  /** Returns the latest revision of the report already generated for the given triplet. */
+  public static Optional<Integer> getCurrentRevision(String tld, DateTime date, RdeMode mode) {
+    int nextRevision = getNextRevision(tld, date, mode);
+    if (nextRevision == 0) {
+      return Optional.empty();
+    }
+    return Optional.of(nextRevision - 1);
+  }
+
   /**
    * Sets the revision ID for a given triplet.
    *

core/src/main/java/google/registry/model/registry/label/PremiumListDualDao.java

@@ -25,6 +25,7 @@ import google.registry.model.registry.label.PremiumList.PremiumListEntry;
 import google.registry.schema.tld.PremiumListSqlDao;
 import java.util.List;
 import java.util.Optional;
+import org.joda.money.BigMoney;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 
@@ -182,7 +183,7 @@ public class PremiumListDualDao {
           .map(
               premiumEntry ->
                   new PremiumListEntry.Builder()
-                      .setPrice(Money.of(currencyUnit, premiumEntry.getPrice()))
+                      .setPrice(BigMoney.of(currencyUnit, premiumEntry.getPrice()).toMoney())
                       .setLabel(premiumEntry.getDomainLabel())
                       .build())
           .collect(toImmutableList());

core/src/main/java/google/registry/model/server/ServerSecret.java

@@ -15,8 +15,6 @@
 package google.registry.model.server;
 
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -35,10 +33,10 @@ import google.registry.model.common.CrossTldSingleton;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.NonReplicatedEntity;
 import java.nio.ByteBuffer;
+import java.util.Optional;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import javax.persistence.Column;
-import javax.persistence.Id;
 import javax.persistence.PostLoad;
 import javax.persistence.Transient;
 
@@ -67,22 +65,28 @@ public class ServerSecret extends CrossTldSingleton implements NonReplicatedEnti
               });
 
   private static ServerSecret retrieveAndSaveSecret() {
-    VKey<ServerSecret> key =
+    VKey<ServerSecret> vkey =
         VKey.create(
             ServerSecret.class,
             SINGLETON_ID,
             Key.create(getCrossTldKey(), ServerSecret.class, SINGLETON_ID));
+    if (tm().isOfy()) {
+      // Attempt a quick load if we're in ofy first to short-circuit sans transaction
+      Optional<ServerSecret> secretWithoutTransaction = tm().loadByKeyIfPresent(vkey);
+      if (secretWithoutTransaction.isPresent()) {
+        return secretWithoutTransaction.get();
+      }
+    }
     return tm().transact(
             () -> {
-              // transactionally create a new ServerSecret (once per app setup) if necessary.
-              // return the ofy() result during Datastore-primary phase
-              ServerSecret secret =
-                  ofyTm().loadByKeyIfPresent(key).orElseGet(() -> create(UUID.randomUUID()));
-              // During a dual-write period, write it to both Datastore and SQL
-              // even if we didn't have to retrieve it from the DB
-              ofyTm().transact(() -> ofyTm().putWithoutBackup(secret));
-              jpaTm().transact(() -> jpaTm().putWithoutBackup(secret));
-              return secret;
+              // Make sure we're in a transaction and attempt to load any existing secret, then
+              // create it if it's absent.
+              Optional<ServerSecret> secret = tm().loadByKeyIfPresent(vkey);
+              if (!secret.isPresent()) {
+                secret = Optional.of(create(UUID.randomUUID()));
+                tm().insertWithoutBackup(secret.get());
+              }
+              return secret.get();
             });
   }
 
@@ -102,7 +106,6 @@ public class ServerSecret extends CrossTldSingleton implements NonReplicatedEnti
   @Transient long leastSignificant;
 
   /** The UUID value itself. */
-  @Id
   @Column(columnDefinition = "uuid")
   @Ignore
   UUID secret;

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -25,15 +25,11 @@ import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
-import google.registry.model.tmch.TmchCrl.TmchCrlId;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.NonReplicatedEntity;
-import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
 import javax.persistence.Column;
-import javax.persistence.Id;
-import javax.persistence.IdClass;
 import org.joda.time.DateTime;
 
 /** Datastore singleton for ICANN's TMCH CA certificate revocation list (CRL). */
@@ -41,22 +37,26 @@ import org.joda.time.DateTime;
 @javax.persistence.Entity
 @Immutable
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-@IdClass(TmchCrlId.class)
 public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEntity {
 
-  @Id String crl;
+  @Column(name = "certificateRevocations", nullable = false)
+  String crl;
 
-  @Id DateTime updated;
+  @Column(name = "updateTimestamp", nullable = false)
+  DateTime updated;
 
-  @Id String url;
+  @Column(nullable = false)
+  String url;
 
   /** Returns the singleton instance of this entity, without memoization. */
   public static Optional<TmchCrl> get() {
-    VKey<TmchCrl> key =
-        VKey.create(
-            TmchCrl.class, SINGLETON_ID, Key.create(getCrossTldKey(), TmchCrl.class, SINGLETON_ID));
-    // return the ofy() result during Datastore-primary phase
-    return ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(key));
+    return tm().transact(
+            () ->
+                tm().loadByKeyIfPresent(
+                        VKey.create(
+                            TmchCrl.class,
+                            SINGLETON_ID,
+                            Key.create(getCrossTldKey(), TmchCrl.class, SINGLETON_ID))));
   }
 
   /**
@@ -75,13 +75,7 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
               tmchCrl.crl = checkNotNull(crl, "crl");
               tmchCrl.url = checkNotNull(url, "url");
               ofyTm().transactNew(() -> ofyTm().putWithoutBackup(tmchCrl));
-              jpaTm()
-                  .transactNew(
-                      () -> {
-                        // Delete the old one and insert the new one
-                        jpaTm().query("DELETE FROM TmchCrl").executeUpdate();
-                        jpaTm().putWithoutBackup(tmchCrl);
-                      });
+              jpaTm().transactNew(() -> jpaTm().putWithoutBackup(tmchCrl));
             });
   }
 
@@ -99,26 +93,4 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
   public final DateTime getUpdated() {
     return updated;
   }
-
-  static class TmchCrlId implements Serializable {
-
-    @Column(name = "certificateRevocations")
-    String crl;
-
-    @Column(name = "updateTimestamp")
-    DateTime updated;
-
-    String url;
-
-    /** Hibernate requires this default constructor. */
-    private TmchCrlId() {}
-
-    static TmchCrlId create(String crl, DateTime updated, String url) {
-      TmchCrlId result = new TmchCrlId();
-      result.crl = crl;
-      result.updated = updated;
-      result.url = url;
-      return result;
-    }
-  }
 }

core/src/main/java/google/registry/module/frontend/FrontendComponent.java

@@ -30,6 +30,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.frontend.FrontendRequestComponent.FrontendRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
@@ -58,6 +59,7 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
       UrlFetchTransportModule.class,

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -30,6 +30,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.pubapi.PubApiRequestComponent.PubApiRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
@@ -56,6 +57,7 @@ import javax.inject.Singleton;
       KmsModule.class,
       NetHttpTransportModule.class,
       PubApiRequestComponentModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
       UrlFetchTransportModule.class,

core/src/main/java/google/registry/module/tools/ToolsComponent.java

@@ -32,6 +32,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.tools.ToolsRequestComponent.ToolsRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
@@ -61,6 +62,7 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      SecretManagerModule.class,
       ServerTridProviderModule.class,
       StackdriverModule.class,
       ToolsRequestComponentModule.class,

core/src/main/java/google/registry/persistence/transaction/CriteriaQueryBuilder.java

@@ -18,6 +18,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import com.google.common.collect.ImmutableList;
 import java.util.Collection;
+import javax.persistence.EntityManager;
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Expression;
@@ -35,7 +36,7 @@ import javax.persistence.criteria.Root;
 public class CriteriaQueryBuilder<T> {
 
   /** Functional interface that defines the 'where' operator, e.g. {@link CriteriaBuilder#equal}. */
-  public interface WhereClause<U> {
+  public interface WhereOperator<U> {
     Predicate predicate(Expression<U> expression, U object);
   }
 
@@ -50,7 +51,8 @@ public class CriteriaQueryBuilder<T> {
   }
 
   /** Adds a WHERE clause to the query, given the specified operation, field, and value. */
-  public <V> CriteriaQueryBuilder<T> where(String fieldName, WhereClause<V> whereClause, V value) {
+  public <V> CriteriaQueryBuilder<T> where(
+      String fieldName, WhereOperator<V> whereClause, V value) {
     Expression<V> expression = root.get(fieldName);
     return where(whereClause.predicate(expression, value));
   }
@@ -94,7 +96,12 @@ public class CriteriaQueryBuilder<T> {
 
   /** Creates a query builder that will SELECT from the given class. */
   public static <T> CriteriaQueryBuilder<T> create(Class<T> clazz) {
-    CriteriaQuery<T> query = jpaTm().getEntityManager().getCriteriaBuilder().createQuery(clazz);
+    return create(jpaTm().getEntityManager(), clazz);
+  }
+
+  /** Creates a query builder for the given entity manager. */
+  public static <T> CriteriaQueryBuilder<T> create(EntityManager em, Class<T> clazz) {
+    CriteriaQuery<T> query = em.getCriteriaBuilder().createQuery(clazz);
     Root<T> root = query.from(clazz);
     query = query.select(root);
     return new CriteriaQueryBuilder<>(query, root);

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -43,10 +43,12 @@ import google.registry.util.Clock;
 import google.registry.util.Retrier;
 import google.registry.util.SystemSleeper;
 import java.lang.reflect.Field;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 import javax.persistence.EntityManager;
 import javax.persistence.EntityManagerFactory;
@@ -530,6 +532,11 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     delete(entity);
   }
 
+  @Override
+  public <T> QueryComposer<T> createQueryComposer(Class<T> entity) {
+    return new JpaQueryComposerImpl<T>(entity, getEntityManager());
+  }
+
   @Override
   public void clearSessionCache() {
     // This is an intended no-op method as there is no session cache in Postgresql.
@@ -681,4 +688,44 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
       }
     }
   }
+
+  private static class JpaQueryComposerImpl<T> extends QueryComposer<T> {
+
+    EntityManager em;
+
+    JpaQueryComposerImpl(Class<T> entityClass, EntityManager em) {
+      super(entityClass);
+      this.em = em;
+    }
+
+    private TypedQuery<T> buildQuery() {
+      CriteriaQueryBuilder<T> queryBuilder = CriteriaQueryBuilder.create(em, entityClass);
+
+      for (WhereClause<?> pred : predicates) {
+        pred.addToCriteriaQueryBuilder(queryBuilder);
+      }
+
+      if (orderBy != null) {
+        queryBuilder.orderByAsc(orderBy);
+      }
+
+      return em.createQuery(queryBuilder.build());
+    }
+
+    @Override
+    public Optional<T> first() {
+      List<T> results = buildQuery().setMaxResults(1).getResultList();
+      return results.size() > 0 ? Optional.of(results.get(0)) : Optional.empty();
+    }
+
+    @Override
+    public T getSingleResult() {
+      return buildQuery().getSingleResult();
+    }
+
+    @Override
+    public Stream<T> stream() {
+      return buildQuery().getResultStream();
+    }
+  }
 }

core/src/main/java/google/registry/persistence/transaction/QueryComposer.java

@@ -0,0 +1,190 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.base.Function;
+import google.registry.persistence.transaction.CriteriaQueryBuilder.WhereOperator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+import javax.annotation.Nullable;
+import javax.persistence.criteria.CriteriaBuilder;
+
+/**
+ * Creates queries that can be used both for objectify and JPA.
+ *
+ * <p>Example usage:
+ *
+ * <pre>
+ *    tm().createQueryComposer(EntityType.class)
+ *        .where("fieldName", Comparator.EQ, "value")
+ *        .orderBy("fieldName")
+ *        .stream()
+ * </pre>
+ */
+public abstract class QueryComposer<T> {
+
+  // The class whose entities we're querying.  Note that this limits us to single table queries in
+  // SQL.  In datastore, there's really no other kind of query.
+  protected Class<T> entityClass;
+
+  // Field to order by, if any.  Null if we don't care about order.
+  @Nullable protected String orderBy;
+
+  protected List<WhereClause<?>> predicates = new ArrayList<WhereClause<?>>();
+
+  protected QueryComposer(Class<T> entityClass) {
+    this.entityClass = entityClass;
+  }
+
+  /**
+   * Introduce a "where" clause to the query.
+   *
+   * <p>Causes the query to return only results where the field and value have the relationship
+   * specified by the comparator. For example, "field EQ value", "field GT value" etc.
+   */
+  public <U extends Comparable<? super U>> QueryComposer<T> where(
+      String fieldName, Comparator comparator, U value) {
+    predicates.add(new WhereClause(fieldName, comparator, value));
+    return this;
+  }
+
+  /**
+   * Order the query results by the value of the specified field.
+   *
+   * <p>TODO(mmuller): add the ability to do descending sort order.
+   */
+  public QueryComposer<T> orderBy(String fieldName) {
+    orderBy = fieldName;
+    return this;
+  }
+
+  /** Returns the first result of the query or an empty optional if there is none. */
+  public abstract Optional<T> first();
+
+  /**
+   * Returns the one and only result of a query.
+   *
+   * <p>Throws a {@link javax.persistence.NonUniqueResultException} if there is more than one
+   * result, throws {@link javax.persistence.NoResultException} if no results are found.
+   */
+  public abstract T getSingleResult();
+
+  /** Returns the results of the query as a stream. */
+  public abstract Stream<T> stream();
+
+  // We have to wrap the CriteriaQueryBuilder predicate factories in our own functions because at
+  // the point where we pass them to the Comparator constructor, the compiler can't determine which
+  // of the overloads to use since there is no "value" object for context.
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> equal(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::equal;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> lessThan(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::lessThan;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> lessThanOrEqualTo(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::lessThanOrEqualTo;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> greaterThanOrEqualTo(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::greaterThanOrEqualTo;
+  }
+
+  public static <U extends Comparable<? super U>> WhereOperator<U> greaterThan(
+      CriteriaBuilder criteriaBuilder) {
+    return criteriaBuilder::greaterThan;
+  }
+
+  /**
+   * Enum used to specify comparison operations, e.g. {@code where("fieldName", Comparator.NE,
+   * "someval")'}.
+   *
+   * <p>These contain values that specify the comparison behavior for both objectify and criteria
+   * queries. For objectify, we provide a string to be appended to the field name in a {@code
+   * filter()} expression. For criteria queries we provide a function that knows how to obtain a
+   * {@link WhereOperator} from a {@link CriteriaBuilder}.
+   *
+   * <p>Note that the objectify strings for comparators other than equality are preceded by a space
+   * because {@code filter()} expects the fieldname to be separated from the operator by a space.
+   */
+  public enum Comparator {
+    /**
+     * Return only records whose field is equal to the value.
+     *
+     * <p>Note that the datastore string for this is empty, which is consistent with the way {@code
+     * filter()} works (it uses an unadorned field name to check for equality).
+     */
+    EQ("", QueryComposer::equal),
+
+    /** Return only records whose field is less than the value. */
+    LT(" <", QueryComposer::lessThan),
+
+    /** Return only records whose field is less than or equal to the value. */
+    LTE(" <=", QueryComposer::lessThanOrEqualTo),
+
+    /** Return only records whose field is greater than or equal to the value. */
+    GTE(" >=", QueryComposer::greaterThanOrEqualTo),
+
+    /** Return only records whose field is greater than the value. */
+    GT(" >", QueryComposer::greaterThan);
+
+    private final String datastoreString;
+
+    @SuppressWarnings("ImmutableEnumChecker") // Functions are immutable.
+    private final Function<CriteriaBuilder, WhereOperator<?>> operatorFactory;
+
+    Comparator(
+        String datastoreString, Function<CriteriaBuilder, WhereOperator<?>> operatorFactory) {
+      this.datastoreString = datastoreString;
+      this.operatorFactory = operatorFactory;
+    }
+
+    public String getDatastoreString() {
+      return datastoreString;
+    }
+
+    public Function<CriteriaBuilder, WhereOperator<?>> getComparisonFactory() {
+      return operatorFactory;
+    }
+  };
+
+  protected static class WhereClause<U extends Comparable<? super U>> {
+    public String fieldName;
+    public Comparator comparator;
+    public U value;
+
+    WhereClause(String fieldName, Comparator comparator, U value) {
+      this.fieldName = fieldName;
+      this.comparator = comparator;
+      this.value = value;
+    }
+
+    public void addToCriteriaQueryBuilder(CriteriaQueryBuilder queryBuilder) {
+      CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
+      queryBuilder.where(
+          fieldName, comparator.getComparisonFactory().apply(criteriaBuilder), value);
+    }
+  }
+}

core/src/main/java/google/registry/persistence/transaction/TransactionManager.java

@@ -273,6 +273,9 @@ public interface TransactionManager {
    */
   void deleteWithoutBackup(Object entity);
 
+  /** Returns a QueryComposer which can be used to perform queries against the current database. */
+  <T> QueryComposer<T> createQueryComposer(Class<T> entity);
+
   /** Clears the session cache if the underlying database is Datastore, otherwise it is a no-op. */
   void clearSessionCache();
 

core/src/main/java/google/registry/privileges/secretmanager/KeyringSecretStore.java

@@ -0,0 +1,60 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Storage for 'keyring' secrets, backed by the Secret Manager.
+ *
+ * <p>This store is for secrets and credentials that must be set up manually and/or do not require
+ * non-disruptive password changes, e.g., passwords to regulatory reporting websites, which are used
+ * by cron jobs.
+ *
+ * <p>In contrast, the {@link SqlCredentialStore} is designed to support non-disruptive credential
+ * changes with Cloud SQL.
+ */
+public class KeyringSecretStore {
+  static final String SECRET_NAME_PREFIX = "keyring-";
+
+  private final SecretManagerClient csmClient;
+
+  @Inject
+  public KeyringSecretStore(SecretManagerClient csmClient) {
+    this.csmClient = csmClient;
+  }
+
+  public void createOrUpdateSecret(String label, byte[] data) {
+    String secretId = decorateLabel(label);
+    csmClient.createSecretIfAbsent(secretId);
+    csmClient.addSecretVersion(secretId, new String(data, StandardCharsets.UTF_8));
+  }
+
+  public byte[] getSecret(String label) {
+    return csmClient
+        .getSecretData(decorateLabel(label), Optional.empty())
+        .getBytes(StandardCharsets.UTF_8);
+  }
+
+  static String decorateLabel(String label) {
+    checkArgument(!isNullOrEmpty(label), "null or empty label");
+    return SECRET_NAME_PREFIX + label;
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClient.java

@@ -44,6 +44,15 @@ public interface SecretManagerClient {
   /** Returns the {@link SecretVersionState} of all secrets with {@code secretId}. */
   Iterable<SecretVersionState> listSecretVersions(String secretId);
 
+  /** Creates a secret if it does not already exists. */
+  default void createSecretIfAbsent(String secretId) {
+    try {
+      createSecret(secretId);
+    } catch (SecretAlreadyExistsException ignore) {
+      // Not a problem.
+    }
+  }
+
   /**
    * Returns the version strings of all secrets in the given {@code state} with {@code secretId}.
    */

core/src/main/java/google/registry/privileges/secretmanager/SqlCredentialStore.java

@@ -17,7 +17,6 @@ package google.registry.privileges.secretmanager;
 import com.google.cloud.secretmanager.v1.SecretVersionName;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
-import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
 import java.util.Optional;
 import javax.inject.Inject;
 
@@ -74,17 +73,9 @@ public class SqlCredentialStore {
     }
   }
 
-  private void createSecretIfAbsent(String secretId) {
-    try {
-      csmClient.createSecret(secretId);
-    } catch (SecretAlreadyExistsException ignore) {
-      // Not a problem.
-    }
-  }
-
   private SecretVersionName saveCredentialData(SqlUser user, String password) {
     String credentialDataSecretId = getCredentialDataSecretId(user, dbInstance);
-    createSecretIfAbsent(credentialDataSecretId);
+    csmClient.createSecretIfAbsent(credentialDataSecretId);
     String credentialVersion =
         csmClient.addSecretVersion(
             credentialDataSecretId,
@@ -94,7 +85,7 @@ public class SqlCredentialStore {
 
   private void saveLiveLabel(SqlUser user, SecretVersionName dataVersionName) {
     String liveLabelSecretId = getLiveLabelSecretId(user, dbInstance);
-    createSecretIfAbsent(liveLabelSecretId);
+    csmClient.createSecretIfAbsent(liveLabelSecretId);
     csmClient.addSecretVersion(liveLabelSecretId, dataVersionName.toString());
   }
 

core/src/main/java/google/registry/rde/RdeReportAction.java

@@ -24,6 +24,7 @@ import static google.registry.request.Action.Method.POST;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteStreams;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
@@ -31,6 +32,7 @@ import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.common.Cursor;
 import google.registry.model.common.Cursor.CursorType;
 import google.registry.model.rde.RdeNamingUtils;
+import google.registry.model.rde.RdeRevision;
 import google.registry.model.registry.Registry;
 import google.registry.rde.EscrowTaskRunner.EscrowTask;
 import google.registry.request.Action;
@@ -41,6 +43,7 @@ import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Optional;
 import javax.inject.Inject;
 import org.bouncycastle.openpgp.PGPPrivateKey;
 import org.joda.time.DateTime;
@@ -56,6 +59,8 @@ import org.joda.time.Duration;
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public final class RdeReportAction implements Runnable, EscrowTask {
 
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   static final String PATH = "/_dr/task/rdeReport";
 
   @Inject GcsUtils gcsUtils;
@@ -76,8 +81,9 @@ public final class RdeReportAction implements Runnable, EscrowTask {
 
   @Override
   public void runWithLock(DateTime watermark) throws Exception {
-    Cursor cursor =
-        transactIfJpaTm(() -> tm().loadByKey(Cursor.createVKey(CursorType.RDE_UPLOAD, tld)));
+    Optional<Cursor> cursor =
+        transactIfJpaTm(
+            () -> tm().loadByKeyIfPresent(Cursor.createVKey(CursorType.RDE_UPLOAD, tld)));
     DateTime cursorTime = getCursorTimeOrStartOfTime(cursor);
     if (isBeforeOrAt(cursorTime, watermark)) {
       throw new NoContentException(
@@ -86,12 +92,17 @@ public final class RdeReportAction implements Runnable, EscrowTask {
                   + "last upload completion was at %s",
               tld, watermark, cursorTime));
     }
-    String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, 0);
+    int revision =
+        RdeRevision.getCurrentRevision(tld, watermark, FULL)
+            .orElseThrow(
+                () -> new IllegalStateException("RdeRevision was not set on generated deposit"));
+    String prefix = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
     GcsFilename reportFilename = new GcsFilename(bucket, prefix + "-report.xml.ghostryde");
     verify(gcsUtils.existsAndNotEmpty(reportFilename), "Missing file: %s", reportFilename);
     reporter.send(readReportFromGcs(reportFilename));
     response.setContentType(PLAIN_TEXT_UTF_8);
     response.setPayload(String.format("OK %s %s\n", tld, watermark));
+    logger.atInfo().log("Successfully sent report %s.", reportFilename);
   }
 
   /** Reads and decrypts the XML file from cloud storage. */

core/src/main/java/google/registry/rde/RdeStagingReducer.java

@@ -20,8 +20,8 @@ import static com.google.appengine.tools.cloudstorage.GcsServiceFactory.createGc
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
@@ -210,7 +210,11 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
     tm().transact(
             () -> {
               Registry registry = Registry.get(tld);
-              Cursor cursor = ofy().load().key(Cursor.createKey(key.cursor(), registry)).now();
+              Optional<Cursor> cursor =
+                  transactIfJpaTm(
+                      () ->
+                          tm().loadByKeyIfPresent(
+                                  Cursor.createVKey(key.cursor(), registry.getTldStr())));
               DateTime position = getCursorTimeOrStartOfTime(cursor);
               checkState(key.interval() != null, "Interval must be present");
               DateTime newPosition = key.watermark().plus(key.interval());

core/src/main/java/google/registry/rde/RdeUploadAction.java

@@ -64,6 +64,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
+import java.util.Optional;
 import javax.inject.Inject;
 import javax.inject.Named;
 import org.bouncycastle.openpgp.PGPKeyPair;
@@ -133,7 +134,8 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
   @Override
   public void runWithLock(final DateTime watermark) throws Exception {
     logger.atInfo().log("Verifying readiness to upload the RDE deposit.");
-    Cursor cursor = transactIfJpaTm(() -> tm().loadByKey(Cursor.createVKey(RDE_STAGING, tld)));
+    Optional<Cursor> cursor =
+        transactIfJpaTm(() -> tm().loadByKeyIfPresent(Cursor.createVKey(RDE_STAGING, tld)));
     DateTime stagingCursorTime = getCursorTimeOrStartOfTime(cursor);
     if (isBeforeOrAt(stagingCursorTime, watermark)) {
       throw new NoContentException(
@@ -158,8 +160,10 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
               sftpCursorTime,
               timeSinceLastSftp.getStandardMinutes()));
     }
-    int revision = RdeRevision.getNextRevision(tld, watermark, FULL) - 1;
-    verify(revision >= 0, "RdeRevision was not set on generated deposit");
+    int revision =
+        RdeRevision.getCurrentRevision(tld, watermark, FULL)
+            .orElseThrow(
+                () -> new IllegalStateException("RdeRevision was not set on generated deposit"));
     final String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, revision);
     final GcsFilename xmlFilename = new GcsFilename(bucket, name + ".xml.ghostryde");
     final GcsFilename xmlLengthFilename = new GcsFilename(bucket, name + ".xml.length");

core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java

@@ -16,7 +16,6 @@ package google.registry.reporting.icann;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
-import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -107,10 +106,10 @@ public final class IcannReportingUploadAction implements Runnable {
 
           // If cursor time is before now, upload the corresponding report
           cursors.entrySet().stream()
-              .filter(entry -> getCursorTimeOrStartOfTime(entry.getKey()).isBefore(clock.nowUtc()))
+              .filter(entry -> entry.getKey().getCursorTime().isBefore(clock.nowUtc()))
               .forEach(
                   entry -> {
-                    DateTime cursorTime = getCursorTimeOrStartOfTime(entry.getKey());
+                    DateTime cursorTime = entry.getKey().getCursorTime();
                     uploadReport(
                         cursorTime,
                         entry.getKey().getType(),

core/src/main/java/google/registry/reporting/spec11/Spec11EmailUtils.java

@@ -17,7 +17,9 @@ package google.registry.reporting.spec11;
 import static com.google.common.base.Throwables.getRootCause;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.io.Resources.getResource;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.QueryComposer.Comparator;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -129,17 +131,20 @@ public class Spec11EmailUtils {
   private RegistrarThreatMatches filterOutNonPublishedMatches(
       RegistrarThreatMatches registrarThreatMatches) {
     ImmutableList<ThreatMatch> filteredMatches =
-        registrarThreatMatches.threatMatches().stream()
-            .filter(
-                threatMatch ->
-                    ofy()
-                        .load()
-                        .type(DomainBase.class)
-                        .filter("fullyQualifiedDomainName", threatMatch.fullyQualifiedDomainName())
-                        .first()
-                        .now()
-                        .shouldPublishToDns())
-            .collect(toImmutableList());
+        transactIfJpaTm(
+            () -> {
+              return registrarThreatMatches.threatMatches().stream()
+                  .filter(
+                      threatMatch ->
+                          tm().createQueryComposer(DomainBase.class)
+                              .where(
+                                  "fullyQualifiedDomainName",
+                                  Comparator.EQ,
+                                  threatMatch.fullyQualifiedDomainName())
+                              .getSingleResult()
+                              .shouldPublishToDns())
+                  .collect(toImmutableList());
+            });
     return RegistrarThreatMatches.create(registrarThreatMatches.clientId(), filteredMatches);
   }
 

core/src/main/java/google/registry/schema/replay/ReplaySpecializer.java

@@ -26,21 +26,30 @@ import java.lang.reflect.Method;
  * to invoke special class methods if they are present.
  */
 public class ReplaySpecializer {
+
   public static void beforeSqlDelete(VKey<?> key) {
+    invokeMethod(key.getKind(), "beforeSqlDelete", key);
+  }
+
+  public static void beforeSqlSave(SqlEntity sqlEntity) {
+    invokeMethod(sqlEntity.getClass(), "beforeSqlSave", sqlEntity);
+  }
+
+  private static <T> void invokeMethod(Class<T> clazz, String methodName, Object argument) {
     try {
-      Method method = key.getKind().getMethod("beforeSqlDelete", VKey.class);
-      method.invoke(null, new Object[] {key});
+      Method method = clazz.getMethod(methodName, argument.getClass());
+      method.invoke(null, argument);
     } catch (NoSuchMethodException e) {
       // Ignore, this just means that the class doesn't need this hook.
     } catch (IllegalAccessException e) {
       throw new RuntimeException(
-          "beforeSqlDelete() method is defined for class "
-              + key.getKind().getName()
-              + " but is not public.",
+          String.format(
+              "%s() method is defined for class %s but is not public.",
+              methodName, clazz.getName()),
           e);
     } catch (InvocationTargetException e) {
       throw new RuntimeException(
-          "beforeSqlDelete() method for class " + key.getKind().getName() + " threw an exception.",
+          String.format("%s() method for class %s threw an exception", methodName, clazz.getName()),
           e);
     }
   }

core/src/main/java/google/registry/tools/ComparePremiumListsCommand.java

@@ -0,0 +1,105 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
+import google.registry.model.registry.label.PremiumList;
+import google.registry.model.registry.label.PremiumList.PremiumListEntry;
+import google.registry.model.registry.label.PremiumListDatastoreDao;
+import google.registry.schema.tld.PremiumEntry;
+import google.registry.schema.tld.PremiumListSqlDao;
+import java.util.Optional;
+import org.joda.money.BigMoney;
+
+/** Command to compare all PremiumLists in Datastore to all PremiumLists in Cloud SQL. */
+@Parameters(
+    separators = " =",
+    commandDescription = "Compare all the PremiumLists in Datastore to those in Cloud SQL.")
+final class ComparePremiumListsCommand implements CommandWithRemoteApi {
+
+  @Override
+  public void run() {
+    ImmutableSet<String> datastoreLists =
+        ofyTm().loadAllOf(PremiumList.class).stream()
+            .map(PremiumList::getName)
+            .collect(toImmutableSet());
+
+    ImmutableSet<String> sqlLists =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAllOf(PremiumList.class).stream()
+                        .map(PremiumList::getName)
+                        .collect(toImmutableSet()));
+
+    int listsWithDiffs = 0;
+
+    for (String listName : Sets.difference(datastoreLists, sqlLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "PremiumList '%s' is present in Datastore, but not in Cloud SQL.%n", listName);
+    }
+    for (String listName : Sets.difference(sqlLists, datastoreLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "PremiumList '%s' is present in Cloud SQL, but not in Datastore.%n", listName);
+    }
+
+    for (String listName : Sets.intersection(datastoreLists, sqlLists)) {
+      Optional<PremiumList> sqlList = PremiumListSqlDao.getLatestRevision(listName);
+
+      // Datastore and Cloud SQL use different objects to represent premium list entries
+      // so the best way to compare them is to compare their string representations.
+      ImmutableSet<String> datastoreListStrings =
+          Streams.stream(
+                  PremiumListDatastoreDao.loadPremiumListEntriesUncached(
+                      PremiumListDatastoreDao.getLatestRevision(listName).get()))
+              .map(PremiumListEntry::toString)
+              .collect(toImmutableSet());
+
+      Iterable<PremiumEntry> sqlListEntries =
+          jpaTm().transact(() -> PremiumListSqlDao.loadPremiumListEntriesUncached(sqlList.get()));
+
+      ImmutableSet<String> sqlListStrings =
+          Streams.stream(sqlListEntries)
+              .map(
+                  premiumEntry ->
+                      new PremiumListEntry.Builder()
+                          .setPrice(
+                              BigMoney.of(sqlList.get().getCurrency(), premiumEntry.getPrice())
+                                  .toMoney())
+                          .setLabel(premiumEntry.getDomainLabel())
+                          .build()
+                          .toString())
+              .collect(toImmutableSet());
+
+      // This will only print out the name of the unequal list. GetPremiumListCommand
+      // should be used to determine what the actual differences are.
+      if (!datastoreListStrings.equals(sqlListStrings)) {
+        listsWithDiffs++;
+        System.out.printf("PremiumList '%s' has different entries in each database.%n", listName);
+      }
+    }
+
+    System.out.printf("Found %d unequal list(s).%n", listsWithDiffs);
+  }
+}

core/src/main/java/google/registry/tools/CompareReservedListsCommand.java

@@ -0,0 +1,82 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import google.registry.model.registry.label.ReservedList;
+import google.registry.model.registry.label.ReservedList.ReservedListEntry;
+import google.registry.model.registry.label.ReservedListDatastoreDao;
+import google.registry.model.registry.label.ReservedListSqlDao;
+
+/** Command to compare all ReservedLists in Datastore to all ReservedLists in Cloud SQL. */
+@Parameters(
+    separators = " =",
+    commandDescription = "Compare all the ReservedLists in Datastore to those in Cloud SQL.")
+final class CompareReservedListsCommand implements CommandWithRemoteApi {
+
+  @Override
+  public void run() {
+    ImmutableSet<String> datastoreLists =
+        ofy().load().type(ReservedList.class).ancestor(getCrossTldKey()).list().stream()
+            .map(ReservedList::getName)
+            .collect(toImmutableSet());
+
+    ImmutableSet<String> cloudSqlLists =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAllOf(ReservedList.class).stream()
+                        .map(ReservedList::getName)
+                        .collect(toImmutableSet()));
+
+    int listsWithDiffs = 0;
+
+    for (String listName : Sets.difference(datastoreLists, cloudSqlLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "ReservedList '%s' is present in Datastore, but not in Cloud SQL.%n", listName);
+    }
+    for (String listName : Sets.difference(cloudSqlLists, datastoreLists)) {
+      listsWithDiffs++;
+      System.out.printf(
+          "ReservedList '%s' is present in Cloud SQL, but not in Datastore.%n", listName);
+    }
+
+    for (String listName : Sets.intersection(datastoreLists, cloudSqlLists)) {
+      ImmutableMap<String, ReservedListEntry> namesInSql =
+          ReservedListSqlDao.getLatestRevision(listName).get().getReservedListEntries();
+
+      ImmutableMap<String, ReservedListEntry> namesInDatastore =
+          ReservedListDatastoreDao.getLatestRevision(listName).get().getReservedListEntries();
+
+      // This will only print out the name of the unequal list. GetReservedListCommand should be
+      // used to determine what the actual differences are.
+      if (!namesInDatastore.equals(namesInSql)) {
+        listsWithDiffs++;
+        System.out.printf("ReservedList '%s' has different entries in each database.%n", listName);
+      }
+    }
+
+    System.out.printf("Found %d unequal list(s).%n", listsWithDiffs);
+  }
+}

core/src/main/java/google/registry/tools/MigrateKmsKeyringCommand.java

@@ -0,0 +1,78 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.beust.jcommander.Parameters;
+import google.registry.keyring.api.Keyring;
+import google.registry.keyring.kms.KmsKeyring;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
+import java.util.Map;
+import javax.inject.Inject;
+
+/** Migrates secrets from the KMS keyring to the Secret Manager. */
+@Parameters(
+    separators = " =",
+    commandDescription = "Migrate values of secrets in KmsKeyring to Secret Manager.")
+public class MigrateKmsKeyringCommand extends ConfirmingCommand implements CommandWithRemoteApi {
+
+  @Inject Keyring keyring;
+
+  @Inject KeyringSecretStore secretStore;
+
+  Map<String, Runnable> migrationTasks;
+
+  @Inject
+  MigrateKmsKeyringCommand() {}
+
+  @Override
+  protected void init() {
+
+    checkState(
+        keyring instanceof KmsKeyring,
+        "Expecting KmsKeyring, found %s",
+        keyring.getClass().getSimpleName());
+
+    migrationTasks = ((KmsKeyring) keyring).migrationPlan();
+  }
+
+  @Override
+  protected boolean dontRunCommand() {
+    return migrationTasks.isEmpty();
+  }
+
+  @Override
+  protected String prompt() {
+    if (migrationTasks.isEmpty()) {
+      return "All keys are up to date.";
+    }
+    return String.format("Migrate %s keys?", migrationTasks.size());
+  }
+
+  @Override
+  protected String execute() {
+    int errors = 0;
+    for (Map.Entry<String, Runnable> entry : migrationTasks.entrySet()) {
+      try {
+        entry.getValue().run();
+      } catch (Exception e) {
+        System.err.printf("Failed to migrate %s: %s", entry.getKey(), e.getMessage());
+        errors++;
+      }
+    }
+    return errors == 0 ? "Success!" : "Failed to migrate " + errors + "keys.";
+  }
+}

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -17,6 +17,7 @@ package google.registry.tools;
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
 import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
+import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 
@@ -37,6 +38,8 @@ public final class RegistryTool {
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
+          .put("compare_premium_lists", ComparePremiumListsCommand.class)
+          .put("compare_reserved_lists", CompareReservedListsCommand.class)
           .put("convert_idn", ConvertIdnCommand.class)
           .put("count_domains", CountDomainsCommand.class)
           .put("create_anchor_tenant", CreateAnchorTenantCommand.class)
@@ -53,6 +56,7 @@ public final class RegistryTool {
           .put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class)
           .put("dedupe_recurring_billing_event_ids", DedupeRecurringBillingEventIdsCommand.class)
           .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class)
+          .put("delete_contact_by_roid", DeleteContactByRoidCommand.class)
           .put("delete_domain", DeleteDomainCommand.class)
           .put("delete_host", DeleteHostCommand.class)
           .put("delete_premium_list", DeletePremiumListCommand.class)
@@ -101,6 +105,7 @@ public final class RegistryTool {
           .put("lock_domain", LockDomainCommand.class)
           .put("login", LoginCommand.class)
           .put("logout", LogoutCommand.class)
+          .put("migrate_kms_keyring", MigrateKmsKeyringCommand.class)
           .put("pending_escrow", PendingEscrowCommand.class)
           .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -42,6 +42,7 @@ import google.registry.request.Modules.URLFetchServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
+import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.util.UtilsModule;
 import google.registry.whois.NonCachingWhoisModule;
 import javax.annotation.Nullable;
@@ -104,6 +105,8 @@ interface RegistryToolComponent {
 
   void inject(CreateTldCommand command);
 
+  void inject(DeleteContactByRoidCommand command);
+
   void inject(DeployInvoicingPipelineCommand command);
 
   void inject(DeploySpec11PipelineCommand command);
@@ -138,6 +141,8 @@ interface RegistryToolComponent {
 
   void inject(LogoutCommand command);
 
+  void inject(MigrateKmsKeyringCommand command);
+
   void inject(PendingEscrowCommand command);
 
   void inject(RenewDomainCommand command);

core/src/main/java/google/registry/tools/SendEscrowReportToIcannCommand.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
+import dagger.Lazy;
 import google.registry.rde.RdeReporter;
 import google.registry.tools.params.PathParameter;
 import java.nio.file.Files;
@@ -33,13 +34,12 @@ final class SendEscrowReportToIcannCommand implements CommandWithRemoteApi {
       required = true)
   private List<Path> files;
 
-  @Inject
-  RdeReporter rdeReporter;
+  @Inject Lazy<RdeReporter> rdeReporter;
 
   @Override
   public void run() throws Exception {
     for (Path file : files) {
-      rdeReporter.send(Files.readAllBytes(file));
+      rdeReporter.get().send(Files.readAllBytes(file));
       System.out.printf("Uploaded: %s\n", file);
     }
   }

core/src/main/java/google/registry/tools/SetDatabaseTransitionScheduleCommand.java

@@ -14,6 +14,8 @@
 
 package google.registry.tools;
 
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.collect.ImmutableSortedMap;
@@ -23,14 +25,13 @@ import google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabaseTr
 import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
 import google.registry.model.common.TimedTransitionProperty;
 import google.registry.tools.params.TransitionListParameter.PrimaryDatabaseTransitions;
-import java.util.Optional;
 import org.joda.time.DateTime;
 
 /** Command to update {@link DatabaseTransitionSchedule}. */
 @Parameters(
     separators = " =",
     commandDescription = "Set the database transition schedule for transition id.")
-public class SetDatabaseTransitionScheduleCommand extends MutatingCommand {
+public class SetDatabaseTransitionScheduleCommand extends ConfirmingCommand {
 
   @Parameter(
       names = "--transition_schedule",
@@ -43,20 +44,25 @@ public class SetDatabaseTransitionScheduleCommand extends MutatingCommand {
 
   @Parameter(
       names = "--transition_id",
+      required = true,
       description = "Transition id string for the schedule being updated")
   private TransitionId transitionId;
 
   @Override
-  protected void init() {
-    Optional<DatabaseTransitionSchedule> currentSchedule =
-        DatabaseTransitionSchedule.get(transitionId);
+  protected String prompt() {
+    return String.format(
+        "Insert new schedule %s for transition ID %s?", transitionSchedule, transitionId);
+  }
 
+  @Override
+  protected String execute() {
     DatabaseTransitionSchedule newSchedule =
         DatabaseTransitionSchedule.create(
             transitionId,
             TimedTransitionProperty.fromValueMap(
                 transitionSchedule, PrimaryDatabaseTransition.class));
-
-    stageEntityChange(currentSchedule.orElse(null), newSchedule);
+    ofyTm().transact(() -> ofyTm().put(newSchedule));
+    return String.format(
+        "Inserted new schedule %s for transition ID %s.", transitionSchedule, transitionId);
   }
 }

core/src/main/java/google/registry/tools/UnrenewDomainCommand.java

@@ -20,8 +20,8 @@ import static google.registry.flows.domain.DomainFlowUtils.newAutorenewBillingEv
 import static google.registry.flows.domain.DomainFlowUtils.newAutorenewPollMessage;
 import static google.registry.flows.domain.DomainFlowUtils.updateAutorenewRecurrenceEndTime;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.leapSafeSubtractYears;
 
@@ -33,12 +33,12 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.Period.Unit;
 import google.registry.model.eppcommon.StatusValue;
-import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
+import google.registry.model.index.ForeignKeyIndex;
 import google.registry.model.poll.PollMessage;
-import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.HistoryEntry.Type;
 import google.registry.util.Clock;
 import google.registry.util.NonFinalForTesting;
@@ -87,7 +87,7 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
         new ImmutableMap.Builder<>();
 
     for (String domainName : mainParameters) {
-      if (ofy().load().type(ForeignKeyDomainIndex.class).id(domainName).now() == null) {
+      if (ForeignKeyIndex.load(DomainBase.class, domainName, START_OF_TIME) == null) {
         domainsNonexistentBuilder.add(domainName);
         continue;
       }
@@ -183,8 +183,8 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
 
     DateTime newExpirationTime =
         leapSafeSubtractYears(domain.getRegistrationExpirationTime(), period);
-    HistoryEntry historyEntry =
-        new HistoryEntry.Builder()
+    DomainHistory domainHistory =
+        new DomainHistory.Builder()
             .setParent(domain)
             .setModificationTime(now)
             .setBySuperuser(true)
@@ -201,19 +201,19 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
                 String.format(
                     "Domain %s was unrenewed by %d years; now expires at %s.",
                     domainName, period, newExpirationTime))
-            .setParent(historyEntry)
+            .setParent(domainHistory)
             .setEventTime(now)
             .build();
     // Create a new autorenew billing event and poll message starting at the new expiration time.
     BillingEvent.Recurring newAutorenewEvent =
         newAutorenewBillingEvent(domain)
             .setEventTime(newExpirationTime)
-            .setParent(historyEntry)
+            .setParent(domainHistory)
             .build();
     PollMessage.Autorenew newAutorenewPollMessage =
         newAutorenewPollMessage(domain)
             .setEventTime(newExpirationTime)
-            .setParent(historyEntry)
+            .setParent(domainHistory)
             .build();
     // End the old autorenew billing event and poll message now.
     updateAutorenewRecurrenceEndTime(domain, now);
@@ -229,11 +229,9 @@ class UnrenewDomainCommand extends ConfirmingCommand implements CommandWithRemot
     // In order to do it'll need to write out a new HistoryEntry (likely of type SYNTHETIC), a new
     // autorenew billing event and poll message, and a new one time poll message at the present time
     // informing the registrar of this out-of-band change.
-    ofy()
-        .save()
-        .entities(
+    tm().putAll(
             newDomain,
-            historyEntry,
+            domainHistory,
             oneTimePollMessage,
             newAutorenewEvent,
             newAutorenewPollMessage);

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -19,6 +19,7 @@ import static com.google.common.collect.ImmutableListMultimap.flatteningToImmuta
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -135,12 +136,7 @@ public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
             flatteningToImmutableListMultimap(
                 Function.identity(),
                 (domainName) -> {
-                  List<DomainBase> domains =
-                      ofy()
-                          .load()
-                          .type(DomainBase.class)
-                          .filter("fullyQualifiedDomainName", domainName)
-                          .list();
+                  List<DomainBase> domains = loadDomainsForFqdn(domainName);
                   domains.sort(Comparator.comparing(DomainBase::getCreationTime).reversed());
                   checkState(
                       !domains.isEmpty(),
@@ -150,6 +146,25 @@ public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
                 }));
   }
 
+  /** Loads in all {@link DomainBase} objects for a given FQDN. */
+  private List<DomainBase> loadDomainsForFqdn(String fullyQualifiedDomainName) {
+    if (tm().isOfy()) {
+      return ofy()
+          .load()
+          .type(DomainBase.class)
+          .filter("fullyQualifiedDomainName", fullyQualifiedDomainName)
+          .list();
+    } else {
+      return jpaTm()
+          .transact(
+              () ->
+                  jpaTm()
+                      .query("FROM Domain WHERE fullyQualifiedDomainName = :fqdn", DomainBase.class)
+                      .setParameter("fqdn", fullyQualifiedDomainName)
+                      .getResultList());
+    }
+  }
+
   /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
   private Spec11ThreatMatch threatMatchToCloudSqlObject(
       ThreatMatch threatMatch,

core/src/main/java/google/registry/tools/javascrap/DeleteContactByRoidCommand.java

@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.googlecode.objectify.Key;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.eppcommon.StatusValue;
+import google.registry.model.index.EppResourceIndex;
+import google.registry.model.index.ForeignKeyIndex;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.SystemClock;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Deletes a {@link google.registry.model.contact.ContactResource} by its ROID.
+ *
+ * <p>This is a short-term tool for race condition clean up while the bug is being fixed.
+ */
+@Parameters(separators = " =", commandDescription = "Delete a contact by its ROID.")
+public class DeleteContactByRoidCommand extends ConfirmingCommand implements CommandWithRemoteApi {
+
+  @Parameter(names = "--roid", description = "The roid of the contact to be deleted.")
+  String roid;
+
+  @Parameter(
+      names = "--contact_id",
+      description = "The user provided contactId, for verification purpose.")
+  String contactId;
+
+  ImmutableList<Key<?>> toDelete;
+
+  @Override
+  protected void init() throws Exception {
+    System.out.printf("Deleting %s, which refers to %s.\n", roid, contactId);
+    tm().transact(
+            () -> {
+              Key<ContactResource> targetKey = Key.create(ContactResource.class, roid);
+              ContactResource targetContact = ofy().load().key(targetKey).now();
+              verify(
+                  Objects.equals(targetContact.getContactId(), contactId),
+                  "contactId does not match.");
+              verify(
+                  Objects.equals(targetContact.getStatusValues(), ImmutableSet.of(StatusValue.OK)));
+              System.out.println("Target contact has the expected contactId");
+              String canonicalResource =
+                  ForeignKeyIndex.load(ContactResource.class, contactId, new SystemClock().nowUtc())
+                      .getResourceKey()
+                      .getOfyKey()
+                      .getName();
+              verify(!Objects.equals(canonicalResource, roid), "Contact still in ForeignKeyIndex.");
+              System.out.printf(
+                  "It is safe to delete %s, since the contactId is mapped to a different entry in"
+                      + " the Foreign key index (%s).\n\n",
+                  roid, canonicalResource);
+
+              List<Object> ancestors =
+                  ofy().load().ancestor(Key.create(ContactResource.class, roid)).list();
+
+              System.out.println("Ancestor query returns: ");
+              for (Object entity : ancestors) {
+                System.out.println(Key.create(entity));
+              }
+
+              ImmutableSet<String> deletetableKinds =
+                  ImmutableSet.of("HistoryEntry", "ContactResource");
+              toDelete =
+                  ancestors.stream()
+                      .map(Key::create)
+                      .filter(key -> deletetableKinds.contains(key.getKind()))
+                      .collect(ImmutableList.toImmutableList());
+
+              EppResourceIndex eppResourceIndex =
+                  ofy().load().entity(EppResourceIndex.create(targetKey)).now();
+              verify(eppResourceIndex.getKey().equals(targetKey), "Wrong EppResource Index loaded");
+              System.out.printf("\n\nEppResourceIndex found (%s).\n", Key.create(eppResourceIndex));
+
+              toDelete =
+                  new ImmutableList.Builder<Key<?>>()
+                      .addAll(toDelete)
+                      .add(Key.create(eppResourceIndex))
+                      .build();
+
+              System.out.printf("\n\nAbout to delete %s entities:\n", toDelete.size());
+              toDelete.forEach(key -> System.out.println(key));
+            });
+  }
+
+  @Override
+  protected String execute() {
+    tm().transact(() -> ofy().delete().keys(toDelete).now());
+    return "Done";
+  }
+}

core/src/main/java/google/registry/tools/javascrap/PopulateNullRegistrarFieldsCommand.java

@@ -15,8 +15,6 @@
 package google.registry.tools.javascrap;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
-import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 
 import com.beust.jcommander.Parameters;
 import com.google.common.collect.ImmutableList;
@@ -38,7 +36,7 @@ public class PopulateNullRegistrarFieldsCommand extends MutatingCommand {
 
   @Override
   protected void init() {
-    for (Registrar registrar : ofy().load().type(Registrar.class).ancestor(getCrossTldKey())) {
+    for (Registrar registrar : Registrar.loadAll()) {
       Registrar.Builder changeBuilder = registrar.asBuilder();
       changeBuilder.setRegistrarName(
           firstNonNull(registrar.getRegistrarName(), registrar.getClientId()));

core/src/main/java/google/registry/tools/javascrap/RemoveIpAddressCommand.java

@@ -14,13 +14,15 @@
 
 package google.registry.tools.javascrap;
 
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.template.soy.data.SoyMapData;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.tools.MutatingEppToolCommand;
 import google.registry.tools.params.PathParameter;
 import google.registry.tools.soy.RemoveIpAddressSoyInfo;
@@ -30,6 +32,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 /**
  * Command to remove external IP Addresses from HostResources identified by text file listing
@@ -56,14 +59,15 @@ public class RemoveIpAddressCommand extends MutatingEppToolCommand {
 
     for (String roid : roids) {
       // Look up the HostResource from its roid.
-      HostResource host = ofy().load().type(HostResource.class).id(roid).now();
-      if (host == null) {
+      Optional<HostResource> host =
+          transactIfJpaTm(() -> tm().loadByKeyIfPresent(VKey.create(HostResource.class, roid)));
+      if (!host.isPresent()) {
         System.err.printf("Record for %s not found.\n", roid);
         continue;
       }
 
       ArrayList<SoyMapData> ipAddresses = new ArrayList<>();
-      for (InetAddress address : host.getInetAddresses()) {
+      for (InetAddress address : host.get().getInetAddresses()) {
         SoyMapData dataMap = new SoyMapData(
             "address", address.getHostAddress(),
             "version", address instanceof Inet6Address ? "v6" : "v4");
@@ -76,7 +80,7 @@ public class RemoveIpAddressCommand extends MutatingEppToolCommand {
       addSoyRecord(
           registrarId,
           new SoyMapData(
-              "name", host.getHostName(),
+              "name", host.get().getHostName(),
               "ipAddresses", ipAddresses,
               "requestedByRegistrar", registrarId));
     }

core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java

@@ -125,6 +125,7 @@ public class ReplayCommitLogsToSqlActionTest {
     action.diffLister.gcsBucket = GCS_BUCKET;
     action.diffLister.executor = newDirectExecutorService();
     RegistryConfig.overrideCloudSqlReplayCommitLogs(true);
+    TestObject.beforeSqlSaveCallCount = 0;
     TestObject.beforeSqlDeleteCallCount = 0;
   }
 
@@ -442,6 +443,21 @@ public class ReplayCommitLogsToSqlActionTest {
         .isEqualTo("Can't acquire SQL commit log replay lock, aborting.");
   }
 
+  @Test
+  void testSuccess_beforeSqlSaveCallback() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    Key<CommitLogBucket> bucketKey = getBucketKey(1);
+    Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(bucketKey, now, null),
+        CommitLogMutation.create(manifestKey, TestObject.create("a")));
+    runAndAssertSuccess(now.minusMinutes(1));
+    assertThat(TestObject.beforeSqlSaveCallCount).isEqualTo(1);
+  }
+
   @Test
   void testSuccess_deleteSqlCallback() throws Exception {
     DateTime now = fakeClock.nowUtc();

core/src/test/java/google/registry/flows/FlowTestCase.java

@@ -42,7 +42,6 @@ import google.registry.model.eppinput.EppInput;
 import google.registry.model.eppoutput.EppOutput;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.model.tmch.ClaimsListShard.ClaimsListSingleton;
 import google.registry.monitoring.whitebox.EppMetric;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.EppLoader;
@@ -112,8 +111,7 @@ public abstract class FlowTestCase<F extends Flow> {
     sessionMetadata = new HttpSessionMetadata(new FakeHttpSession());
     sessionMetadata.setClientId("TheRegistrar");
     sessionMetadata.setServiceExtensionUris(ProtocolDefinition.getVisibleServiceExtensionUris());
-    ofy().saveWithoutBackup().entity(new ClaimsListSingleton()).now();
- }
+  }
 
   protected void removeServiceExtensionUri(String uri) {
     sessionMetadata.setServiceExtensionUris(

core/src/test/java/google/registry/keyring/kms/KmsKeyringTest.java

@@ -20,6 +20,8 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import google.registry.keyring.api.KeySerializer;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
+import google.registry.privileges.secretmanager.FakeSecretManagerClient;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.DualDatabaseTest;
@@ -45,7 +47,9 @@ class KmsKeyringTest {
 
   @BeforeEach
   void beforeEach() {
-    keyring = new KmsKeyring(new FakeKmsConnection());
+    keyring =
+        new KmsKeyring(
+            new FakeKmsConnection(), new KeyringSecretStore(new FakeSecretManagerClient()));
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/keyring/kms/KmsUpdaterTest.java

@@ -24,6 +24,8 @@ import google.registry.keyring.api.KeySerializer;
 import google.registry.model.server.KmsSecret;
 import google.registry.model.server.KmsSecretRevision;
 import google.registry.model.server.KmsSecretRevisionSqlDao;
+import google.registry.privileges.secretmanager.FakeSecretManagerClient;
+import google.registry.privileges.secretmanager.KeyringSecretStore;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.DualDatabaseTest;
@@ -49,7 +51,9 @@ public class KmsUpdaterTest {
 
   @BeforeEach
   void beforeEach() {
-    updater = new KmsUpdater(new FakeKmsConnection());
+    updater =
+        new KmsUpdater(
+            new FakeKmsConnection(), new KeyringSecretStore(new FakeSecretManagerClient()));
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/model/OteAccountBuilderTest.java

@@ -34,6 +34,8 @@ import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import google.registry.util.CidrAddressBlock;
 import google.registry.util.SystemClock;
 import org.joda.money.Money;
@@ -41,16 +43,16 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
+@DualDatabaseTest
 public final class OteAccountBuilderTest {
 
   @RegisterExtension
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder().withDatastoreAndCloudSql().build();
 
-  @Test
+  @TestOfyAndSql
   void testGetRegistrarToTldMap() {
     assertThat(OteAccountBuilder.forClientId("myclientid").getClientIdToTldMap())
         .containsExactly(
@@ -102,7 +104,7 @@ public final class OteAccountBuilderTest {
     assertThat(contact.getGaeUserId()).isNotEmpty();
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_success() {
     OteAccountBuilder.forClientId("myclientid").addContact("email@example.com").buildAndPersist();
 
@@ -119,7 +121,7 @@ public final class OteAccountBuilderTest {
     assertContactExists("myclientid-5", "email@example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_multipleContacts_success() {
     OteAccountBuilder.forClientId("myclientid")
         .addContact("email@example.com")
@@ -148,7 +150,7 @@ public final class OteAccountBuilderTest {
     assertContactExists("myclientid-5", "someone@example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_setPassword() {
     OteAccountBuilder.forClientId("myclientid").setPassword("myPassword").buildAndPersist();
 
@@ -156,7 +158,7 @@ public final class OteAccountBuilderTest {
         .isTrue();
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_setCertificate() {
     OteAccountBuilder.forClientId("myclientid")
         .setCertificate(SAMPLE_CERT, new SystemClock().nowUtc())
@@ -168,7 +170,7 @@ public final class OteAccountBuilderTest {
         .hasValue(SAMPLE_CERT);
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_setIpAllowList() {
     OteAccountBuilder.forClientId("myclientid")
         .setIpAllowList(ImmutableList.of("1.1.1.0/24"))
@@ -178,7 +180,7 @@ public final class OteAccountBuilderTest {
         .containsExactly(CidrAddressBlock.create("1.1.1.0/24"));
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_invalidClientId_fails() {
     assertThat(
             assertThrows(
@@ -187,7 +189,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: 3blo-bio");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_clientIdTooShort_fails() {
     assertThat(
             assertThrows(IllegalArgumentException.class, () -> OteAccountBuilder.forClientId("bl")))
@@ -195,7 +197,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: bl");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_clientIdTooLong_fails() {
     assertThat(
             assertThrows(
@@ -205,7 +207,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: blobiotoooolong");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_clientIdBadCharacter_fails() {
     assertThat(
             assertThrows(
@@ -214,7 +216,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid registrar name: blo#bio");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_registrarExists_failsWhenNotReplaceExisting() {
     persistSimpleResource(makeRegistrar1().asBuilder().setClientId("myclientid-1").build());
 
@@ -225,7 +227,7 @@ public final class OteAccountBuilderTest {
         .contains("Found existing object(s) conflicting with OT&E objects");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_tldExists_failsWhenNotReplaceExisting() {
     createTld("myclientid-ga", START_DATE_SUNRISE);
 
@@ -236,7 +238,7 @@ public final class OteAccountBuilderTest {
         .contains("Found existing object(s) conflicting with OT&E objects");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_entitiesExist_succeedsWhenReplaceExisting() {
     persistSimpleResource(makeRegistrar1().asBuilder().setClientId("myclientid-1").build());
     // we intentionally create the -ga TLD with the wrong state, to make sure it's overwritten.
@@ -251,7 +253,7 @@ public final class OteAccountBuilderTest {
     assertRegistrarExists("myclientid-3", "myclientid-ga");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_doubleCreation_actuallyReplaces() {
     OteAccountBuilder.forClientId("myclientid")
         .setPassword("oldPassword")
@@ -273,7 +275,7 @@ public final class OteAccountBuilderTest {
         .isTrue();
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateOteEntities_doubleCreation_keepsOldContacts() {
     OteAccountBuilder.forClientId("myclientid").addContact("email@example.com").buildAndPersist();
 
@@ -288,7 +290,7 @@ public final class OteAccountBuilderTest {
     assertContactExists("myclientid-3", "email@example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateClientIdToTldMap_validEntries() {
     assertThat(OteAccountBuilder.createClientIdToTldMap("myclientid"))
         .containsExactly(
@@ -298,7 +300,7 @@ public final class OteAccountBuilderTest {
             "myclientid-5", "myclientid-eap");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreateClientIdToTldMap_invalidId() {
     IllegalArgumentException exception =
         assertThrows(
@@ -306,12 +308,12 @@ public final class OteAccountBuilderTest {
     assertThat(exception).hasMessageThat().isEqualTo("Invalid registrar name: a");
   }
 
-  @Test
+  @TestOfyAndSql
   void testGetBaseClientId_validOteId() {
     assertThat(OteAccountBuilder.getBaseClientId("myclientid-4")).isEqualTo("myclientid");
   }
 
-  @Test
+  @TestOfyAndSql
   void testGetBaseClientId_invalidInput_malformed() {
     assertThat(
             assertThrows(
@@ -321,7 +323,7 @@ public final class OteAccountBuilderTest {
         .isEqualTo("Invalid OT&E client ID: myclientid");
   }
 
-  @Test
+  @TestOfyAndSql
   void testGetBaseClientId_invalidInput_wrongForBase() {
     assertThat(
             assertThrows(

core/src/test/java/google/registry/model/ofy/EntityWritePrioritiesTest.java

@@ -40,7 +40,7 @@ class EntityWritePrioritiesTest {
             Key.create(HistoryEntry.class, 200), "fake history entry",
             Key.create(Registrar.class, 300), "fake registrar");
     ImmutableMap<Long, Integer> expectedValues =
-        ImmutableMap.of(100L, EntityWritePriorities.DELETE_RANGE + 10, 200L, -10, 300L, 0);
+        ImmutableMap.of(100L, EntityWritePriorities.DELETE_RANGE - 20, 200L, 20, 300L, 0);
 
     for (ImmutableMap.Entry<Key<?>, Object> entry : actions.entrySet()) {
       assertThat(

core/src/test/java/google/registry/model/registry/label/PremiumListDualDaoTest.java

@@ -43,6 +43,7 @@ import google.registry.testing.TestCacheExtension;
 import google.registry.testing.TestOfyAndSql;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -61,7 +62,7 @@ public class PremiumListDualDaoTest extends EntityTestCase {
   @BeforeEach
   void before() {
     createTld("tld");
-
+    fakeClock.setAutoIncrementStep(Duration.millis(1));
     fakeClock.setTo(DateTime.parse("1984-12-21T00:00:00.000Z"));
     DatabaseTransitionSchedule schedule =
         DatabaseTransitionSchedule.create(
@@ -77,6 +78,11 @@ public class PremiumListDualDaoTest extends EntityTestCase {
     tm().transactNew(() -> ofyTm().putWithoutBackup(schedule));
   }
 
+  @AfterEach
+  void after() {
+    fakeClock.setAutoIncrementStep(Duration.ZERO);
+  }
+
   @TestOfyAndSql
   void testGetPremiumPrice_secondaryLoadMissingSql() {
     PremiumListSqlDao.delete(PremiumListSqlDao.getLatestRevision("tld").get());

core/src/test/java/google/registry/model/server/ServerSecretTest.java

@@ -15,16 +15,19 @@
 package google.registry.model.server;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.loadByEntity;
+import static google.registry.testing.DatabaseHelper.persistResource;
 
 import google.registry.model.EntityTestCase;
 import google.registry.model.ofy.RequestCapturingAsyncDatastoreService;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import java.util.UUID;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link ServerSecret}. */
+@DualDatabaseTest
 public class ServerSecretTest extends EntityTestCase {
 
   ServerSecretTest() {
@@ -36,24 +39,22 @@ public class ServerSecretTest extends EntityTestCase {
     ServerSecret.resetCache();
   }
 
-  @Test
+  @TestOfyAndSql
   void testGet_bootstrapping_savesSecretToDatastore() {
     ServerSecret secret = ServerSecret.get();
     assertThat(secret).isNotNull();
-    assertThat(ofy().load().entity(new ServerSecret()).now()).isEqualTo(secret);
-    assertThat(loadFromSql()).isEqualTo(secret);
+    assertThat(loadByEntity(new ServerSecret())).isEqualTo(secret);
   }
 
-  @Test
+  @TestOfyAndSql
   void testGet_existingSecret_returned() {
     ServerSecret secret = ServerSecret.create(new UUID(123, 456));
-    ofy().saveWithoutBackup().entity(secret).now();
+    persistResource(secret);
     assertThat(ServerSecret.get()).isEqualTo(secret);
-    assertThat(ofy().load().entity(new ServerSecret()).now()).isEqualTo(secret);
-    assertThat(loadFromSql()).isEqualTo(secret);
+    assertThat(loadByEntity(new ServerSecret())).isEqualTo(secret);
   }
 
-  @Test
+  @TestOfyOnly // relies on request-capturing datastore
   void testGet_cachedSecret() {
     int numInitialReads = RequestCapturingAsyncDatastoreService.getReads().size();
     ServerSecret secret = ServerSecret.get();
@@ -63,21 +64,9 @@ public class ServerSecretTest extends EntityTestCase {
     assertThat(RequestCapturingAsyncDatastoreService.getReads()).hasSize(numReads);
   }
 
-  @Test
+  @TestOfyAndSql
   void testAsBytes() {
     byte[] bytes = ServerSecret.create(new UUID(123, 0x456)).asBytes();
     assertThat(bytes).isEqualTo(new byte[] {0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0x4, 0x56});
   }
-
-  private static ServerSecret loadFromSql() {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .query("FROM ServerSecret", ServerSecret.class)
-                    .setMaxResults(1)
-                    .getResultStream()
-                    .findFirst()
-                    .get());
-  }
 }

core/src/test/java/google/registry/model/tmch/TmchCrlTest.java

@@ -15,61 +15,43 @@
 package google.registry.model.tmch;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.testing.DatabaseHelper.loadByEntity;
 
 import google.registry.model.EntityTestCase;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import java.util.Optional;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link TmchCrl}. */
+@DualDatabaseTest
 public class TmchCrlTest extends EntityTestCase {
 
   TmchCrlTest() {
     super(JpaEntityCoverageCheck.ENABLED);
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess() {
     assertThat(TmchCrl.get()).isEqualTo(Optional.empty());
     TmchCrl.set("lolcat", "https://lol.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("lolcat");
   }
 
-  @Test
+  @TestOfyAndSql
   void testDualWrite() {
     TmchCrl expected = new TmchCrl();
     expected.crl = "lolcat";
     expected.url = "https://lol.cat";
     expected.updated = fakeClock.nowUtc();
     TmchCrl.set("lolcat", "https://lol.cat");
-    assertThat(ofy().load().entity(new TmchCrl()).now()).isEqualTo(expected);
-    assertThat(loadFromSql()).isEqualTo(expected);
+    assertThat(loadByEntity(new TmchCrl())).isEqualTo(expected);
   }
 
-  @Test
+  @TestOfyAndSql
   void testMultipleWrites() {
     TmchCrl.set("first", "https://first.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("first");
     TmchCrl.set("second", "https://second.cat");
     assertThat(TmchCrl.get().get().getCrl()).isEqualTo("second");
-    jpaTm()
-        .transact(
-            () ->
-                assertThat(
-                        jpaTm().query("SELECT COUNT(*) FROM TmchCrl", Long.class).getSingleResult())
-                    .isEqualTo(1L));
-  }
-
-  private static TmchCrl loadFromSql() {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .query("FROM TmchCrl", TmchCrl.class)
-                    .setMaxResults(1)
-                    .getResultStream()
-                    .findFirst()
-                    .get());
   }
 }

core/src/test/java/google/registry/persistence/transaction/QueryComposerTest.java

@@ -0,0 +1,269 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.QueryComposer.Comparator;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
+import static org.junit.Assert.assertThrows;
+
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Index;
+import google.registry.model.ImmutableObject;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.FakeClock;
+import google.registry.testing.TestOfyAndSql;
+import java.util.Optional;
+import javax.persistence.Column;
+import javax.persistence.NoResultException;
+import javax.persistence.NonUniqueResultException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@DualDatabaseTest
+public class QueryComposerTest {
+
+  private final FakeClock fakeClock = new FakeClock();
+
+  TestEntity alpha = new TestEntity("alpha", 3);
+  TestEntity bravo = new TestEntity("bravo", 2);
+  TestEntity charlie = new TestEntity("charlie", 1);
+
+  @RegisterExtension
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withClock(fakeClock)
+          .withDatastoreAndCloudSql()
+          .withOfyTestEntities(TestEntity.class)
+          .withJpaUnitTestEntities(TestEntity.class)
+          .build();
+
+  public QueryComposerTest() {}
+
+  @BeforeEach
+  void setUp() {
+    tm().transact(
+            () -> {
+              tm().insert(alpha);
+              tm().insert(bravo);
+              tm().insert(charlie);
+            });
+  }
+
+  @TestOfyAndSql
+  public void testFirstQueries() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "bravo")
+                        .first()
+                        .get()))
+        .isEqualTo(bravo);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "bravo")
+                        .first()
+                        .get()))
+        .isEqualTo(charlie);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GTE, "charlie")
+                        .first()
+                        .get()))
+        .isEqualTo(charlie);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LT, "bravo")
+                        .first()
+                        .get()))
+        .isEqualTo(alpha);
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LTE, "alpha")
+                        .first()
+                        .get()))
+        .isEqualTo(alpha);
+  }
+
+  @TestOfyAndSql
+  public void testGetSingleResult() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "alpha")
+                        .getSingleResult()))
+        .isEqualTo(alpha);
+  }
+
+  @TestOfyAndSql
+  public void testGetSingleResult_noResults() {
+    assertThrows(
+        NoResultException.class,
+        () ->
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "ziggy")
+                        .getSingleResult()));
+  }
+
+  @TestOfyAndSql
+  public void testGetSingleResult_nonUniqueResult() {
+    assertThrows(
+        NonUniqueResultException.class,
+        () ->
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "alpha")
+                        .getSingleResult()));
+  }
+
+  @TestOfyAndSql
+  public void testStreamQueries() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.EQ, "alpha")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(alpha));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "alpha")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(bravo, charlie));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GTE, "bravo")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(bravo, charlie));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LT, "charlie")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(alpha, bravo));
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.LTE, "bravo")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(alpha, bravo));
+  }
+
+  @TestOfyAndSql
+  public void testNonPrimaryKey() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("val", Comparator.EQ, 2)
+                        .first()
+                        .get()))
+        .isEqualTo(bravo);
+  }
+
+  @TestOfyAndSql
+  public void testOrderBy() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("val", Comparator.GT, 1)
+                        .orderBy("val")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of(bravo, alpha));
+  }
+
+  @TestOfyAndSql
+  public void testEmptyQueries() {
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm().createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "foxtrot")
+                        .first()))
+        .isEqualTo(Optional.empty());
+    assertThat(
+            transactIfJpaTm(
+                () ->
+                    tm()
+                        .createQueryComposer(TestEntity.class)
+                        .where("name", Comparator.GT, "foxtrot")
+                        .stream()
+                        .collect(toImmutableList())))
+        .isEqualTo(ImmutableList.of());
+  }
+
+  @javax.persistence.Entity
+  @Entity(name = "QueryComposerTestEntity")
+  private static class TestEntity extends ImmutableObject {
+    @javax.persistence.Id @Id private String name;
+
+    @Index
+    // Renaming this implicitly verifies that property names work for hibernate queries.
+    @Column(name = "some_value")
+    private int val;
+
+    public TestEntity() {}
+
+    public TestEntity(String name, int val) {
+      this.name = name;
+      this.val = val;
+    }
+
+    public int getVal() {
+      return val;
+    }
+
+    public String getName() {
+      return name;
+    }
+  }
+}

core/src/test/java/google/registry/privileges/secretmanager/FakeSecretManagerClient.java

@@ -30,7 +30,7 @@ public class FakeSecretManagerClient implements SecretManagerClient {
   private final HashMap<String, SecretEntry> secrets = new HashMap<>();
 
   @Inject
-  FakeSecretManagerClient() {}
+  public FakeSecretManagerClient() {}
 
   @Override
   public String getProject() {

core/src/test/java/google/registry/privileges/secretmanager/KeyringSecretStoreTest.java

@@ -0,0 +1,53 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link KeyringSecretStore}. */
+public class KeyringSecretStoreTest {
+
+  private final SecretManagerClient csmClient = new FakeSecretManagerClient();
+  private final KeyringSecretStore secretStore = new KeyringSecretStore(csmClient);
+  private final byte[] data = {1, 2, 3, 0};
+
+  @Test
+  void createSecret() {
+    secretStore.createOrUpdateSecret("a", data);
+    byte[] persistedData = secretStore.getSecret("a");
+    assertThat(persistedData).isEqualTo(data);
+  }
+
+  @Test
+  void createSecret_underTheHood() {
+    secretStore.createOrUpdateSecret("a", data);
+    byte[] persistedData =
+        csmClient.getSecretData("keyring-a", Optional.empty()).getBytes(StandardCharsets.UTF_8);
+    assertThat(persistedData).isEqualTo(data);
+  }
+
+  @Test
+  void updateSecret() {
+    secretStore.createOrUpdateSecret("a", data);
+    byte[] newData = {0, 1, 2, 3};
+    secretStore.createOrUpdateSecret("a", newData);
+    byte[] persistedData = secretStore.getSecret("a");
+    assertThat(persistedData).isEqualTo(newData);
+  }
+}

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -19,9 +19,12 @@ import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
 import static google.registry.model.common.Cursor.CursorType.RDE_UPLOAD;
+import static google.registry.model.rde.RdeMode.FULL;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.loadByKey;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.GcsTestingUtils.deleteGcsFile;
 import static google.registry.testing.GcsTestingUtils.writeGcsFile;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -46,6 +49,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSource;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.common.Cursor;
+import google.registry.model.rde.RdeRevision;
 import google.registry.model.registry.Registry;
 import google.registry.request.HttpException.InternalServerErrorException;
 import google.registry.request.HttpException.NoContentException;
@@ -123,6 +127,7 @@ public class RdeReportActionTest {
     persistResource(
         Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
     writeGcsFile(gcsService, reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
+    tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
   }
 
   @TestOfyAndSql
@@ -162,6 +167,33 @@ public class RdeReportActionTest {
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
   }
 
+  @TestOfyAndSql
+  void testRunWithLock_regeneratedReport() throws Exception {
+    deleteGcsFile(gcsService, reportFile);
+    GcsFilename newReport =
+        new GcsFilename("tub", "test_2006-06-06_full_S1_R1-report.xml.ghostryde");
+    PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
+    writeGcsFile(gcsService, newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
+    tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
+    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
+    createAction().runWithLock(loadRdeReportCursor());
+    assertThat(response.getStatus()).isEqualTo(200);
+  }
+
+  void testRunWithLock_nonexistentCursor_throws204() {
+    tm().transact(() -> tm().delete(Cursor.createVKey(RDE_UPLOAD, "test")));
+    NoContentException thrown =
+        assertThrows(
+            NoContentException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Waiting on RdeUploadAction for TLD test to send 2006-06-06T00:00:00.000Z report; last"
+                + " upload completion was at 1970-01-01T00:00:00.000Z");
+  }
+
   @TestOfyAndSql
   void testRunWithLock_uploadNotFinished_throws204() {
     persistResource(

core/src/test/java/google/registry/rde/RdeUploadActionTest.java

@@ -318,6 +318,23 @@ public class RdeUploadActionTest {
     assertThat(stderr).contains("rde-unittest@registry.test");
   }
 
+  @TestOfyAndSql
+  void testRunWithLock_nonexistentCursor_throws204() throws Exception {
+    int port = sftpd.serve("user", "password", folder);
+    URI uploadUrl = URI.create(String.format("sftp://user:password@localhost:%d/", port));
+    DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
+    RdeUploadAction action = createAction(uploadUrl);
+    NoContentException thrown =
+        assertThrows(NoContentException.class, () -> action.runWithLock(uploadCursor));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Waiting on RdeStagingAction for TLD tld to send 2010-10-17T00:00:00.000Z upload; last"
+                + " RDE staging completion was at 1970-01-01T00:00:00.000Z");
+    assertNoTasksEnqueued("rde-upload");
+    assertThat(folder.list()).isEmpty();
+  }
+
   @TestOfyAndSql
   void testRunWithLock_stagingNotFinished_throws204() {
     URI url = URI.create("sftp://user:password@localhost:32323/");

core/src/test/java/google/registry/reporting/spec11/Spec11EmailUtilsTest.java

@@ -17,11 +17,11 @@ package google.registry.reporting.spec11;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.eppcommon.StatusValue.CLIENT_HOLD;
 import static google.registry.model.eppcommon.StatusValue.SERVER_HOLD;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.getMatchA;
 import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.getMatchB;
 import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
 import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.loadByEntity;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistActiveHost;
 import static google.registry.testing.DatabaseHelper.persistResource;
@@ -39,6 +39,8 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.reporting.spec11.soy.Spec11EmailSoyInfo;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import google.registry.util.EmailMessage;
 import google.registry.util.SendEmailService;
 import java.util.LinkedHashSet;
@@ -48,11 +50,11 @@ import javax.mail.MessagingException;
 import javax.mail.internet.InternetAddress;
 import org.joda.time.LocalDate;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.mockito.ArgumentCaptor;
 
 /** Unit tests for {@link Spec11EmailUtils}. */
+@DualDatabaseTest
 class Spec11EmailUtilsTest {
 
   private static final ImmutableList<String> FAKE_RESOURCES = ImmutableList.of("foo");
@@ -128,7 +130,7 @@ class Spec11EmailUtilsTest {
     persistDomainWithHost("c.com", host);
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_emailMonthlySpec11Reports() throws Exception {
     emailUtils.emailSpec11Reports(
         date,
@@ -166,7 +168,7 @@ class Spec11EmailUtilsTest {
         Optional.empty());
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_emailDailySpec11Reports() throws Exception {
     emailUtils.emailSpec11Reports(
         date,
@@ -204,13 +206,11 @@ class Spec11EmailUtilsTest {
         Optional.empty());
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_skipsInactiveDomain() throws Exception {
     // CLIENT_HOLD and SERVER_HOLD mean no DNS so we don't need to email it out
-    persistResource(
-        ofy().load().entity(aDomain).now().asBuilder().addStatusValue(SERVER_HOLD).build());
-    persistResource(
-        ofy().load().entity(bDomain).now().asBuilder().addStatusValue(CLIENT_HOLD).build());
+    persistResource(loadByEntity(aDomain).asBuilder().addStatusValue(SERVER_HOLD).build());
+    persistResource(loadByEntity(bDomain).asBuilder().addStatusValue(CLIENT_HOLD).build());
     emailUtils.emailSpec11Reports(
         date,
         Spec11EmailSoyInfo.MONTHLY_SPEC_11_EMAIL,
@@ -237,7 +237,7 @@ class Spec11EmailUtilsTest {
         Optional.empty());
   }
 
-  @Test
+  @TestOfyAndSql
   void testOneFailure_sendsAlert() throws Exception {
     // If there is one failure, we should still send the other message and then an alert email
     LinkedHashSet<RegistrarThreatMatches> matches = new LinkedHashSet<>();
@@ -292,7 +292,7 @@ class Spec11EmailUtilsTest {
         Optional.empty());
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_sendAlertEmail() throws Exception {
     emailUtils.sendAlertEmail("Spec11 Pipeline Alert: 2018-07", "Alert!");
     verify(emailService).sendEmail(contentCaptor.capture());
@@ -306,7 +306,7 @@ class Spec11EmailUtilsTest {
         Optional.empty());
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_useWhoisAbuseEmailIfAvailable() throws Exception {
     // if John Doe is the whois abuse contact, email them instead of the regular email
     persistResource(
@@ -325,7 +325,7 @@ class Spec11EmailUtilsTest {
         .containsExactly(new InternetAddress("johndoe@theregistrar.com"));
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_badClientId() {
     RuntimeException thrown =
         assertThrows(

core/src/test/java/google/registry/testing/GcsTestingUtils.java

@@ -40,5 +40,9 @@ public final class GcsTestingUtils {
     gcsService.createOrReplace(file, GcsFileOptions.getDefaultInstance(), ByteBuffer.wrap(data));
   }
 
+  public static void deleteGcsFile(GcsService gcsService, GcsFilename file) throws IOException {
+    gcsService.delete(file);
+  }
+
   private GcsTestingUtils() {}
 }

core/src/test/java/google/registry/testing/TestObject.java

@@ -34,6 +34,7 @@ import javax.persistence.Transient;
 @EntityForTesting
 public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity {
 
+  public static int beforeSqlSaveCallCount;
   public static int beforeSqlDeleteCallCount;
 
   @Parent @Transient Key<EntityGroupRoot> parent;
@@ -74,6 +75,10 @@ public class TestObject extends ImmutableObject implements DatastoreAndSqlEntity
     beforeSqlDeleteCallCount++;
   }
 
+  public static void beforeSqlSave(TestObject testObject) {
+    beforeSqlSaveCallCount++;
+  }
+
   /** A test @VirtualEntity model object, which should not be persisted. */
   @Entity
   @VirtualEntity

core/src/test/java/google/registry/tools/ComparePremiumListsCommandTest.java

@@ -0,0 +1,86 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.testing.DatabaseHelper.persistPremiumList;
+import static org.joda.money.CurrencyUnit.USD;
+
+import com.google.common.collect.ImmutableMap;
+import google.registry.model.registry.label.PremiumList;
+import google.registry.model.registry.label.PremiumListDatastoreDao;
+import google.registry.schema.tld.PremiumListSqlDao;
+import java.math.BigDecimal;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+public class ComparePremiumListsCommandTest extends CommandTestCase<ComparePremiumListsCommand> {
+
+  @BeforeEach
+  void beforeEach() {
+    persistPremiumList("xn--q9jyb4c", "rich,USD 100");
+    persistPremiumList("how", "richer,JPY 10000");
+  }
+
+  @Test
+  void test_success() throws Exception {
+    runCommand();
+    assertThat(getStdoutAsString()).isEqualTo("Found 0 unequal list(s).\n");
+  }
+
+  @Test
+  void test_listMissingFromCloudSql() throws Exception {
+    jpaTm()
+        .transact(
+            () -> {
+              PremiumList premiumList = PremiumListSqlDao.getLatestRevision("how").get();
+              PremiumListSqlDao.delete(premiumList);
+            });
+    runCommand();
+    assertThat(getStdoutAsString())
+        .isEqualTo(
+            "PremiumList 'how' is present in Datastore, but not in Cloud SQL.\n"
+                + "Found 1 unequal list(s).\n");
+  }
+
+  @Test
+  void test_listMissingFromDatastore() throws Exception {
+    PremiumList premiumList = PremiumListDatastoreDao.getLatestRevision("how").get();
+    ofyTm().transact(() -> ofyTm().delete(premiumList));
+    runCommand();
+    assertThat(getStdoutAsString())
+        .isEqualTo(
+            "PremiumList 'how' is present in Cloud SQL, but not in Datastore.\n"
+                + "Found 1 unequal list(s).\n");
+  }
+
+  @Test
+  void test_listsDiffer() throws Exception {
+    PremiumListSqlDao.save(
+        new PremiumList.Builder()
+            .setName("how")
+            .setCurrency(USD)
+            .setLabelsToPrices(ImmutableMap.of("silver", BigDecimal.valueOf(30.03)))
+            .setCreationTime(fakeClock.nowUtc())
+            .build());
+    runCommand();
+    assertThat(getStdoutAsString())
+        .isEqualTo(
+            "PremiumList 'how' has different entries in each database.\n"
+                + "Found 1 unequal list(s).\n");
+  }
+}

core/src/test/java/google/registry/tools/CompareReservedListCommandTest.java

@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.testing.DatabaseHelper.persistReservedList;
+
+import com.google.common.collect.ImmutableMap;
+import google.registry.model.registry.label.ReservationType;
+import google.registry.model.registry.label.ReservedList;
+import google.registry.model.registry.label.ReservedList.ReservedListEntry;
+import google.registry.model.registry.label.ReservedListDatastoreDao;
+import google.registry.model.registry.label.ReservedListDualDatabaseDao;
+import google.registry.model.registry.label.ReservedListSqlDao;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link CompareReservedListsCommand}. */
+public class CompareReservedListCommandTest extends CommandTestCase<CompareReservedListsCommand> {
+
+  @BeforeEach
+  void setUp() {
+    persistReservedList(
+        "testlist", "food, RESERVED_FOR_SPECIFIC_USE", "music, FULLY_BLOCKED # fully blocked");
+    persistReservedList("testlist2", "candy, ALLOWED_IN_SUNRISE");
+  }
+
+  @Test
+  void test_success() throws Exception {
+    runCommand();
+    assertThat(getStdoutAsString()).isEqualTo("Found 0 unequal list(s).\n");
+  }
+
+  @Test
+  void test_listMissingFromCloudSql() throws Exception {
+    jpaTm().transact(() -> jpaTm().delete(ReservedListSqlDao.getLatestRevision("testlist").get()));
+    runCommand();
+    assertThat(getStdoutAsString())
+        .isEqualTo(
+            "ReservedList 'testlist' is present in Datastore, but not in Cloud SQL.\n"
+                + "Found 1 unequal list(s).\n");
+  }
+
+  @Test
+  void test_listMissingFromDatastore() throws Exception {
+    ofyTm()
+        .transact(
+            () -> ofyTm().delete(ReservedListDatastoreDao.getLatestRevision("testlist").get()));
+    runCommand();
+    assertThat(getStdoutAsString())
+        .isEqualTo(
+            "ReservedList 'testlist' is present in Cloud SQL, but not in Datastore.\n"
+                + "Found 1 unequal list(s).\n");
+  }
+
+  @Test
+  void test_listsDiffer() throws Exception {
+    ImmutableMap<String, ReservedListEntry> newReservations =
+        ImmutableMap.of(
+            "food",
+            ReservedListEntry.create("food", ReservationType.RESERVED_FOR_SPECIFIC_USE, null));
+    ReservedList secondList =
+        new ReservedList.Builder()
+            .setName("testlist")
+            .setLastUpdateTime(fakeClock.nowUtc())
+            .setShouldPublish(false)
+            .setReservedListMap(newReservations)
+            .build();
+    ReservedListDatastoreDao.save(secondList);
+    runCommand();
+    assertThat(getStdoutAsString())
+        .isEqualTo(
+            "ReservedList 'testlist' has different entries in each database.\n"
+                + "Found 1 unequal list(s).\n");
+  }
+
+  @Test
+  void test_listsDifferAndMissing() throws Exception {
+    ofyTm()
+        .transact(
+            () -> ofyTm().delete(ReservedListDualDatabaseDao.getLatestRevision("testlist2").get()));
+    ImmutableMap<String, ReservedListEntry> newReservations =
+        ImmutableMap.of(
+            "food",
+            ReservedListEntry.create("food", ReservationType.RESERVED_FOR_SPECIFIC_USE, null));
+    ReservedList secondList =
+        new ReservedList.Builder()
+            .setName("testlist")
+            .setLastUpdateTime(fakeClock.nowUtc())
+            .setShouldPublish(false)
+            .setReservedListMap(newReservations)
+            .build();
+    ReservedListDatastoreDao.save(secondList);
+    runCommand();
+    assertThat(getStdoutAsString())
+        .isEqualTo(
+            "ReservedList 'testlist2' is present in Cloud SQL, but not in Datastore.\n"
+                + "ReservedList 'testlist' has different entries in each database.\n"
+                + "Found 2 unequal list(s).\n");
+  }
+}

core/src/test/java/google/registry/tools/SetDatabaseTransitionScheduleCommandTest.java

@@ -16,18 +16,21 @@ package google.registry.tools;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.beust.jcommander.ParameterException;
 import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.truth.Truth8;
 import com.googlecode.objectify.Key;
 import google.registry.model.common.DatabaseTransitionSchedule;
 import google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabase;
 import google.registry.model.common.DatabaseTransitionSchedule.PrimaryDatabaseTransition;
 import google.registry.model.common.DatabaseTransitionSchedule.TransitionId;
 import google.registry.model.common.TimedTransitionProperty;
+import google.registry.persistence.VKey;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
@@ -37,17 +40,29 @@ import org.junit.jupiter.api.Test;
 public class SetDatabaseTransitionScheduleCommandTest
     extends CommandTestCase<SetDatabaseTransitionScheduleCommand> {
 
-  Key<DatabaseTransitionSchedule> key;
-
   @BeforeEach
   void setup() {
-    key = Key.create(getCrossTldKey(), DatabaseTransitionSchedule.class, "test");
     fakeClock.setTo(DateTime.parse("2020-12-01T00:00:00Z"));
   }
 
+  @Test
+  void testFailure_noTransitionId() throws Exception {
+    ParameterException thrown =
+        assertThrows(
+            ParameterException.class,
+            () -> runCommandForced("--transition_schedule=2021-04-14T00:00:00.000Z=DATASTORE"));
+    assertThat(thrown).hasMessageThat().contains("--transition_id");
+  }
+
   @Test
   void testSuccess_currentScheduleIsEmpty() throws Exception {
-    assertThat(ofy().load().key(key).now()).isNull();
+    Truth8.assertThat(
+            ofyTm()
+                .loadByKeyIfPresent(
+                    VKey.createOfy(
+                        DatabaseTransitionSchedule.class,
+                        Key.create(getCrossTldKey(), DatabaseTransitionSchedule.class, "test"))))
+        .isEmpty();
     runCommandForced(
         "--transition_id=SIGNED_MARK_REVOCATION_LIST",
         "--transition_schedule=1970-01-01T00:00:00.000Z=DATASTORE");
@@ -59,7 +74,10 @@ public class SetDatabaseTransitionScheduleCommandTest
                             .get()
                             .getPrimaryDatabase()))
         .isEqualTo(PrimaryDatabase.DATASTORE);
-    assertThat(command.prompt()).contains("Create DatabaseTransitionSchedule");
+    assertThat(command.prompt())
+        .isEqualTo(
+            "Insert new schedule {1970-01-01T00:00:00.000Z=DATASTORE} "
+                + "for transition ID SIGNED_MARK_REVOCATION_LIST?");
   }
 
   @Test
@@ -81,7 +99,8 @@ public class SetDatabaseTransitionScheduleCommandTest
         .isEqualTo(transitionMap);
     runCommandForced(
         "--transition_id=SIGNED_MARK_REVOCATION_LIST",
-        "--transition_schedule=1970-01-01T00:00:00.000Z=DATASTORE,2020-11-30T00:00:00.000Z=CLOUD_SQL,2020-12-06T00:00:00.000Z=DATASTORE");
+        "--transition_schedule=1970-01-01T00:00:00.000Z=DATASTORE,"
+            + "2020-11-30T00:00:00.000Z=CLOUD_SQL,2020-12-06T00:00:00.000Z=DATASTORE");
     ImmutableSortedMap<DateTime, PrimaryDatabase> retrievedTransitionMap =
         ofyTm()
             .transact(
@@ -106,6 +125,10 @@ public class SetDatabaseTransitionScheduleCommandTest
                             .get()
                             .getPrimaryDatabase()))
         .isEqualTo(PrimaryDatabase.DATASTORE);
-    assertThat(command.prompt()).contains("Update DatabaseTransitionSchedule");
+    assertThat(command.prompt())
+        .isEqualTo(
+            "Insert new schedule {1970-01-01T00:00:00.000Z=DATASTORE, "
+                + "2020-11-30T00:00:00.000Z=CLOUD_SQL, 2020-12-06T00:00:00.000Z=DATASTORE} "
+                + "for transition ID SIGNED_MARK_REVOCATION_LIST?");
   }
 }

core/src/test/java/google/registry/tools/UnrenewDomainCommandTest.java

@@ -18,13 +18,13 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
 import static google.registry.model.eppcommon.StatusValue.PENDING_TRANSFER;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.reporting.HistoryEntry.Type.SYNTHETIC;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.assertBillingEventsEqual;
 import static google.registry.testing.DatabaseHelper.assertPollMessagesEqual;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.getOnlyHistoryEntryOfType;
+import static google.registry.testing.DatabaseHelper.getPollMessages;
+import static google.registry.testing.DatabaseHelper.loadByKey;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
@@ -45,64 +45,79 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.ofy.Ofy;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.testing.FakeClock;
+import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.InjectExtension;
+import google.registry.testing.TestOfyAndSql;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link UnrenewDomainCommand}. */
+@DualDatabaseTest
 public class UnrenewDomainCommandTest extends CommandTestCase<UnrenewDomainCommand> {
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  private final FakeClock clock = new FakeClock(DateTime.parse("2016-12-06T13:55:01Z"));
-
   @BeforeEach
   void beforeEach() {
     createTld("tld");
-    inject.setStaticField(Ofy.class, "clock", clock);
-    command.clock = clock;
+    fakeClock.setTo(DateTime.parse("2016-12-06T13:55:01Z"));
+    inject.setStaticField(Ofy.class, "clock", fakeClock);
+    command.clock = fakeClock;
   }
 
-  @Test
+  @TestOfyAndSql
   void test_unrenewTwoDomains_worksSuccessfully() throws Exception {
     ContactResource contact = persistActiveContact("jd1234");
-    clock.advanceOneMilli();
+    fakeClock.advanceOneMilli();
     persistDomainWithDependentResources(
-        "foo", "tld", contact, clock.nowUtc(), clock.nowUtc(), clock.nowUtc().plusYears(5));
-    clock.advanceOneMilli();
+        "foo",
+        "tld",
+        contact,
+        fakeClock.nowUtc(),
+        fakeClock.nowUtc(),
+        fakeClock.nowUtc().plusYears(5));
+    fakeClock.advanceOneMilli();
     persistDomainWithDependentResources(
-        "bar", "tld", contact, clock.nowUtc(), clock.nowUtc(), clock.nowUtc().plusYears(4));
-    clock.advanceOneMilli();
+        "bar",
+        "tld",
+        contact,
+        fakeClock.nowUtc(),
+        fakeClock.nowUtc(),
+        fakeClock.nowUtc().plusYears(4));
+    fakeClock.advanceOneMilli();
     runCommandForced("-p", "2", "foo.tld", "bar.tld");
-    clock.advanceOneMilli();
+    fakeClock.advanceOneMilli();
     assertThat(
-            loadByForeignKey(DomainBase.class, "foo.tld", clock.nowUtc())
+            loadByForeignKey(DomainBase.class, "foo.tld", fakeClock.nowUtc())
                 .get()
                 .getRegistrationExpirationTime())
         .isEqualTo(DateTime.parse("2019-12-06T13:55:01.001Z"));
     assertThat(
-            loadByForeignKey(DomainBase.class, "bar.tld", clock.nowUtc())
+            loadByForeignKey(DomainBase.class, "bar.tld", fakeClock.nowUtc())
                 .get()
                 .getRegistrationExpirationTime())
         .isEqualTo(DateTime.parse("2018-12-06T13:55:01.002Z"));
     assertInStdout("Successfully unrenewed all domains.");
   }
 
-  @Test
+  @TestOfyAndSql
   void test_unrenewDomain_savesDependentEntitiesCorrectly() throws Exception {
     ContactResource contact = persistActiveContact("jd1234");
-    clock.advanceOneMilli();
+    fakeClock.advanceOneMilli();
     persistDomainWithDependentResources(
-        "foo", "tld", contact, clock.nowUtc(), clock.nowUtc(), clock.nowUtc().plusYears(5));
-    DateTime newExpirationTime = clock.nowUtc().plusYears(3);
-    clock.advanceOneMilli();
+        "foo",
+        "tld",
+        contact,
+        fakeClock.nowUtc(),
+        fakeClock.nowUtc(),
+        fakeClock.nowUtc().plusYears(5));
+    DateTime newExpirationTime = fakeClock.nowUtc().plusYears(3);
+    fakeClock.advanceOneMilli();
     runCommandForced("-p", "2", "foo.tld");
-    DateTime unrenewTime = clock.nowUtc();
-    clock.advanceOneMilli();
-    DomainBase domain = loadByForeignKey(DomainBase.class, "foo.tld", clock.nowUtc()).get();
+    DateTime unrenewTime = fakeClock.nowUtc();
+    fakeClock.advanceOneMilli();
+    DomainBase domain = loadByForeignKey(DomainBase.class, "foo.tld", fakeClock.nowUtc()).get();
 
     assertAboutHistoryEntries()
         .that(getOnlyHistoryEntryOfType(domain, SYNTHETIC))
@@ -120,7 +135,7 @@ public class UnrenewDomainCommandTest extends CommandTestCase<UnrenewDomainComma
     HistoryEntry synthetic = getOnlyHistoryEntryOfType(domain, SYNTHETIC);
 
     assertBillingEventsEqual(
-        tm().loadByKey(domain.getAutorenewBillingEvent()),
+        loadByKey(domain.getAutorenewBillingEvent()),
         new BillingEvent.Recurring.Builder()
             .setParent(synthetic)
             .setReason(Reason.RENEW)
@@ -130,7 +145,7 @@ public class UnrenewDomainCommandTest extends CommandTestCase<UnrenewDomainComma
             .setEventTime(newExpirationTime)
             .build());
     assertPollMessagesEqual(
-        ofy().load().type(PollMessage.class).ancestor(synthetic).list(),
+        getPollMessages(domain),
         ImmutableSet.of(
             new PollMessage.OneTime.Builder()
                 .setParent(synthetic)
@@ -156,7 +171,7 @@ public class UnrenewDomainCommandTest extends CommandTestCase<UnrenewDomainComma
     assertThat(domain.getLastEppUpdateClientId()).isEqualTo("TheRegistrar");
   }
 
-  @Test
+  @TestOfyAndSql
   void test_periodTooLow_fails() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -164,7 +179,7 @@ public class UnrenewDomainCommandTest extends CommandTestCase<UnrenewDomainComma
     assertThat(thrown).hasMessageThat().isEqualTo("Period must be in the range 1-9");
   }
 
-  @Test
+  @TestOfyAndSql
   void test_periodTooHigh_fails() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -172,9 +187,9 @@ public class UnrenewDomainCommandTest extends CommandTestCase<UnrenewDomainComma
     assertThat(thrown).hasMessageThat().isEqualTo("Period must be in the range 1-9");
   }
 
-  @Test
+  @TestOfyAndSql
   void test_varietyOfInvalidDomains_displaysErrors() {
-    DateTime now = clock.nowUtc();
+    DateTime now = fakeClock.nowUtc();
     persistResource(
         newDomainBase("deleting.tld")
             .asBuilder()

core/src/test/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommandTest.java

@@ -37,14 +37,16 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.reporting.Spec11ThreatMatch;
 import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
 import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import google.registry.tools.CommandTestCase;
 import java.io.IOException;
 import org.joda.time.DateTime;
 import org.joda.time.LocalDate;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 /** Tests for {@link BackfillSpec11ThreatMatchesCommand}. */
+@DualDatabaseTest
 public class BackfillSpec11ThreatMatchesCommandTest
     extends CommandTestCase<BackfillSpec11ThreatMatchesCommand> {
 
@@ -67,7 +69,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
         .thenReturn(ImmutableSet.of());
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_singleFile() throws Exception {
     when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
         .thenReturn(sampleThreatMatches());
@@ -77,7 +79,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
     verifyExactlyThreeEntriesInDbFromLastDay();
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_sameDomain_multipleDays() throws Exception {
     // If the same domains show up on multiple days, there should be multiple entries for them
     when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
@@ -106,14 +108,14 @@ public class BackfillSpec11ThreatMatchesCommandTest
             });
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_empty() throws Exception {
     runCommandForced();
     assertInStdout("Backfill Spec11 results from 692 files?");
     assertInStdout("Successfully parsed through 692 files with 0 threats.");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_sameDayTwice() throws Exception {
     when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
         .thenReturn(sampleThreatMatches());
@@ -122,7 +124,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
     verifyExactlyThreeEntriesInDbFromLastDay();
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_threeDomainsForDomainName() throws Exception {
     // We should use the repo ID from the proper DomainBase object at the scan's point in time.
     // First, domain was created at START_OF_TIME and deleted one year ago
@@ -161,7 +163,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
     assertThat(threatMatchRepoId).isNotEqualTo(thirdSave.getRepoId());
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_skipsExistingDatesWithoutOverwrite() throws Exception {
     when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
         .thenReturn(sampleThreatMatches());
@@ -183,7 +185,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
         .isEqualExceptFields(previous, "id");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_overwritesExistingDatesWhenSpecified() throws Exception {
     when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
         .thenReturn(sampleThreatMatches());
@@ -201,7 +203,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
     verifyExactlyThreeEntriesInDbFromLastDay();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_oneFileFails() throws Exception {
     // If there are any exceptions, we should fail loud and fast
     when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
@@ -215,7 +217,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
     jpaTm().transact(() -> assertThat(jpaTm().loadAllOf(Spec11ThreatMatch.class)).isEmpty());
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_noDomainForDomainName() throws Exception {
     deleteResource(domainA);
     when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
@@ -225,7 +227,7 @@ public class BackfillSpec11ThreatMatchesCommandTest
         .isEqualTo("Domain name a.com had no associated DomainBase objects.");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_noDomainAtTimeOfScan() throws Exception {
     // If the domain existed at some point(s) in time but not the time of the scan, fail.
     // First, domain was created at START_OF_TIME and deleted one year ago

db/gradle/dependency-locks/testCompile.lockfile

@@ -32,8 +32,8 @@ org.ow2.asm:asm:9.0
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2

db/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -32,8 +32,8 @@ org.ow2.asm:asm:9.0
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2

db/gradle/dependency-locks/testRuntime.lockfile

@@ -34,8 +34,8 @@ org.ow2.asm:asm:9.0
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2

db/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -67,8 +67,8 @@ org.postgresql:postgresql:42.2.18
 org.rnorth.duct-tape:duct-tape:1.0.8
 org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:junit-jupiter:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2

db/src/main/resources/sql/er_diagram/brief_er_diagram.html

@@ -261,19 +261,19 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-     <td class="property_value">2021-03-24 01:27:00.824998</td> 
+     <td class="property_value">2021-04-12 17:28:08.926599</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V90__update_timestamp.sql</td>
+     <td id="lastFlywayFile" class="property_value">V92__singletons.sql</td>
     </tr>
    </tbody>
   </table> 
   <p>&nbsp;</p> 
   <p>&nbsp;</p> 
-  <svg viewbox="0.00 0.00 4221.44 2624.18" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="erDiagram" style="overflow: hidden; width: 100%; height: 800px"> <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 2620.18)"> 
+  <svg viewbox="0.00 0.00 4221.44 2586.18" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="erDiagram" style="overflow: hidden; width: 100%; height: 800px"> <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 2582.18)"> 
     <title>SchemaCrawler_Diagram</title> 
-    <polygon fill="white" stroke="transparent" points="-4,4 -4,-2620.18 4217.44,-2620.18 4217.44,4 -4,4" /> 
+    <polygon fill="white" stroke="transparent" points="-4,4 -4,-2582.18 4217.44,-2582.18 4217.44,4 -4,4" /> 
     <text text-anchor="start" x="3944.94" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
      generated by
     </text> 
@@ -284,7 +284,7 @@ td.section {
      generated on
     </text> 
     <text text-anchor="start" x="4027.94" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2021-03-24 01:27:00.824998
+     2021-04-12 17:28:08.926599
     </text> 
     <polygon fill="none" stroke="#888888" points="3940.44,-4 3940.44,-44 4205.44,-44 4205.44,-4 3940.44,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 
@@ -2849,23 +2849,23 @@ td.section {
     </g> <!-- serversecret_6cc90f09 --> 
     <g id="node32" class="node"> 
      <title>serversecret_6cc90f09</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="3944.5,-2108.68 3944.5,-2127.68 4075.5,-2127.68 4075.5,-2108.68 3944.5,-2108.68" /> 
-     <text text-anchor="start" x="3946.5" y="-2115.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="3946.5,-2108.68 3946.5,-2127.68 4077.5,-2127.68 4077.5,-2108.68 3946.5,-2108.68" /> 
+     <text text-anchor="start" x="3948.5" y="-2115.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.ServerSecret
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4075.5,-2108.68 4075.5,-2127.68 4152.5,-2127.68 4152.5,-2108.68 4075.5,-2108.68" /> 
-     <text text-anchor="start" x="4113.5" y="-2114.48" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4077.5,-2108.68 4077.5,-2127.68 4151.5,-2127.68 4151.5,-2108.68 4077.5,-2108.68" /> 
+     <text text-anchor="start" x="4112.5" y="-2114.48" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="3946.5" y="-2096.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
-      secret
+     <text text-anchor="start" x="3948.5" y="-2096.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+      id
      </text> 
-     <text text-anchor="start" x="4029.5" y="-2095.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4018.5" y="-2095.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4077.5" y="-2095.48" font-family="Helvetica,sans-Serif" font-size="14.00">
-      uuid not null
+     <text text-anchor="start" x="4079.5" y="-2095.48" font-family="Helvetica,sans-Serif" font-size="14.00">
+      int8 not null
      </text> 
-     <polygon fill="none" stroke="#888888" points="3943.5,-2088.68 3943.5,-2128.68 4153.5,-2128.68 4153.5,-2088.68 3943.5,-2088.68" /> 
+     <polygon fill="none" stroke="#888888" points="3945,-2088.68 3945,-2128.68 4152,-2128.68 4152,-2088.68 3945,-2088.68" /> 
     </g> <!-- signedmarkrevocationentry_99c39721 --> 
     <g id="node33" class="node"> 
      <title>signedmarkrevocationentry_99c39721</title> 
@@ -3004,64 +3004,48 @@ td.section {
     </g> <!-- tmchcrl_d282355 --> 
     <g id="node38" class="node"> 
      <title>tmchcrl_d282355</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="3905.5,-2506.68 3905.5,-2525.68 4065.5,-2525.68 4065.5,-2506.68 3905.5,-2506.68" /> 
-     <text text-anchor="start" x="3907.5" y="-2513.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="3959.5,-2468.68 3959.5,-2487.68 4063.5,-2487.68 4063.5,-2468.68 3959.5,-2468.68" /> 
+     <text text-anchor="start" x="3961.5" y="-2475.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.TmchCrl
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4065.5,-2506.68 4065.5,-2525.68 4191.5,-2525.68 4191.5,-2506.68 4065.5,-2506.68" /> 
-     <text text-anchor="start" x="4152.5" y="-2512.48" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4063.5,-2468.68 4063.5,-2487.68 4137.5,-2487.68 4137.5,-2468.68 4063.5,-2468.68" /> 
+     <text text-anchor="start" x="4098.5" y="-2474.48" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="3907.5" y="-2494.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
-      certificate_revocations
+     <text text-anchor="start" x="3961.5" y="-2456.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+      id
      </text> 
-     <text text-anchor="start" x="4059.5" y="-2493.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4017.5" y="-2455.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4067.5" y="-2493.48" font-family="Helvetica,sans-Serif" font-size="14.00">
-      text not null
+     <text text-anchor="start" x="4065.5" y="-2455.48" font-family="Helvetica,sans-Serif" font-size="14.00">
+      int8 not null
      </text> 
-     <text text-anchor="start" x="3907.5" y="-2475.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
-      update_timestamp
-     </text> 
-     <text text-anchor="start" x="4059.5" y="-2474.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4067.5" y="-2474.48" font-family="Helvetica,sans-Serif" font-size="14.00">
-      timestamptz not null
-     </text> 
-     <text text-anchor="start" x="3907.5" y="-2456.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
-      url
-     </text> 
-     <text text-anchor="start" x="4059.5" y="-2455.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4067.5" y="-2455.48" font-family="Helvetica,sans-Serif" font-size="14.00">
-      text not null
-     </text> 
-     <polygon fill="none" stroke="#888888" points="3904.5,-2448.68 3904.5,-2526.68 4192.5,-2526.68 4192.5,-2448.68 3904.5,-2448.68" /> 
+     <polygon fill="none" stroke="#888888" points="3958.5,-2448.68 3958.5,-2488.68 4138.5,-2488.68 4138.5,-2448.68 3958.5,-2448.68" /> 
     </g> <!-- transaction_d50389d4 --> 
     <g id="node39" class="node"> 
      <title>transaction_d50389d4</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="3931.5,-2591.68 3931.5,-2610.68 4056.5,-2610.68 4056.5,-2591.68 3931.5,-2591.68" /> 
-     <text text-anchor="start" x="3933.5" y="-2598.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="3931.5,-2553.68 3931.5,-2572.68 4056.5,-2572.68 4056.5,-2553.68 3931.5,-2553.68" /> 
+     <text text-anchor="start" x="3933.5" y="-2560.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.Transaction
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4056.5,-2591.68 4056.5,-2610.68 4166.5,-2610.68 4166.5,-2591.68 4056.5,-2591.68" /> 
-     <text text-anchor="start" x="4127.5" y="-2597.48" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4056.5,-2553.68 4056.5,-2572.68 4166.5,-2572.68 4166.5,-2553.68 4056.5,-2553.68" /> 
+     <text text-anchor="start" x="4127.5" y="-2559.48" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="3933.5" y="-2579.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="3933.5" y="-2541.48" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       id
      </text> 
-     <text text-anchor="start" x="4000.5" y="-2578.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4000.5" y="-2540.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4058.5" y="-2578.48" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4058.5" y="-2540.48" font-family="Helvetica,sans-Serif" font-size="14.00">
       bigserial not null
      </text> 
-     <text text-anchor="start" x="4000.5" y="-2559.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4000.5" y="-2521.48" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4058.5" y="-2559.48" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4058.5" y="-2521.48" font-family="Helvetica,sans-Serif" font-size="14.00">
       auto-incremented
      </text> 
-     <polygon fill="none" stroke="#888888" points="3930,-2553.18 3930,-2612.18 4167,-2612.18 4167,-2553.18 3930,-2553.18" /> 
+     <polygon fill="none" stroke="#888888" points="3930,-2515.18 3930,-2574.18 4167,-2574.18 4167,-2515.18 3930,-2515.18" /> 
     </g> 
    </g> 
   </svg> 
@@ -6427,8 +6411,8 @@ td.section {
    <tbody>
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth"><b><i>secret</i></b></td> 
-     <td class="minwidth">uuid not null</td> 
+     <td class="minwidth"><b><i>id</i></b></td> 
+     <td class="minwidth">int8 not null</td> 
     </tr> 
     <tr> 
      <td colspan="3"></td> 
@@ -6445,7 +6429,7 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">secret</td> 
+     <td class="minwidth">id</td> 
      <td class="minwidth"></td> 
     </tr> 
    </tbody>
@@ -6708,18 +6692,8 @@ td.section {
    <tbody>
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth"><b><i>certificate_revocations</i></b></td> 
-     <td class="minwidth">text not null</td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth"><b><i>update_timestamp</i></b></td> 
-     <td class="minwidth">timestamptz not null</td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth"><b><i>url</i></b></td> 
-     <td class="minwidth">text not null</td> 
+     <td class="minwidth"><b><i>id</i></b></td> 
+     <td class="minwidth">int8 not null</td> 
     </tr> 
     <tr> 
      <td colspan="3"></td> 
@@ -6736,17 +6710,7 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">certificate_revocations</td> 
-     <td class="minwidth"></td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth">update_timestamp</td> 
-     <td class="minwidth"></td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth">url</td> 
+     <td class="minwidth">id</td> 
      <td class="minwidth"></td> 
     </tr> 
    </tbody>

db/src/main/resources/sql/er_diagram/full_er_diagram.html

@@ -261,32 +261,32 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-     <td class="property_value">2021-03-24 01:26:58.684653</td> 
+     <td class="property_value">2021-04-12 17:28:07.032036</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V90__update_timestamp.sql</td>
+     <td id="lastFlywayFile" class="property_value">V92__singletons.sql</td>
     </tr>
    </tbody>
   </table> 
   <p>&nbsp;</p> 
   <p>&nbsp;</p> 
-  <svg viewbox="0.00 0.00 4867.18 4862.91" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="erDiagram" style="overflow: hidden; width: 100%; height: 800px"> <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 4858.91)"> 
+  <svg viewbox="0.00 0.00 4850.02 4900.91" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="erDiagram" style="overflow: hidden; width: 100%; height: 800px"> <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 4896.91)"> 
     <title>SchemaCrawler_Diagram</title> 
-    <polygon fill="white" stroke="transparent" points="-4,4 -4,-4858.91 4863.18,-4858.91 4863.18,4 -4,4" /> 
-    <text text-anchor="start" x="4590.68" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+    <polygon fill="white" stroke="transparent" points="-4,4 -4,-4896.91 4846.02,-4896.91 4846.02,4 -4,4" /> 
+    <text text-anchor="start" x="4573.52" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
      generated by
     </text> 
-    <text text-anchor="start" x="4673.68" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+    <text text-anchor="start" x="4656.52" y="-29.8" font-family="Helvetica,sans-Serif" font-size="14.00">
      SchemaCrawler 16.10.1
     </text> 
-    <text text-anchor="start" x="4589.68" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+    <text text-anchor="start" x="4572.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
      generated on
     </text> 
-    <text text-anchor="start" x="4673.68" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2021-03-24 01:26:58.684653
+    <text text-anchor="start" x="4656.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
+     2021-04-12 17:28:07.032036
     </text> 
-    <polygon fill="none" stroke="#888888" points="4586.18,-4 4586.18,-44 4851.18,-44 4851.18,-4 4586.18,-4" /> <!-- allocationtoken_a08ccbef --> 
+    <polygon fill="none" stroke="#888888" points="4569.02,-4 4569.02,-44 4834.02,-44 4834.02,-4 4569.02,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 
      <title>allocationtoken_a08ccbef</title> 
      <polygon fill="#ebcef2" stroke="transparent" points="3001.5,-2741.91 3001.5,-2760.91 3200.5,-2760.91 3200.5,-2741.91 3001.5,-2741.91" /> 
@@ -5985,142 +5985,142 @@ td.section {
     </g> <!-- registrylock_ac88663e --> 
     <g id="node29" class="node"> 
      <title>registrylock_ac88663e</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4491.5,-4010.91 4491.5,-4029.91 4687.5,-4029.91 4687.5,-4010.91 4491.5,-4010.91" /> 
-     <text text-anchor="start" x="4493.5" y="-4017.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4510.5,-4010.91 4510.5,-4029.91 4669.5,-4029.91 4669.5,-4010.91 4510.5,-4010.91" /> 
+     <text text-anchor="start" x="4512.5" y="-4017.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.RegistryLock
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4687.5,-4010.91 4687.5,-4029.91 4813.5,-4029.91 4813.5,-4010.91 4687.5,-4010.91" /> 
-     <text text-anchor="start" x="4774.5" y="-4016.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4669.5,-4010.91 4669.5,-4029.91 4795.5,-4029.91 4795.5,-4010.91 4669.5,-4010.91" /> 
+     <text text-anchor="start" x="4756.5" y="-4016.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3998.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3998.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       revision_id
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3997.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       bigserial not null
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3978.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       auto-incremented
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      lock_completion_timestamp
+     <text text-anchor="start" x="4512.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      lock_completion_time
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3959.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      lock_request_timestamp
+     <text text-anchor="start" x="4512.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      lock_request_time
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3940.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz not null
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       domain_name
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3921.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       is_superuser
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3902.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       bool not null
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       registrar_id
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3883.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       registrar_poc_id
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3864.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       repo_id
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3845.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       verification_code
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3826.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      unlock_request_timestamp
+     <text text-anchor="start" x="4512.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      unlock_request_time
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3807.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      unlock_completion_timestamp
+     <text text-anchor="start" x="4512.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      unlock_completion_time
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3788.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       last_update_time
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz not null
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       relock_revision_id
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       int8
      </text> 
-     <text text-anchor="start" x="4493.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4512.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       relock_duration
      </text> 
-     <text text-anchor="start" x="4681.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4663.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4689.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4671.5" y="-3731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       interval
      </text> 
-     <polygon fill="none" stroke="#888888" points="4490.5,-3724.91 4490.5,-4030.91 4814.5,-4030.91 4814.5,-3724.91 4490.5,-3724.91" /> 
+     <polygon fill="none" stroke="#888888" points="4509,-3724.91 4509,-4030.91 4796,-4030.91 4796,-3724.91 4509,-3724.91" /> 
     </g> <!-- registrylock_ac88663e&#45;&gt;registrylock_ac88663e --> 
     <g id="edge72" class="edge"> 
      <title>registrylock_ac88663e:w-&gt;registrylock_ac88663e:e</title> 
-     <path fill="none" stroke="black" d="M4478.18,-3766.88C4416.43,-3841.38 4430.31,-4052.91 4652.5,-4052.91 4879.78,-4052.91 4889.1,-4041.87 4822.46,-4006.57" /> 
-     <polygon fill="black" stroke="black" points="4484.34,-3760.88 4494.64,-3757.13 4487.92,-3757.39 4491.5,-3753.91 4491.5,-3753.91 4491.5,-3753.91 4487.92,-3757.39 4488.36,-3750.68 4484.34,-3760.88 4484.34,-3760.88" /> 
-     <ellipse fill="none" stroke="black" cx="4481.47" cy="-3763.67" rx="4" ry="4" /> 
-     <polygon fill="black" stroke="black" points="4812.08,-4006.8 4816.7,-3997.93 4818.47,-3998.86 4813.85,-4007.73 4812.08,-4006.8" /> 
-     <polyline fill="none" stroke="black" points="4813.5,-4001.91 4817.93,-4004.22 " /> 
-     <polygon fill="black" stroke="black" points="4816.51,-4009.11 4821.13,-4000.24 4822.9,-4001.17 4818.29,-4010.04 4816.51,-4009.11" /> 
-     <polyline fill="none" stroke="black" points="4817.93,-4004.22 4822.37,-4006.52 " /> 
+     <path fill="none" stroke="black" d="M4497.18,-3766.88C4435.35,-3841.38 4448.16,-4052.91 4653,-4052.91 4862.54,-4052.91 4871.13,-4041.87 4804.46,-4006.57" /> 
+     <polygon fill="black" stroke="black" points="4503.33,-3760.88 4513.64,-3757.13 4506.92,-3757.39 4510.5,-3753.91 4510.5,-3753.91 4510.5,-3753.91 4506.92,-3757.39 4507.36,-3750.68 4503.33,-3760.88 4503.33,-3760.88" /> 
+     <ellipse fill="none" stroke="black" cx="4500.47" cy="-3763.67" rx="4" ry="4" /> 
+     <polygon fill="black" stroke="black" points="4794.08,-4006.8 4798.7,-3997.93 4800.47,-3998.86 4795.85,-4007.73 4794.08,-4006.8" /> 
+     <polyline fill="none" stroke="black" points="4795.5,-4001.91 4799.94,-4004.21 " /> 
+     <polygon fill="black" stroke="black" points="4798.51,-4009.11 4803.13,-4000.24 4804.9,-4001.17 4800.29,-4010.04 4798.51,-4009.11" /> 
+     <polyline fill="none" stroke="black" points="4799.94,-4004.21 4804.37,-4006.52 " /> 
      <text text-anchor="start" x="4571" y="-4056.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       fk2lhcwpxlnqijr96irylrh1707
      </text> 
@@ -6233,139 +6233,155 @@ td.section {
     </g> <!-- serversecret_6cc90f09 --> 
     <g id="node32" class="node"> 
      <title>serversecret_6cc90f09</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4548.5,-4233.91 4548.5,-4252.91 4679.5,-4252.91 4679.5,-4233.91 4548.5,-4233.91" /> 
-     <text text-anchor="start" x="4550.5" y="-4240.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4548.5,-4251.91 4548.5,-4270.91 4679.5,-4270.91 4679.5,-4251.91 4548.5,-4251.91" /> 
+     <text text-anchor="start" x="4550.5" y="-4258.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.ServerSecret
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4679.5,-4233.91 4679.5,-4252.91 4756.5,-4252.91 4756.5,-4233.91 4679.5,-4233.91" /> 
-     <text text-anchor="start" x="4717.5" y="-4239.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4679.5,-4251.91 4679.5,-4270.91 4756.5,-4270.91 4756.5,-4251.91 4679.5,-4251.91" /> 
+     <text text-anchor="start" x="4717.5" y="-4257.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="4550.5" y="-4221.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4550.5" y="-4238.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       secret
      </text> 
-     <text text-anchor="start" x="4633.5" y="-4220.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4632.5" y="-4238.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4681.5" y="-4220.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4681.5" y="-4238.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       uuid not null
      </text> 
-     <polygon fill="none" stroke="#888888" points="4547.5,-4213.91 4547.5,-4253.91 4757.5,-4253.91 4757.5,-4213.91 4547.5,-4213.91" /> 
+     <text text-anchor="start" x="4550.5" y="-4220.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+      id
+     </text> 
+     <text text-anchor="start" x="4632.5" y="-4219.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4681.5" y="-4219.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      int8 not null
+     </text> 
+     <polygon fill="none" stroke="#888888" points="4547.5,-4213.41 4547.5,-4272.41 4757.5,-4272.41 4757.5,-4213.41 4547.5,-4213.41" /> 
     </g> <!-- signedmarkrevocationentry_99c39721 --> 
     <g id="node33" class="node"> 
      <title>signedmarkrevocationentry_99c39721</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4473.5,-4337.91 4473.5,-4356.91 4706.5,-4356.91 4706.5,-4337.91 4473.5,-4337.91" /> 
-     <text text-anchor="start" x="4475.5" y="-4344.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4473.5,-4356.91 4473.5,-4375.91 4706.5,-4375.91 4706.5,-4356.91 4473.5,-4356.91" /> 
+     <text text-anchor="start" x="4475.5" y="-4363.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.SignedMarkRevocationEntry
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4706.5,-4337.91 4706.5,-4356.91 4832.5,-4356.91 4832.5,-4337.91 4706.5,-4337.91" /> 
-     <text text-anchor="start" x="4793.5" y="-4343.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4706.5,-4356.91 4706.5,-4375.91 4832.5,-4375.91 4832.5,-4356.91 4706.5,-4356.91" /> 
+     <text text-anchor="start" x="4793.5" y="-4362.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="4475.5" y="-4325.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4475.5" y="-4344.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       revision_id
      </text> 
+     <text text-anchor="start" x="4638.5" y="-4343.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4708.5" y="-4343.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      int8 not null
+     </text> 
+     <text text-anchor="start" x="4475.5" y="-4324.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      revocation_time
+     </text> 
      <text text-anchor="start" x="4638.5" y="-4324.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4708.5" y="-4324.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      int8 not null
+      timestamptz not null
      </text> 
-     <text text-anchor="start" x="4475.5" y="-4305.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      revocation_time
+     <text text-anchor="start" x="4475.5" y="-4306.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+      smd_id
      </text> 
      <text text-anchor="start" x="4638.5" y="-4305.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4708.5" y="-4305.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      timestamptz not null
-     </text> 
-     <text text-anchor="start" x="4475.5" y="-4287.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
-      smd_id
-     </text> 
-     <text text-anchor="start" x="4638.5" y="-4286.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4708.5" y="-4286.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <polygon fill="none" stroke="#888888" points="4472,-4279.91 4472,-4357.91 4833,-4357.91 4833,-4279.91 4472,-4279.91" /> 
+     <polygon fill="none" stroke="#888888" points="4472,-4298.91 4472,-4376.91 4833,-4376.91 4833,-4298.91 4472,-4298.91" /> 
     </g> <!-- signedmarkrevocationlist_c5d968fb --> 
     <g id="node34" class="node"> 
      <title>signedmarkrevocationlist_c5d968fb</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="3827,-4337.91 3827,-4356.91 4049,-4356.91 4049,-4337.91 3827,-4337.91" /> 
-     <text text-anchor="start" x="3829" y="-4344.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="3827,-4356.91 3827,-4375.91 4049,-4375.91 4049,-4356.91 3827,-4356.91" /> 
+     <text text-anchor="start" x="3829" y="-4363.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.SignedMarkRevocationList
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4049,-4337.91 4049,-4356.91 4159,-4356.91 4159,-4337.91 4049,-4337.91" /> 
-     <text text-anchor="start" x="4120" y="-4343.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4049,-4356.91 4049,-4375.91 4159,-4375.91 4159,-4356.91 4049,-4356.91" /> 
+     <text text-anchor="start" x="4120" y="-4362.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="3829" y="-4325.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="3829" y="-4344.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       revision_id
      </text> 
+     <text text-anchor="start" x="3979" y="-4343.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4051" y="-4343.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      bigserial not null
+     </text> 
      <text text-anchor="start" x="3979" y="-4324.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4051" y="-4324.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      bigserial not null
+      auto-incremented
+     </text> 
+     <text text-anchor="start" x="3829" y="-4305.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      creation_time
      </text> 
      <text text-anchor="start" x="3979" y="-4305.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4051" y="-4305.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      auto-incremented
-     </text> 
-     <text text-anchor="start" x="3829" y="-4286.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      creation_time
-     </text> 
-     <text text-anchor="start" x="3979" y="-4286.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4051" y="-4286.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz
      </text> 
-     <polygon fill="none" stroke="#888888" points="3826,-4279.91 3826,-4357.91 4160,-4357.91 4160,-4279.91 3826,-4279.91" /> 
+     <polygon fill="none" stroke="#888888" points="3826,-4298.91 3826,-4376.91 4160,-4376.91 4160,-4298.91 3826,-4298.91" /> 
     </g> <!-- signedmarkrevocationentry_99c39721&#45;&gt;signedmarkrevocationlist_c5d968fb --> 
     <g id="edge74" class="edge"> 
      <title>signedmarkrevocationentry_99c39721:w-&gt;signedmarkrevocationlist_c5d968fb:e</title> 
-     <path fill="none" stroke="black" d="M4454.39,-4328.91C4333.55,-4328.91 4294.69,-4328.91 4170.3,-4328.91" /> 
-     <polygon fill="black" stroke="black" points="4462.5,-4328.91 4472.5,-4333.41 4467.5,-4328.91 4472.5,-4328.91 4472.5,-4328.91 4472.5,-4328.91 4467.5,-4328.91 4472.5,-4324.41 4462.5,-4328.91 4462.5,-4328.91" /> 
-     <ellipse fill="none" stroke="black" cx="4458.5" cy="-4328.91" rx="4" ry="4" /> 
-     <polygon fill="black" stroke="black" points="4161,-4333.91 4161,-4323.91 4163,-4323.91 4163,-4333.91 4161,-4333.91" /> 
-     <polyline fill="none" stroke="black" points="4160,-4328.91 4165,-4328.91 " /> 
-     <polygon fill="black" stroke="black" points="4166,-4333.91 4166,-4323.91 4168,-4323.91 4168,-4333.91 4166,-4333.91" /> 
-     <polyline fill="none" stroke="black" points="4165,-4328.91 4170,-4328.91 " /> 
-     <text text-anchor="start" x="4264" y="-4332.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <path fill="none" stroke="black" d="M4454.39,-4347.91C4333.55,-4347.91 4294.69,-4347.91 4170.3,-4347.91" /> 
+     <polygon fill="black" stroke="black" points="4462.5,-4347.91 4472.5,-4352.41 4467.5,-4347.91 4472.5,-4347.91 4472.5,-4347.91 4472.5,-4347.91 4467.5,-4347.91 4472.5,-4343.41 4462.5,-4347.91 4462.5,-4347.91" /> 
+     <ellipse fill="none" stroke="black" cx="4458.5" cy="-4347.91" rx="4" ry="4" /> 
+     <polygon fill="black" stroke="black" points="4161,-4352.91 4161,-4342.91 4163,-4342.91 4163,-4352.91 4161,-4352.91" /> 
+     <polyline fill="none" stroke="black" points="4160,-4347.91 4165,-4347.91 " /> 
+     <polygon fill="black" stroke="black" points="4166,-4352.91 4166,-4342.91 4168,-4342.91 4168,-4352.91 4166,-4352.91" /> 
+     <polyline fill="none" stroke="black" points="4165,-4347.91 4170,-4347.91 " /> 
+     <text text-anchor="start" x="4264" y="-4351.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       fk5ivlhvs3121yx2li5tqh54u4
      </text> 
     </g> <!-- spec11threatmatch_a61228a6 --> 
     <g id="node35" class="node"> 
      <title>spec11threatmatch_a61228a6</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4509.5,-4536.91 4509.5,-4555.91 4685.5,-4555.91 4685.5,-4536.91 4509.5,-4536.91" /> 
-     <text text-anchor="start" x="4511.5" y="-4543.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4509.5,-4555.91 4509.5,-4574.91 4685.5,-4574.91 4685.5,-4555.91 4509.5,-4555.91" /> 
+     <text text-anchor="start" x="4511.5" y="-4562.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.Spec11ThreatMatch
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4685.5,-4536.91 4685.5,-4555.91 4795.5,-4555.91 4795.5,-4536.91 4685.5,-4536.91" /> 
-     <text text-anchor="start" x="4756.5" y="-4542.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4685.5,-4555.91 4685.5,-4574.91 4795.5,-4574.91 4795.5,-4555.91 4685.5,-4555.91" /> 
+     <text text-anchor="start" x="4756.5" y="-4561.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="4511.5" y="-4524.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4511.5" y="-4543.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       id
      </text> 
+     <text text-anchor="start" x="4646.5" y="-4542.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4687.5" y="-4542.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      bigserial not null
+     </text> 
      <text text-anchor="start" x="4646.5" y="-4523.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4687.5" y="-4523.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      bigserial not null
+      auto-incremented
+     </text> 
+     <text text-anchor="start" x="4511.5" y="-4504.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      check_date
      </text> 
      <text text-anchor="start" x="4646.5" y="-4504.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4687.5" y="-4504.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      auto-incremented
+      date not null
      </text> 
      <text text-anchor="start" x="4511.5" y="-4485.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      check_date
+      domain_name
      </text> 
      <text text-anchor="start" x="4646.5" y="-4485.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4687.5" y="-4485.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      date not null
+      text not null
      </text> 
      <text text-anchor="start" x="4511.5" y="-4466.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      domain_name
+      domain_repo_id
      </text> 
      <text text-anchor="start" x="4646.5" y="-4466.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
@@ -6373,7 +6389,7 @@ td.section {
       text not null
      </text> 
      <text text-anchor="start" x="4511.5" y="-4447.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      domain_repo_id
+      registrar_id
      </text> 
      <text text-anchor="start" x="4646.5" y="-4447.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
@@ -6381,127 +6397,127 @@ td.section {
       text not null
      </text> 
      <text text-anchor="start" x="4511.5" y="-4428.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      registrar_id
+      threat_types
      </text> 
      <text text-anchor="start" x="4646.5" y="-4428.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4687.5" y="-4428.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      text not null
+      _text not null
      </text> 
      <text text-anchor="start" x="4511.5" y="-4409.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      threat_types
+      tld
      </text> 
      <text text-anchor="start" x="4646.5" y="-4409.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4687.5" y="-4409.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      _text not null
-     </text> 
-     <text text-anchor="start" x="4511.5" y="-4390.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      tld
-     </text> 
-     <text text-anchor="start" x="4646.5" y="-4390.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4687.5" y="-4390.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <polygon fill="none" stroke="#888888" points="4508.5,-4384.41 4508.5,-4557.41 4796.5,-4557.41 4796.5,-4384.41 4508.5,-4384.41" /> 
+     <polygon fill="none" stroke="#888888" points="4508.5,-4403.41 4508.5,-4576.41 4796.5,-4576.41 4796.5,-4403.41 4508.5,-4403.41" /> 
     </g> <!-- sqlreplaycheckpoint_342081b3 --> 
     <g id="node36" class="node"> 
      <title>sqlreplaycheckpoint_342081b3</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4496.5,-4621.91 4496.5,-4640.91 4683.5,-4640.91 4683.5,-4621.91 4496.5,-4621.91" /> 
-     <text text-anchor="start" x="4498.5" y="-4628.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4496.5,-4640.91 4496.5,-4659.91 4683.5,-4659.91 4683.5,-4640.91 4496.5,-4640.91" /> 
+     <text text-anchor="start" x="4498.5" y="-4647.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.SqlReplayCheckpoint
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4683.5,-4621.91 4683.5,-4640.91 4809.5,-4640.91 4809.5,-4621.91 4683.5,-4621.91" /> 
-     <text text-anchor="start" x="4770.5" y="-4627.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4683.5,-4640.91 4683.5,-4659.91 4809.5,-4659.91 4809.5,-4640.91 4683.5,-4640.91" /> 
+     <text text-anchor="start" x="4770.5" y="-4646.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="4498.5" y="-4609.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4498.5" y="-4628.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       revision_id
      </text> 
+     <text text-anchor="start" x="4639.5" y="-4627.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4685.5" y="-4627.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      int8 not null
+     </text> 
+     <text text-anchor="start" x="4498.5" y="-4608.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      last_replay_time
+     </text> 
      <text text-anchor="start" x="4639.5" y="-4608.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4685.5" y="-4608.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      int8 not null
-     </text> 
-     <text text-anchor="start" x="4498.5" y="-4589.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      last_replay_time
-     </text> 
-     <text text-anchor="start" x="4639.5" y="-4589.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4685.5" y="-4589.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz not null
      </text> 
-     <polygon fill="none" stroke="#888888" points="4495,-4583.41 4495,-4642.41 4810,-4642.41 4810,-4583.41 4495,-4583.41" /> 
+     <polygon fill="none" stroke="#888888" points="4495,-4602.41 4495,-4661.41 4810,-4661.41 4810,-4602.41 4495,-4602.41" /> 
     </g> <!-- tmchcrl_d282355 --> 
     <g id="node38" class="node"> 
      <title>tmchcrl_d282355</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4509.5,-4726.91 4509.5,-4745.91 4669.5,-4745.91 4669.5,-4726.91 4509.5,-4726.91" /> 
-     <text text-anchor="start" x="4511.5" y="-4733.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4516.5,-4763.91 4516.5,-4782.91 4663.5,-4782.91 4663.5,-4763.91 4516.5,-4763.91" /> 
+     <text text-anchor="start" x="4518.5" y="-4770.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.TmchCrl
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4669.5,-4726.91 4669.5,-4745.91 4795.5,-4745.91 4795.5,-4726.91 4669.5,-4726.91" /> 
-     <text text-anchor="start" x="4756.5" y="-4732.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4663.5,-4763.91 4663.5,-4782.91 4789.5,-4782.91 4789.5,-4763.91 4663.5,-4763.91" /> 
+     <text text-anchor="start" x="4750.5" y="-4769.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="4511.5" y="-4714.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4518.5" y="-4750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       certificate_revocations
      </text> 
-     <text text-anchor="start" x="4663.5" y="-4713.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4657.5" y="-4750.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4671.5" y="-4713.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4665.5" y="-4750.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <text text-anchor="start" x="4511.5" y="-4695.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4518.5" y="-4731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       update_timestamp
      </text> 
-     <text text-anchor="start" x="4663.5" y="-4694.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4657.5" y="-4731.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4671.5" y="-4694.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4665.5" y="-4731.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       timestamptz not null
      </text> 
-     <text text-anchor="start" x="4511.5" y="-4676.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4518.5" y="-4712.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       url
      </text> 
-     <text text-anchor="start" x="4663.5" y="-4675.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     <text text-anchor="start" x="4657.5" y="-4712.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
-     <text text-anchor="start" x="4671.5" y="-4675.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <text text-anchor="start" x="4665.5" y="-4712.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       text not null
      </text> 
-     <polygon fill="none" stroke="#888888" points="4508.5,-4668.91 4508.5,-4746.91 4796.5,-4746.91 4796.5,-4668.91 4508.5,-4668.91" /> 
+     <text text-anchor="start" x="4518.5" y="-4694.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+      id
+     </text> 
+     <text text-anchor="start" x="4657.5" y="-4693.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4665.5" y="-4693.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      int8 not null
+     </text> 
+     <polygon fill="none" stroke="#888888" points="4515,-4687.41 4515,-4784.41 4790,-4784.41 4790,-4687.41 4515,-4687.41" /> 
     </g> <!-- transaction_d50389d4 --> 
     <g id="node39" class="node"> 
      <title>transaction_d50389d4</title> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4535.5,-4830.91 4535.5,-4849.91 4660.5,-4849.91 4660.5,-4830.91 4535.5,-4830.91" /> 
-     <text text-anchor="start" x="4537.5" y="-4837.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4535.5,-4868.91 4535.5,-4887.91 4660.5,-4887.91 4660.5,-4868.91 4535.5,-4868.91" /> 
+     <text text-anchor="start" x="4537.5" y="-4875.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       public.Transaction
      </text> 
-     <polygon fill="#ebcef2" stroke="transparent" points="4660.5,-4830.91 4660.5,-4849.91 4770.5,-4849.91 4770.5,-4830.91 4660.5,-4830.91" /> 
-     <text text-anchor="start" x="4731.5" y="-4836.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+     <polygon fill="#ebcef2" stroke="transparent" points="4660.5,-4868.91 4660.5,-4887.91 4770.5,-4887.91 4770.5,-4868.91 4660.5,-4868.91" /> 
+     <text text-anchor="start" x="4731.5" y="-4874.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       [table]
      </text> 
-     <text text-anchor="start" x="4537.5" y="-4818.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
+     <text text-anchor="start" x="4537.5" y="-4856.71" font-family="Helvetica,sans-Serif" font-weight="bold" font-style="italic" font-size="14.00">
       id
      </text> 
+     <text text-anchor="start" x="4623.5" y="-4855.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4662.5" y="-4855.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      bigserial not null
+     </text> 
+     <text text-anchor="start" x="4623.5" y="-4836.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
+     </text> 
+     <text text-anchor="start" x="4662.5" y="-4836.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      auto-incremented
+     </text> 
+     <text text-anchor="start" x="4537.5" y="-4817.71" font-family="Helvetica,sans-Serif" font-size="14.00">
+      contents
+     </text> 
      <text text-anchor="start" x="4623.5" y="-4817.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
      </text> 
      <text text-anchor="start" x="4662.5" y="-4817.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      bigserial not null
-     </text> 
-     <text text-anchor="start" x="4623.5" y="-4798.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4662.5" y="-4798.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      auto-incremented
-     </text> 
-     <text text-anchor="start" x="4537.5" y="-4779.71" font-family="Helvetica,sans-Serif" font-size="14.00">
-      contents
-     </text> 
-     <text text-anchor="start" x="4623.5" y="-4779.71" font-family="Helvetica,sans-Serif" font-size="14.00"> 
-     </text> 
-     <text text-anchor="start" x="4662.5" y="-4779.71" font-family="Helvetica,sans-Serif" font-size="14.00">
       bytea
      </text> 
-     <polygon fill="none" stroke="#888888" points="4534,-4772.91 4534,-4850.91 4771,-4850.91 4771,-4772.91 4534,-4772.91" /> 
+     <polygon fill="none" stroke="#888888" points="4534,-4810.91 4534,-4888.91 4771,-4888.91 4771,-4810.91 4534,-4810.91" /> 
     </g> 
    </g> 
   </svg> 
@@ -12706,12 +12722,12 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">lock_completion_timestamp</td> 
+     <td class="minwidth">lock_completion_time</td> 
      <td class="minwidth">timestamptz</td> 
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">lock_request_timestamp</td> 
+     <td class="minwidth">lock_request_time</td> 
      <td class="minwidth">timestamptz not null</td> 
     </tr> 
     <tr> 
@@ -12746,12 +12762,12 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">unlock_request_timestamp</td> 
+     <td class="minwidth">unlock_request_time</td> 
      <td class="minwidth">timestamptz</td> 
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">unlock_completion_timestamp</td> 
+     <td class="minwidth">unlock_completion_time</td> 
      <td class="minwidth">timestamptz</td> 
     </tr> 
     <tr> 
@@ -13062,9 +13078,14 @@ td.section {
    <tbody>
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth"><b><i>secret</i></b></td> 
+     <td class="minwidth">secret</td> 
      <td class="minwidth">uuid not null</td> 
     </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth"><b><i>id</i></b></td> 
+     <td class="minwidth">int8 not null</td> 
+    </tr> 
     <tr> 
      <td colspan="3"></td> 
     </tr> 
@@ -13080,7 +13101,7 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">secret</td> 
+     <td class="minwidth">id</td> 
      <td class="minwidth"></td> 
     </tr> 
     <tr> 
@@ -13098,7 +13119,7 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">secret</td> 
+     <td class="minwidth">id</td> 
      <td class="minwidth">ascending</td> 
     </tr> 
    </tbody>
@@ -13707,19 +13728,24 @@ td.section {
    <tbody>
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth"><b><i>certificate_revocations</i></b></td> 
+     <td class="minwidth">certificate_revocations</td> 
      <td class="minwidth">text not null</td> 
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth"><b><i>update_timestamp</i></b></td> 
+     <td class="minwidth">update_timestamp</td> 
      <td class="minwidth">timestamptz not null</td> 
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth"><b><i>url</i></b></td> 
+     <td class="minwidth">url</td> 
      <td class="minwidth">text not null</td> 
     </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth"><b><i>id</i></b></td> 
+     <td class="minwidth">int8 not null</td> 
+    </tr> 
     <tr> 
      <td colspan="3"></td> 
     </tr> 
@@ -13735,17 +13761,7 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">certificate_revocations</td> 
-     <td class="minwidth"></td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth">update_timestamp</td> 
-     <td class="minwidth"></td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth">url</td> 
+     <td class="minwidth">id</td> 
      <td class="minwidth"></td> 
     </tr> 
     <tr> 
@@ -13763,17 +13779,7 @@ td.section {
     </tr> 
     <tr> 
      <td class="spacer"></td> 
-     <td class="minwidth">certificate_revocations</td> 
-     <td class="minwidth">ascending</td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth">update_timestamp</td> 
-     <td class="minwidth">ascending</td> 
-    </tr> 
-    <tr> 
-     <td class="spacer"></td> 
-     <td class="minwidth">url</td> 
+     <td class="minwidth">id</td> 
      <td class="minwidth">ascending</td> 
     </tr> 
    </tbody>

db/src/main/resources/sql/flyway.txt

@@ -88,3 +88,5 @@ V87__fix_super_domain_fk.sql
 V88__transfer_billing_cancellation_history_id.sql
 V89__host_history_host_deferred.sql
 V90__update_timestamp.sql
+V91__defer_fkeys.sql
+V92__singletons.sql

db/src/main/resources/sql/flyway/V91__defer_fkeys.sql

@@ -0,0 +1,48 @@
+-- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- We require that *History table writes come after their corresponding
+-- EppResource writes when replaying transactions from Datastore.
+-- The alterations here serve to break cycles necessary to write the
+-- resource first.
+
+ALTER TABLE "BillingEvent" DROP CONSTRAINT fk_billing_event_domain_history;
+ALTER TABLE "BillingEvent" ADD CONSTRAINT fk_billing_event_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "BillingRecurrence" DROP CONSTRAINT fk_billing_recurrence_domain_history;
+ALTER TABLE "BillingRecurrence" ADD CONSTRAINT fk_billing_recurrence_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "BillingCancellation" DROP CONSTRAINT fk_billing_cancellation_domain_history;
+ALTER TABLE "BillingCancellation" ADD CONSTRAINT fk_billing_cancellation_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "PollMessage" DROP CONSTRAINT fk_poll_message_domain_history;
+ALTER TABLE "PollMessage" ADD CONSTRAINT fk_poll_message_domain_history
+   FOREIGN KEY (domain_repo_id, domain_history_revision_id)
+   REFERENCES "DomainHistory"(domain_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "PollMessage" DROP CONSTRAINT fk_poll_message_contact_history;
+ALTER TABLE "PollMessage" ADD CONSTRAINT fk_poll_message_contact_history
+   FOREIGN KEY (contact_repo_id, contact_history_revision_id)
+   REFERENCES "ContactHistory"(contact_repo_id, history_revision_id)
+   DEFERRABLE INITIALLY DEFERRED;

db/src/main/resources/sql/flyway/V92__singletons.sql

@@ -0,0 +1,21 @@
+-- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+ALTER TABLE "ServerSecret" DROP CONSTRAINT "ServerSecret_pkey";
+ALTER TABLE "ServerSecret" ADD COLUMN id bigint NOT NULL;
+ALTER TABLE "ServerSecret" ADD PRIMARY KEY(id);
+
+ALTER TABLE "TmchCrl" DROP CONSTRAINT "TmchCrl_pkey";
+ALTER TABLE "TmchCrl" ADD COLUMN id bigint NOT NULL;
+ALTER TABLE "TmchCrl" ADD PRIMARY KEY(id);

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -665,8 +665,9 @@
     );
 
     create table "ServerSecret" (
-       secret uuid not null,
-        primary key (secret)
+       id int8 not null,
+        secret uuid,
+        primary key (id)
     );
 
     create table "SignedMarkRevocationEntry" (
@@ -742,10 +743,11 @@
     );
 
     create table "TmchCrl" (
-       certificate_revocations text not null,
+       id int8 not null,
+        certificate_revocations text not null,
         update_timestamp timestamptz not null,
         url text not null,
-        primary key (certificate_revocations, update_timestamp, url)
+        primary key (id)
     );
 
     create table "Transaction" (

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -948,7 +948,8 @@ ALTER SEQUENCE public."SafeBrowsingThreat_id_seq" OWNED BY public."Spec11ThreatM
 --
 
 CREATE TABLE public."ServerSecret" (
-    secret uuid NOT NULL
+    secret uuid NOT NULL,
+    id bigint NOT NULL
 );
 
 
@@ -1055,7 +1056,8 @@ CREATE TABLE public."Tld" (
 CREATE TABLE public."TmchCrl" (
     certificate_revocations text NOT NULL,
     update_timestamp timestamp with time zone NOT NULL,
-    url text NOT NULL
+    url text NOT NULL,
+    id bigint NOT NULL
 );
 
 
@@ -1389,7 +1391,7 @@ ALTER TABLE ONLY public."Spec11ThreatMatch"
 --
 
 ALTER TABLE ONLY public."ServerSecret"
-    ADD CONSTRAINT "ServerSecret_pkey" PRIMARY KEY (secret);
+    ADD CONSTRAINT "ServerSecret_pkey" PRIMARY KEY (id);
 
 
 --
@@ -1429,7 +1431,7 @@ ALTER TABLE ONLY public."Tld"
 --
 
 ALTER TABLE ONLY public."TmchCrl"
-    ADD CONSTRAINT "TmchCrl_pkey" PRIMARY KEY (certificate_revocations, update_timestamp, url);
+    ADD CONSTRAINT "TmchCrl_pkey" PRIMARY KEY (id);
 
 
 --
@@ -1920,7 +1922,7 @@ ALTER TABLE ONLY public."BillingCancellation"
 --
 
 ALTER TABLE ONLY public."BillingCancellation"
-    ADD CONSTRAINT fk_billing_cancellation_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_billing_cancellation_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -1952,7 +1954,7 @@ ALTER TABLE ONLY public."BillingEvent"
 --
 
 ALTER TABLE ONLY public."BillingEvent"
-    ADD CONSTRAINT fk_billing_event_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_billing_event_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -1968,7 +1970,7 @@ ALTER TABLE ONLY public."BillingEvent"
 --
 
 ALTER TABLE ONLY public."BillingRecurrence"
-    ADD CONSTRAINT fk_billing_recurrence_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_billing_recurrence_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -2216,7 +2218,7 @@ ALTER TABLE ONLY public."HostHistory"
 --
 
 ALTER TABLE ONLY public."PollMessage"
-    ADD CONSTRAINT fk_poll_message_contact_history FOREIGN KEY (contact_repo_id, contact_history_revision_id) REFERENCES public."ContactHistory"(contact_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_poll_message_contact_history FOREIGN KEY (contact_repo_id, contact_history_revision_id) REFERENCES public."ContactHistory"(contact_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;
 
 
 --
@@ -2232,7 +2234,7 @@ ALTER TABLE ONLY public."PollMessage"
 --
 
 ALTER TABLE ONLY public."PollMessage"
-    ADD CONSTRAINT fk_poll_message_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id);
+    ADD CONSTRAINT fk_poll_message_domain_history FOREIGN KEY (domain_repo_id, domain_history_revision_id) REFERENCES public."DomainHistory"(domain_repo_id, history_revision_id) DEFERRABLE INITIALLY DEFERRED;
 
 
 --

dependencies.gradle

@@ -167,11 +167,11 @@ ext {
       'org.seleniumhq.selenium:selenium-java:3.141.59',
       'org.seleniumhq.selenium:selenium-remote-driver:3.141.59',
       'org.slf4j:slf4j-jdk14:1.7.28',
-      'org.testcontainers:jdbc:1.15.1',
-      'org.testcontainers:junit-jupiter:1.15.1',
-      'org.testcontainers:postgresql:1.15.1',
-      'org.testcontainers:selenium:1.15.1',
-      'org.testcontainers:testcontainers:1.15.1',
+      'org.testcontainers:jdbc:1.15.2',
+      'org.testcontainers:junit-jupiter:1.15.2',
+      'org.testcontainers:postgresql:1.15.2',
+      'org.testcontainers:selenium:1.15.2',
+      'org.testcontainers:testcontainers:1.15.2',
       'org.yaml:snakeyaml:1.17',
       'us.fatehi:schemacrawler:16.10.1',
       'us.fatehi:schemacrawler-api:16.10.1',

docs/gradle/dependency-locks/compile.lockfile

@@ -267,10 +267,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

docs/gradle/dependency-locks/compileClasspath.lockfile

@@ -255,10 +255,10 @@ org.rnorth.visible-assertions:visible-assertions:2.1.2
 org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3

docs/gradle/dependency-locks/default.lockfile

@@ -267,10 +267,10 @@ org.slf4j:jcl-over-slf4j:1.7.30
 org.slf4j:jul-to-slf4j:1.7.30
 org.slf4j:slf4j-api:1.7.30
 org.slf4j:slf4j-jdk14:1.7.28
-org.testcontainers:database-commons:1.15.1
-org.testcontainers:jdbc:1.15.1
-org.testcontainers:postgresql:1.15.1
-org.testcontainers:testcontainers:1.15.1
+org.testcontainers:database-commons:1.15.2
+org.testcontainers:jdbc:1.15.2
+org.testcontainers:postgresql:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.0
 org.tukaani:xz:1.5
 org.w3c.css:sac:1.3