Remove a couple unused variables (#913)

* Remove a couple unused variables

config/presubmits.py

@@ -65,7 +65,7 @@ class PresubmitCheck:
     for pattern in self.skipped_patterns:
       if pattern in file:
         return False
-    with open(file, "r") as f:
+    with open(file, "r", encoding='utf8') as f:
       file_content = f.read()
       matches = re.match(self.regex, file_content, re.DOTALL)
       if self.regex_type == FORBIDDEN:
@@ -241,7 +241,7 @@ def verify_flyway_index():
 
   # Remove the sequence numbers and compare against the index file contents.
   files = [filename[1] for filename in sorted(files)]
-  with open('db/src/main/resources/sql/flyway.txt') as index:
+  with open('db/src/main/resources/sql/flyway.txt', encoding='utf8') as index:
     indexed_files = index.read().splitlines()
   if files != indexed_files:
     unindexed = set(files) - set(indexed_files)

core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java

@@ -14,6 +14,7 @@
 
 package google.registry.batch;
 
+import static google.registry.mapreduce.MapreduceRunner.PARAM_FAST;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
@@ -24,6 +25,7 @@ import google.registry.mapreduce.MapreduceRunner;
 import google.registry.mapreduce.inputs.EppResourceInputs;
 import google.registry.model.EppResource;
 import google.registry.request.Action;
+import google.registry.request.Parameter;
 import google.registry.request.Response;
 import google.registry.request.auth.Auth;
 import javax.inject.Inject;
@@ -39,6 +41,14 @@ import javax.inject.Inject;
  * <p>Because there are no auth settings in the {@link Action} annotation, this command can only be
  * run internally, or by pretending to be internal by setting the X-AppEngine-QueueName header,
  * which only admin users can do.
+ *
+ * <p>If the <code>?fast=true</code> querystring parameter is passed, then entities that are not
+ * changed by {@link EppResource#cloneProjectedAtTime} will not be re-saved. This helps prevent
+ * mutation load on the DB and has the beneficial side effect of writing out smaller commit logs.
+ * Note that this does NOT pick up mutations caused by migrations using the {@link
+ * com.googlecode.objectify.annotation.OnLoad} annotation, so if you are running a one-off schema
+ * migration, do not use fast mode. Fast mode defaults to false for this reason, but is used by the
+ * monthly invocation of the mapreduce.
  */
 @Action(
     service = Action.Service.BACKEND,
@@ -48,15 +58,31 @@ public class ResaveAllEppResourcesAction implements Runnable {
 
   @Inject MapreduceRunner mrRunner;
   @Inject Response response;
-  @Inject ResaveAllEppResourcesAction() {}
+
+  @Inject
+  @Parameter(PARAM_FAST)
+  boolean isFast;
+
+  @Inject
+  ResaveAllEppResourcesAction() {}
+
+  /**
+   * The number of shards to run the map-only mapreduce on.
+   *
+   * <p>This is less than the default of 100 because we only run this action monthly and can afford
+   * it being slower, but we don't want to write out lots of large commit logs in a short period of
+   * time because they make the Cloud SQL migration tougher.
+   */
+  private static final int NUM_SHARDS = 10;
 
   @Override
   public void run() {
     mrRunner
         .setJobName("Re-save all EPP resources")
         .setModuleName("backend")
+        .setDefaultMapShards(NUM_SHARDS)
         .runMapOnly(
-            new ResaveAllEppResourcesActionMapper(),
+            new ResaveAllEppResourcesActionMapper(isFast),
             ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))
         .sendLinkToMapreduceConsole(response);
   }
@@ -66,23 +92,33 @@ public class ResaveAllEppResourcesAction implements Runnable {
       extends Mapper<Key<EppResource>, Void, Void> {
 
     private static final long serialVersionUID = -7721628665138087001L;
-    public ResaveAllEppResourcesActionMapper() {}
+
+    private final boolean isFast;
+
+    ResaveAllEppResourcesActionMapper(boolean isFast) {
+      this.isFast = isFast;
+    }
 
     @Override
     public final void map(final Key<EppResource> resourceKey) {
-      tm()
-          .transact(
-              () -> {
-                EppResource projectedResource =
-                    ofy()
-                        .load()
-                        .key(resourceKey)
-                        .now()
-                        .cloneProjectedAtTime(tm().getTransactionTime());
-                ofy().save().entity(projectedResource).now();
-              });
-      getContext().incrementCounter(String.format("%s entities re-saved", resourceKey.getKind()));
+      boolean resaved =
+          tm().transact(
+                  () -> {
+                    EppResource originalResource = ofy().load().key(resourceKey).now();
+                    EppResource projectedResource =
+                        originalResource.cloneProjectedAtTime(tm().getTransactionTime());
+                    if (isFast && originalResource.equals(projectedResource)) {
+                      return false;
+                    } else {
+                      ofy().save().entity(projectedResource).now();
+                      return true;
+                    }
+                  });
+      getContext()
+          .incrementCounter(
+              String.format(
+                  "%s entities %s",
+                  resourceKey.getKind(), resaved ? "re-saved" : "with no changes skipped"));
     }
   }
 }
-

core/src/main/java/google/registry/beam/initsql/BeamJpaModule.java

@@ -31,6 +31,7 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.JdbcJpaTm;
 import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -168,6 +169,7 @@ public class BeamJpaModule {
         BeamJpaModule.class,
         KmsModule.class,
         PersistenceModule.class,
+        SecretManagerModule.class,
         UtilsModule.class
       })
   public interface JpaTransactionManagerComponent {

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Throwables.throwIfUnchecked;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.JpaRetries.isFailedTxnRetriable;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.setJpaTm;
@@ -42,7 +43,6 @@ import google.registry.backup.CommitLogImports;
 import google.registry.backup.VersionedEntity;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.ofy.ObjectifyService;
-import google.registry.model.ofy.Ofy;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.tools.LevelDbLogReader;
 import google.registry.util.SystemSleeper;
@@ -68,6 +68,7 @@ import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -264,9 +265,9 @@ public final class Transforms {
   }
 
   /**
-   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
-   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
-   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   * Returns a {@link PTransform} that writes a {@link PCollection} of {@link VersionedEntity}s to a
+   * SQL database. and outputs an empty {@code PCollection<Void>}. This allows other operations to
+   * {@link org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
    *
    * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
    * job, we will not add features unless proven necessary.
@@ -282,16 +283,53 @@ public final class Transforms {
       int maxWriters,
       int batchSize,
       SerializableSupplier<JpaTransactionManager> jpaSupplier) {
-    return new PTransform<PCollection<VersionedEntity>, PCollection<Void>>() {
+    return writeToSql(
+        transformId,
+        maxWriters,
+        batchSize,
+        jpaSupplier,
+        (e) -> ofy().toPojo(e.getEntity().get()),
+        TypeDescriptor.of(VersionedEntity.class));
+  }
+
+  /**
+   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
+   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
+   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   *
+   * <p>The converter and type descriptor are generics so that we can convert any type of entity to
+   * an object to be placed in SQL.
+   *
+   * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
+   * job, we will not add features unless proven necessary.
+   *
+   * @param transformId a unique ID for an instance of the returned transform
+   * @param maxWriters the max number of concurrent writes to SQL, which also determines the max
+   *     number of connection pools created
+   * @param batchSize the number of entities to write in each operation
+   * @param jpaSupplier supplier of a {@link JpaTransactionManager}
+   * @param jpaConverter the function that converts the input object to a JPA entity
+   * @param objectDescriptor the type descriptor of the input object
+   */
+  public static <T> PTransform<PCollection<T>, PCollection<Void>> writeToSql(
+      String transformId,
+      int maxWriters,
+      int batchSize,
+      SerializableSupplier<JpaTransactionManager> jpaSupplier,
+      SerializableFunction<T, Object> jpaConverter,
+      TypeDescriptor<T> objectDescriptor) {
+    return new PTransform<PCollection<T>, PCollection<Void>>() {
       @Override
-      public PCollection<Void> expand(PCollection<VersionedEntity> input) {
+      public PCollection<Void> expand(PCollection<T> input) {
         return input
             .apply(
                 "Shard data for " + transformId,
-                MapElements.into(kvs(integers(), TypeDescriptor.of(VersionedEntity.class)))
+                MapElements.into(kvs(integers(), objectDescriptor))
                     .via(ve -> KV.of(ThreadLocalRandom.current().nextInt(maxWriters), ve)))
             .apply("Batch output by shard " + transformId, GroupIntoBatches.ofSize(batchSize))
-            .apply("Write in batch for " + transformId, ParDo.of(new SqlBatchWriter(jpaSupplier)));
+            .apply(
+                "Write in batch for " + transformId,
+                ParDo.of(new SqlBatchWriter<T>(jpaSupplier, jpaConverter)));
       }
     };
   }
@@ -385,18 +423,21 @@ public final class Transforms {
    * to hold the {@code JpaTransactionManager} instance, we must ensure that JpaTransactionManager
    * is not changed or torn down while being used by some instance.
    */
-  private static class SqlBatchWriter extends DoFn<KV<Integer, Iterable<VersionedEntity>>, Void> {
+  private static class SqlBatchWriter<T> extends DoFn<KV<Integer, Iterable<T>>, Void> {
 
     private static int instanceCount = 0;
     private static JpaTransactionManager originalJpa;
 
     private final SerializableSupplier<JpaTransactionManager> jpaSupplier;
+    private final SerializableFunction<T, Object> jpaConverter;
 
-    private transient Ofy ofy;
     private transient SystemSleeper sleeper;
 
-    SqlBatchWriter(SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+    SqlBatchWriter(
+        SerializableSupplier<JpaTransactionManager> jpaSupplier,
+        SerializableFunction<T, Object> jpaConverter) {
       this.jpaSupplier = jpaSupplier;
+      this.jpaConverter = jpaConverter;
     }
 
     @Setup
@@ -405,7 +446,6 @@ public final class Transforms {
 
       try (AppEngineEnvironment env = new AppEngineEnvironment()) {
         ObjectifyService.initOfy();
-        ofy = ObjectifyService.ofy();
       }
 
       synchronized (SqlBatchWriter.class) {
@@ -429,13 +469,11 @@ public final class Transforms {
     }
 
     @ProcessElement
-    public void processElement(@Element KV<Integer, Iterable<VersionedEntity>> kv) {
+    public void processElement(@Element KV<Integer, Iterable<T>> kv) {
       try (AppEngineEnvironment env = new AppEngineEnvironment()) {
         ImmutableList<Object> ofyEntities =
             Streams.stream(kv.getValue())
-                .map(VersionedEntity::getEntity)
-                .map(Optional::get)
-                .map(ofy::toPojo)
+                .map(this.jpaConverter::apply)
                 .collect(ImmutableList.toImmutableList());
         retry(() -> jpaTm().transact(() -> jpaTm().putAll(ofyEntities)));
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -20,7 +20,7 @@ import static google.registry.beam.BeamUtils.getQueryFromFile;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
-import google.registry.backup.AppEngineEnvironment;
+import google.registry.beam.initsql.Transforms;
 import google.registry.beam.initsql.Transforms.SerializableSupplier;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
 import google.registry.config.CredentialModule.LocalCredential;
@@ -43,7 +43,6 @@ import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
-import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -191,34 +190,27 @@ public class Spec11Pipeline implements Serializable {
       PCollection<Subdomain> domains,
       EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
       ValueProvider<String> dateProvider) {
-
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsSql =
         domains.apply("Run through SafeBrowsing API", ParDo.of(evaluateSafeBrowsingFn));
-    /* Store ThreatMatch objects in SQL. */
+    TypeDescriptor<KV<Subdomain, ThreatMatch>> descriptor =
+        new TypeDescriptor<KV<Subdomain, ThreatMatch>>() {};
     subdomainsSql.apply(
-        ParDo.of(
-            new DoFn<KV<Subdomain, ThreatMatch>, Void>() {
-              @ProcessElement
-              public void processElement(ProcessContext context) {
-                // create the Spec11ThreatMatch from Subdomain and ThreatMatch
-                try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-                  Subdomain subdomain = context.element().getKey();
-                  Spec11ThreatMatch threatMatch =
-                      new Spec11ThreatMatch.Builder()
-                          .setThreatTypes(
-                              ImmutableSet.of(
-                                  ThreatType.valueOf(context.element().getValue().threatType())))
-                          .setCheckDate(
-                              LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
-                          .setDomainName(subdomain.domainName())
-                          .setDomainRepoId(subdomain.domainRepoId())
-                          .setRegistrarId(subdomain.registrarId())
-                          .build();
-                  JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
-                  jpaTransactionManager.transact(() -> jpaTransactionManager.insert(threatMatch));
-                }
-              }
-            }));
+        Transforms.writeToSql(
+            "Spec11ThreatMatch",
+            4,
+            4,
+            jpaSupplierFactory,
+            (kv) -> {
+              Subdomain subdomain = kv.getKey();
+              return new Spec11ThreatMatch.Builder()
+                  .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
+                  .setCheckDate(LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
+                  .setDomainName(subdomain.domainName())
+                  .setDomainRepoId(subdomain.domainRepoId())
+                  .setRegistrarId(subdomain.registrarId())
+                  .build();
+            },
+            descriptor));
 
     /* Store ThreatMatch objects in JSON. */
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -415,6 +415,14 @@ public final class RegistryConfig {
       return config.cloudSql.instanceConnectionName;
     }
 
+    @Provides
+    @Config("cloudSqlDbInstanceName")
+    public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
+      // Format of instanceConnectionName: project-id:region:instance-name
+      int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
+      return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);
+    }
+
     @Provides
     @Config("cloudDnsRootUrl")
     public static Optional<String> getCloudDnsRootUrl(RegistryConfigSettings config) {

core/src/main/java/google/registry/env/alpha/default/WEB-INF/cron.xml

@@ -80,7 +80,7 @@
   </cron>
 
   <cron>
-    <url><![CDATA[/_dr/task/resaveAllEppResources]]></url>
+    <url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url>
     <description>
       This job resaves all our resources, projected in time to "now".
       It is needed for "deleteOldCommitLogs" to work correctly.

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -103,7 +103,7 @@
   </cron>
 
   <cron>
-    <url><![CDATA[/_dr/task/resaveAllEppResources]]></url>
+    <url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url>
     <description>
       This job resaves all our resources, projected in time to "now".
       It is needed for "deleteOldCommitLogs" to work correctly.

core/src/main/java/google/registry/env/qa/default/WEB-INF/cron.xml

@@ -21,7 +21,7 @@
   </cron>
 
   <cron>
-    <url><![CDATA[/_dr/task/resaveAllEppResources]]></url>
+    <url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url>
     <description>
       This job resaves all our resources, projected in time to "now".
       It is needed for "deleteOldCommitLogs" to work correctly.

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -87,7 +87,7 @@
   </cron>
 
   <cron>
-    <url><![CDATA[/_dr/task/resaveAllEppResources]]></url>
+    <url><![CDATA[/_dr/task/resaveAllEppResources?fast=true]]></url>
     <description>
       This job resaves all our resources, projected in time to "now".
       It is needed for "deleteOldCommitLogs" to work correctly.

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -110,7 +110,6 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.persistence.VKey;
 import google.registry.tmch.LordnTaskUtils;
 import java.util.Optional;
@@ -372,7 +371,7 @@ public class DomainCreateFlow implements TransactionalFlow {
         && TokenType.SINGLE_USE.equals(allocationToken.get().getTokenType())) {
       entitiesToSave.add(
           allocationTokenFlowUtils.redeemToken(
-              allocationToken.get(), DomainHistoryVKey.create(Key.create(historyEntry))));
+              allocationToken.get(), HistoryEntry.createVKey(Key.create(historyEntry))));
     }
     enqueueTasks(newDomain, hasSignedMarks, hasClaimsNotice);
 

core/src/main/java/google/registry/flows/domain/token/AllocationTokenFlowUtils.java

@@ -15,14 +15,13 @@
 package google.registry.flows.domain.token;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.common.net.InternetDomainName;
-import com.googlecode.objectify.Key;
 import google.registry.flows.EppException;
 import google.registry.flows.EppException.AssociationProhibitsOperationException;
 import google.registry.flows.EppException.AuthorizationErrorException;
@@ -32,7 +31,8 @@ import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.registry.Registry;
-import google.registry.persistence.DomainHistoryVKey;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -107,7 +107,7 @@ public class AllocationTokenFlowUtils {
 
   /** Redeems a SINGLE_USE {@link AllocationToken}, returning the redeemed copy. */
   public AllocationToken redeemToken(
-      AllocationToken token, DomainHistoryVKey redemptionHistoryEntry) {
+      AllocationToken token, VKey<? extends HistoryEntry> redemptionHistoryEntry) {
     checkArgument(
         TokenType.SINGLE_USE.equals(token.getTokenType()),
         "Only SINGLE_USE tokens can be marked as redeemed");
@@ -152,14 +152,15 @@ public class AllocationTokenFlowUtils {
       // See https://tools.ietf.org/html/draft-ietf-regext-allocation-token-04#section-2.1
       throw new InvalidAllocationTokenException();
     }
-    AllocationToken tokenEntity = ofy().load().key(Key.create(AllocationToken.class, token)).now();
-    if (tokenEntity == null) {
+    Optional<AllocationToken> maybeTokenEntity =
+        tm().maybeLoad(VKey.create(AllocationToken.class, token));
+    if (maybeTokenEntity.isEmpty()) {
       throw new InvalidAllocationTokenException();
     }
-    if (tokenEntity.isRedeemed()) {
+    if (maybeTokenEntity.get().isRedeemed()) {
       throw new AlreadyRedeemedAllocationTokenException();
     }
-    return tokenEntity;
+    return maybeTokenEntity.get();
   }
 
   // Note: exception messages should be <= 32 characters long for domain check results

core/src/main/java/google/registry/flows/host/HostCreateFlow.java

@@ -21,10 +21,8 @@ import static google.registry.flows.host.HostFlowUtils.validateHostName;
 import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomainNotInPendingDelete;
 import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomainOwnership;
 import static google.registry.model.EppResourceUtils.createRepoId;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
-import static google.registry.util.CollectionUtils.union;
 
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
@@ -137,13 +135,11 @@ public final class HostCreateFlow implements TransactionalFlow {
     ImmutableSet<ImmutableObject> entitiesToSave =
         ImmutableSet.of(
             newHost,
-            historyBuilder.build(),
+            historyBuilder.build().toChildHistoryEntity(),
             ForeignKeyIndex.create(newHost, newHost.getDeletionTime()),
             EppResourceIndex.create(Key.create(newHost)));
     if (superordinateDomain.isPresent()) {
-      entitiesToSave =
-          union(
-              entitiesToSave,
+      tm().update(
               superordinateDomain
                   .get()
                   .asBuilder()
@@ -153,7 +149,7 @@ public final class HostCreateFlow implements TransactionalFlow {
       // they are only written as NS records from the referencing domain.
       dnsQueue.addHostRefreshTask(targetId);
     }
-    ofy().save().entities(entitiesToSave);
+    tm().insertAll(entitiesToSave);
     return responseBuilder.setResData(HostCreateData.create(targetId, now)).build();
   }
 

core/src/main/java/google/registry/mapreduce/MapreduceModule.java

@@ -15,6 +15,7 @@
 package google.registry.mapreduce;
 
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
+import static google.registry.mapreduce.MapreduceRunner.PARAM_FAST;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_REDUCE_SHARDS;
 import static google.registry.request.RequestParameters.extractBooleanParameter;
@@ -36,6 +37,12 @@ public final class MapreduceModule {
     return extractBooleanParameter(req, PARAM_DRY_RUN);
   }
 
+  @Provides
+  @Parameter(PARAM_FAST)
+  static boolean provideIsFast(HttpServletRequest req) {
+    return extractBooleanParameter(req, PARAM_FAST);
+  }
+
   @Provides
   @Parameter(PARAM_MAP_SHARDS)
   static Optional<Integer> provideMapShards(HttpServletRequest req) {

core/src/main/java/google/registry/mapreduce/MapreduceRunner.java

@@ -55,6 +55,7 @@ public class MapreduceRunner {
   public static final String PARAM_DRY_RUN = "dryRun";
   public static final String PARAM_MAP_SHARDS = "mapShards";
   public static final String PARAM_REDUCE_SHARDS = "reduceShards";
+  public static final String PARAM_FAST = "fast";
 
   private static final String BASE_URL = "/_dr/mapreduce/";
   private static final String QUEUE_NAME = "mapreduce";

core/src/main/java/google/registry/model/DatabaseMigrationUtils.java

@@ -0,0 +1,38 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryEnvironment;
+
+/** Utility methods related to migrating dual-read/dual-write entities. */
+public class DatabaseMigrationUtils {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Throws exceptions only in unit tests, otherwise only logs exceptions. */
+  public static void suppressExceptionUnlessInTest(Runnable work, String message) {
+    try {
+      work.run();
+    } catch (Exception e) {
+      if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)) {
+        throw e;
+      }
+      logger.atWarning().withCause(e).log(message);
+    }
+  }
+
+  private DatabaseMigrationUtils() {}
+}

core/src/main/java/google/registry/model/EppResourceUtils.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.DateTimeUtils.isAtOrAfter;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.latestOf;
@@ -135,7 +136,7 @@ public final class EppResourceUtils {
         useCache
             ? ForeignKeyIndex.loadCached(clazz, ImmutableList.of(foreignKey), now)
                 .getOrDefault(foreignKey, null)
-            : ofy().load().type(ForeignKeyIndex.mapToFkiClass(clazz)).id(foreignKey).now();
+            : ForeignKeyIndex.load(clazz, foreignKey, now);
     // The value of fki.getResourceKey() might be null for hard-deleted prober data.
     if (fki == null || isAtOrAfter(now, fki.getDeletionTime()) || fki.getResourceKey() == null) {
       return Optional.empty();
@@ -143,7 +144,7 @@ public final class EppResourceUtils {
     T resource =
         useCache
             ? EppResource.loadCached(fki.getResourceKey())
-            : tm().maybeLoad(fki.getResourceKey()).orElse(null);
+            : transactIfJpaTm(() -> tm().maybeLoad(fki.getResourceKey()).orElse(null));
     if (resource == null || isAtOrAfter(now, resource.getDeletionTime())) {
       return Optional.empty();
     }

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -22,12 +22,17 @@ import javax.annotation.Nullable;
 import org.joda.time.DateTime;
 
 /**
- * A timestamp that auto-updates on each save to Datastore.
+ * A timestamp that auto-updates on each save to Datastore/Cloud SQL.
  *
  * @see UpdateAutoTimestampTranslatorFactory
  */
 public class UpdateAutoTimestamp extends ImmutableObject {
 
+  // When set to true, database converters/translators should do tha auto update.  When set to
+  // false, auto update should be suspended (this exists to allow us to preserve the original value
+  // during a replay).
+  private static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
+
   DateTime timestamp;
 
   /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
@@ -40,4 +45,30 @@ public class UpdateAutoTimestamp extends ImmutableObject {
     instance.timestamp = timestamp;
     return instance;
   }
+
+  // TODO(b/175610935): Remove the auto-update disabling code below after migration.
+
+  /** Class to allow us to safely disable auto-update in a try-with-resources block. */
+  public static class DisableAutoUpdateResource implements AutoCloseable {
+    DisableAutoUpdateResource() {
+      autoUpdateEnabled.set(false);
+    }
+
+    @Override
+    public void close() {
+      autoUpdateEnabled.set(true);
+    }
+  }
+
+  /**
+   * Resturns a resource that disables auto-updates on all {@link UpdateAutoTimestamp}s in the
+   * current thread, suitable for use with in a try-with-resources block.
+   */
+  public static DisableAutoUpdateResource disableAutoUpdate() {
+    return new DisableAutoUpdateResource();
+  }
+
+  public static boolean autoUpdateEnabled() {
+    return autoUpdateEnabled.get();
+  }
 }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -290,7 +290,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "allocationToken")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
@@ -464,7 +464,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {
 
     /**
@@ -559,7 +559,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "billingTime")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billing time of the charge that is being cancelled. */
@@ -680,7 +680,7 @@ public abstract class BillingEvent extends ImmutableObject
   /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Modification extends BillingEvent implements DatastoreOnlyEntity {
 
     /** The change in cost that should be applied to the original billing event. */

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -75,7 +75,9 @@ public class DomainBase extends DomainContent
   }
 
   @ElementCollection
-  @JoinTable(name = "DomainHost")
+  @JoinTable(
+      name = "DomainHost",
+      indexes = {@Index(columnList = "domain_repo_id,host_repo_id", unique = true)})
   @Access(AccessType.PROPERTY)
   @Column(name = "host_repo_id")
   public Set<VKey<HostResource>> getNsHosts() {

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -96,7 +96,14 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   // TODO(b/166776754): Investigate if we can reuse domainContent.nsHosts for storing host keys.
   @Ignore
   @ElementCollection
-  @JoinTable(name = "DomainHistoryHost")
+  @JoinTable(
+      name = "DomainHistoryHost",
+      indexes = {
+        @Index(
+            columnList =
+                "domain_history_history_revision_id,domain_history_domain_repo_id,host_repo_id",
+            unique = true),
+      })
   @Column(name = "host_repo_id")
   Set<VKey<HostResource>> nsHosts;
 

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -23,6 +23,8 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.ofy.ObjectifyService;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import javax.annotation.Nullable;
@@ -82,10 +84,8 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
     instance.domainRepoId = checkArgumentNotNull(domainRepoId);
     instance.expirationTime = checkArgumentNotNull(expirationTime);
     instance.clientId = checkArgumentNotNull(clientId);
-    instance.billingEventOneTime = billingEventOneTime;
-    instance.billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
-    instance.billingEventRecurring = billingEventRecurring;
-    instance.billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+    instance.billingEventOneTime = BillingEventVKey.create(billingEventOneTime);
+    instance.billingEventRecurring = BillingRecurrenceVKey.create(billingEventRecurring);
     return instance;
   }
 
@@ -178,7 +178,6 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
   public GracePeriod cloneAfterOfyLoad(String domainRepoId) {
     GracePeriod clone = clone(this);
     clone.domainRepoId = checkArgumentNotNull(domainRepoId);
-    clone.restoreHistoryIds();
     return clone;
   }
 
@@ -190,7 +189,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
    */
   public GracePeriod cloneWithRecurringBillingEvent(VKey<BillingEvent.Recurring> recurring) {
     GracePeriod clone = clone(this);
-    clone.billingEventRecurring = recurring;
+    clone.billingEventRecurring = BillingRecurrenceVKey.create(recurring);
     return clone;
   }
 
@@ -232,9 +231,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
       instance.expirationTime = gracePeriod.expirationTime;
       instance.clientId = gracePeriod.clientId;
       instance.billingEventOneTime = gracePeriod.billingEventOneTime;
-      instance.billingEventOneTimeHistoryId = gracePeriod.billingEventOneTimeHistoryId;
       instance.billingEventRecurring = gracePeriod.billingEventRecurring;
-      instance.billingEventRecurringHistoryId = gracePeriod.billingEventRecurringHistoryId;
       return instance;
     }
   }

core/src/main/java/google/registry/model/domain/GracePeriodBase.java

@@ -14,14 +14,14 @@
 
 package google.registry.model.domain;
 
-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ModelUtils;
 import google.registry.model.billing.BillingEvent;
-import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import java.lang.reflect.Field;
 import java.util.LinkedHashMap;
@@ -68,24 +68,16 @@ public class GracePeriodBase extends ImmutableObject {
    * billingEventRecurring}) or for redemption grace periods (since deletes have no cost).
    */
   // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  @Column(name = "billing_event_id")
-  VKey<OneTime> billingEventOneTime = null;
-
-  @Ignore
-  @Column(name = "billing_event_history_id")
-  Long billingEventOneTimeHistoryId;
+  @Access(AccessType.FIELD)
+  BillingEventVKey billingEventOneTime = null;
 
   /**
    * The recurring billing event corresponding to the action that triggered this grace period, if
    * applicable - i.e. if the action was an autorenew - or null in all other cases.
    */
   // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  @Column(name = "billing_recurrence_id")
-  VKey<BillingEvent.Recurring> billingEventRecurring = null;
-
-  @Ignore
-  @Column(name = "billing_recurrence_history_id")
-  Long billingEventRecurringHistoryId;
+  @Access(AccessType.FIELD)
+  BillingRecurrenceVKey billingEventRecurring = null;
 
   public long getGracePeriodId() {
     return gracePeriodId;
@@ -123,8 +115,7 @@ public class GracePeriodBase extends ImmutableObject {
    * period is not AUTO_RENEW.
    */
   public VKey<BillingEvent.OneTime> getOneTimeBillingEvent() {
-    restoreOfyKeys();
-    return billingEventOneTime;
+    return billingEventOneTime == null ? null : billingEventOneTime.createVKey();
   }
 
   /**
@@ -132,18 +123,7 @@ public class GracePeriodBase extends ImmutableObject {
    * period is AUTO_RENEW.
    */
   public VKey<BillingEvent.Recurring> getRecurringBillingEvent() {
-    restoreOfyKeys();
-    return billingEventRecurring;
-  }
-
-  /**
-   * Restores history ids for composite VKeys after a load from datastore.
-   *
-   * <p>For use by DomainContent.load() ONLY.
-   */
-  protected void restoreHistoryIds() {
-    billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
-    billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+    return billingEventRecurring == null ? null : billingEventRecurring.createVKey();
   }
 
   /**
@@ -152,7 +132,6 @@ public class GracePeriodBase extends ImmutableObject {
    */
   @Override
   protected Map<Field, Object> getSignificantFields() {
-    restoreOfyKeys();
     // Can't use streams or ImmutableMap because we can have null values.
     Map<Field, Object> result = new LinkedHashMap();
     for (Map.Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
@@ -162,33 +141,4 @@ public class GracePeriodBase extends ImmutableObject {
     }
     return result;
   }
-
-  /**
-   * Restores Ofy keys in the billing events.
-   *
-   * <p>This must be called by all methods that access the one time or recurring billing event keys.
-   * When the billing event keys are loaded from SQL, they are loaded as asymmetric keys because the
-   * database columns that we load them from do not contain all of the information necessary to
-   * reconsitute the Ofy side of the key. In other cases, we restore the Ofy key during the
-   * hibernate {@link javax.persistence.PostLoad} method from the other fields of the object, but we
-   * have been unable to make this work with hibernate's internal persistence model in this case
-   * because the {@link GracePeriod}'s hash code is evaluated prior to these calls, and would be
-   * invalidated by changing the fields.
-   */
-  private final synchronized void restoreOfyKeys() {
-    if (billingEventOneTime != null && !billingEventOneTime.maybeGetOfyKey().isPresent()) {
-      billingEventOneTime =
-          DomainBase.restoreOfyFrom(
-              Key.create(DomainBase.class, domainRepoId),
-              billingEventOneTime,
-              billingEventOneTimeHistoryId);
-    }
-    if (billingEventRecurring != null && !billingEventRecurring.maybeGetOfyKey().isPresent()) {
-      billingEventRecurring =
-          DomainBase.restoreOfyFrom(
-              Key.create(DomainBase.class, domainRepoId),
-              billingEventRecurring,
-              billingEventRecurringHistoryId);
-    }
-  }
 }

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -111,9 +111,9 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @Nullable
   @Index
   @AttributeOverrides({
-    @AttributeOverride(name = "domainRepoId", column = @Column(name = "redemption_domain_repo_id")),
+    @AttributeOverride(name = "repoId", column = @Column(name = "redemption_domain_repo_id")),
     @AttributeOverride(
-        name = "domainHistoryId",
+        name = "historyRevisionId",
         column = @Column(name = "redemption_domain_history_id"))
   })
   DomainHistoryVKey redemptionHistoryEntry;
@@ -192,8 +192,9 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
     return token;
   }
 
-  public Optional<VKey<HistoryEntry>> getRedemptionHistoryEntry() {
-    return Optional.ofNullable(redemptionHistoryEntry);
+  public Optional<VKey<? extends HistoryEntry>> getRedemptionHistoryEntry() {
+    return Optional.ofNullable(
+        redemptionHistoryEntry == null ? null : redemptionHistoryEntry.createDomainHistoryVKey());
   }
 
   public boolean isRedeemed() {
@@ -291,9 +292,10 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
       return this;
     }
 
-    public Builder setRedemptionHistoryEntry(DomainHistoryVKey redemptionHistoryEntry) {
+    public Builder setRedemptionHistoryEntry(VKey<? extends HistoryEntry> redemptionHistoryEntry) {
+      checkArgumentNotNull(redemptionHistoryEntry, "Redemption history entry must not be null");
       getInstance().redemptionHistoryEntry =
-          checkArgumentNotNull(redemptionHistoryEntry, "Redemption history entry must not be null");
+          DomainHistoryVKey.create(redemptionHistoryEntry.getOfyKey());
       return this;
     }
 

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -15,9 +15,11 @@
 package google.registry.model.index;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.TypeUtils.instantiate;
 
@@ -29,6 +31,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Multimaps;
 import com.google.common.collect.Streams;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -44,6 +47,8 @@ import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.NonFinalForTesting;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
@@ -76,13 +81,21 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource>
       implements DatastoreOnlyEntity {}
 
-  static final ImmutableMap<Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
+  private static final ImmutableMap<
+          Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
       RESOURCE_CLASS_TO_FKI_CLASS =
           ImmutableMap.of(
               ContactResource.class, ForeignKeyContactIndex.class,
               DomainBase.class, ForeignKeyDomainIndex.class,
               HostResource.class, ForeignKeyHostIndex.class);
 
+  private static final ImmutableMap<Class<? extends EppResource>, String>
+      RESOURCE_CLASS_TO_FKI_PROPERTY =
+          ImmutableMap.of(
+              ContactResource.class, "contactId",
+              DomainBase.class, "fullyQualifiedDomainName",
+              HostResource.class, "fullyQualifiedHostName");
+
   @Id String foreignKey;
 
   /**
@@ -179,9 +192,42 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
       Class<E> clazz, Iterable<String> foreignKeys, final DateTime now) {
-    return ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys).entrySet().stream()
-        .filter(e -> now.isBefore(e.getValue().deletionTime))
-        .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+    if (tm().isOfy()) {
+      return ofy().load().type(mapToFkiClass(clazz)).ids(foreignKeys).entrySet().stream()
+          .filter(e -> now.isBefore(e.getValue().deletionTime))
+          .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue));
+    } else {
+      String property = RESOURCE_CLASS_TO_FKI_PROPERTY.get(clazz);
+      List<E> entities =
+          tm().transact(
+                  () -> {
+                    String entityName =
+                        jpaTm().getEntityManager().getMetamodel().entity(clazz).getName();
+                    return jpaTm()
+                        .getEntityManager()
+                        .createQuery(
+                            String.format(
+                                "FROM %s WHERE %s IN :propertyValue and deletionTime > :now ",
+                                entityName, property),
+                            clazz)
+                        .setParameter("propertyValue", foreignKeys)
+                        .setParameter("now", now)
+                        .getResultList();
+                  });
+      // We need to find and return the entities with the maximum deletionTime for each foreign key.
+      return Multimaps.index(entities, EppResource::getForeignKey).asMap().entrySet().stream()
+          .map(
+              entry ->
+                  Maps.immutableEntry(
+                      entry.getKey(),
+                      entry.getValue().stream()
+                          .max(Comparator.comparing(EppResource::getDeletionTime))
+                          .get()))
+          .collect(
+              toImmutableMap(
+                  Map.Entry::getKey,
+                  entry -> create(entry.getValue(), entry.getValue().getDeletionTime())));
+    }
   }
 
   static final CacheLoader<Key<ForeignKeyIndex<?>>, Optional<ForeignKeyIndex<?>>> CACHE_LOADER =
@@ -266,7 +312,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               .filter(entry -> entry.getValue().isPresent())
               .filter(entry -> now.isBefore(entry.getValue().get().getDeletionTime()))
               .collect(
-                  ImmutableMap.toImmutableMap(
+                  toImmutableMap(
                       entry -> entry.getKey().getName(),
                       entry -> (ForeignKeyIndex<E>) entry.getValue().get()));
       return fkisFromCache;

core/src/main/java/google/registry/model/ofy/ObjectifyService.java

@@ -42,8 +42,8 @@ import google.registry.model.translators.CidrAddressBlockTranslatorFactory;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
 import google.registry.model.translators.CreateAutoTimestampTranslatorFactory;
 import google.registry.model.translators.CurrencyUnitTranslatorFactory;
-import google.registry.model.translators.DomainHistoryVKeyTranslatorFactory;
 import google.registry.model.translators.DurationTranslatorFactory;
+import google.registry.model.translators.EppHistoryVKeyTranslatorFactory;
 import google.registry.model.translators.InetAddressTranslatorFactory;
 import google.registry.model.translators.ReadableInstantUtcTranslatorFactory;
 import google.registry.model.translators.UpdateAutoTimestampTranslatorFactory;
@@ -128,7 +128,7 @@ public class ObjectifyService {
             new CreateAutoTimestampTranslatorFactory(),
             new CurrencyUnitTranslatorFactory(),
             new DurationTranslatorFactory(),
-            new DomainHistoryVKeyTranslatorFactory(),
+            new EppHistoryVKeyTranslatorFactory(),
             new InetAddressTranslatorFactory(),
             new MoneyStringTranslatorFactory(),
             new ReadableInstantUtcTranslatorFactory(),

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -279,7 +279,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("ONE_TIME")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends PollMessage {
 
     // Response data. Objectify cannot persist a base class type, so we must have a separate field
@@ -432,7 +432,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("AUTORENEW")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatchDao.java

@@ -21,10 +21,10 @@ import javax.persistence.TemporalType;
 import org.joda.time.LocalDate;
 
 /**
- * Data access object for {@link google.registry.model.reporting.Spec11ThreatMatch}.
+ * Data access object for {@link Spec11ThreatMatch}.
  *
- * <p>A JpaTransactionManager is passed into each static method because they are called from a BEAM
- * pipeline and we don't know where it's coming from.
+ * <p>The transaction manager is passed as a parameter because this could be called either from a
+ * BEAM pipeline or standard non-BEAM code.
  */
 public class Spec11ThreatMatchDao {
 

core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java

@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.isEmpty;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -32,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EmbedMap;
 import com.googlecode.objectify.annotation.Entity;
@@ -82,8 +82,6 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 public class SignedMarkRevocationList extends ImmutableObject implements NonReplicatedEntity {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
   @VisibleForTesting static final int SHARD_SIZE = 10000;
 
   /** Common ancestor for queries. */
@@ -121,12 +119,11 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
       memoizeWithShortExpiration(
           () -> {
             SignedMarkRevocationList datastoreList = loadFromDatastore();
-            // Also load the list from Cloud SQL, compare the two lists, and log if different.
-            try {
-              loadAndCompareCloudSqlList(datastoreList);
-            } catch (Throwable t) {
-              logger.atSevere().withCause(t).log("Error comparing signed mark revocation lists.");
-            }
+            suppressExceptionUnlessInTest(
+                () -> {
+                  loadAndCompareCloudSqlList(datastoreList);
+                },
+                "Error comparing signed mark revocation lists.");
             return datastoreList;
           });
 
@@ -229,11 +226,12 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
           Maps.difference(datastoreList.revokes, cloudSqlList.revokes);
       if (!diff.areEqual()) {
         if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
+          String message =
               String.format(
                   "Unequal SM revocation lists detected, Cloud SQL list with revision id %d has %d"
                       + " different records than the current Datastore list.",
-                  cloudSqlList.revisionId, diff.entriesDiffering().size()));
+                  cloudSqlList.revisionId, diff.entriesDiffering().size());
+          throw new RuntimeException(message);
         } else {
           StringBuilder diffMessage = new StringBuilder("Unequal SM revocation lists detected:\n");
           diff.entriesDiffering()
@@ -243,11 +241,13 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
                           String.format(
                               "SMD %s has key %s in Datastore and key %s in Cloud SQL.\n",
                               label, valueDiff.leftValue(), valueDiff.rightValue())));
-          logger.atWarning().log(diffMessage.toString());
+          throw new RuntimeException(diffMessage.toString());
         }
       }
     } else {
-      logger.atWarning().log("Signed mark revocation list in Cloud SQL is empty.");
+      if (datastoreList.size() != 0) {
+        throw new RuntimeException("Signed mark revocation list in Cloud SQL is empty.");
+      }
     }
   }
 

core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java

@@ -15,6 +15,7 @@
 package google.registry.model.smd;
 
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.common.base.Supplier;
@@ -60,14 +61,14 @@ public class SignedMarkRevocationListDao {
    * the authoritative database.
    */
   static void trySave(SignedMarkRevocationList signedMarkRevocationList) {
-    try {
-      SignedMarkRevocationListDao.save(signedMarkRevocationList);
-      logger.atInfo().log(
-          "Inserted %,d signed mark revocations into Cloud SQL",
-          signedMarkRevocationList.revokes.size());
-    } catch (Throwable e) {
-      logger.atSevere().withCause(e).log("Error inserting signed mark revocations into Cloud SQL");
-    }
+    suppressExceptionUnlessInTest(
+        () -> {
+          SignedMarkRevocationListDao.save(signedMarkRevocationList);
+          logger.atInfo().log(
+              "Inserted %,d signed mark revocations into Cloud SQL.",
+              signedMarkRevocationList.revokes.size());
+        },
+        "Error inserting signed mark revocations into Cloud SQL.");
   }
 
   private static void save(SignedMarkRevocationList signedMarkRevocationList) {

core/src/main/java/google/registry/model/translators/DomainHistoryVKeyTranslatorFactory.java

@@ -1,48 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.model.translators;
-
-import com.google.appengine.api.datastore.Key;
-import google.registry.persistence.DomainHistoryVKey;
-import javax.annotation.Nullable;
-
-/** Translator factory for {@link DomainHistoryVKey}. */
-public class DomainHistoryVKeyTranslatorFactory
-    extends AbstractSimpleTranslatorFactory<DomainHistoryVKey, Key> {
-
-  public DomainHistoryVKeyTranslatorFactory() {
-    super(DomainHistoryVKey.class);
-  }
-
-  @Override
-  SimpleTranslator<DomainHistoryVKey, Key> createTranslator() {
-    return new SimpleTranslator<DomainHistoryVKey, Key>() {
-
-      @Nullable
-      @Override
-      public DomainHistoryVKey loadValue(@Nullable Key datastoreValue) {
-        return datastoreValue == null
-            ? null
-            : DomainHistoryVKey.create(com.googlecode.objectify.Key.create(datastoreValue));
-      }
-
-      @Nullable
-      @Override
-      public Key saveValue(@Nullable DomainHistoryVKey pojoValue) {
-        return pojoValue == null ? null : pojoValue.getOfyKey().getRaw();
-      }
-    };
-  }
-}

core/src/main/java/google/registry/model/translators/EppHistoryVKeyTranslatorFactory.java

@@ -0,0 +1,110 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.translators;
+
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static java.util.function.Function.identity;
+
+import com.google.appengine.api.datastore.Key;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
+import google.registry.persistence.DomainHistoryVKey;
+import google.registry.persistence.EppHistoryVKey;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import javax.annotation.Nullable;
+
+/** Translator factory for {@link EppHistoryVKey}. */
+public class EppHistoryVKeyTranslatorFactory
+    extends AbstractSimpleTranslatorFactory<EppHistoryVKey, Key> {
+
+  public EppHistoryVKeyTranslatorFactory() {
+    super(EppHistoryVKey.class);
+  }
+
+  // This map is used when we need to convert the raw Datastore key to its VKey instance. We have
+  // one dedicated VKey class, e.g. DomainHistoryVKey, for each such kind of entity, and we need
+  // a way to map the raw Datastore key to its VKey class. So, we use the kind path as the key of
+  // the map, and the kind path is created by concatenating all the kind strings in a raw Datastore
+  // key, e.g. the map key for ContactPollMessageVKey is "ContactResource/HistoryEntry/PollMessage".
+  @VisibleForTesting
+  static final ImmutableMap<String, Class<? extends EppHistoryVKey>> kindPathToVKeyClass =
+      ImmutableSet.of(DomainHistoryVKey.class, BillingEventVKey.class, BillingRecurrenceVKey.class)
+          .stream()
+          .collect(toImmutableMap(EppHistoryVKeyTranslatorFactory::getKindPath, identity()));
+
+  /**
+   * Gets the kind path string for the given {@link Class}.
+   *
+   * <p>This method calls the getKindPath method on an instance of the given {@link Class} to get
+   * the kind path string.
+   */
+  private static String getKindPath(Class<? extends EppHistoryVKey> clazz) {
+    try {
+      Constructor<?> constructor = clazz.getDeclaredConstructor();
+      constructor.setAccessible(true);
+      Object instance = constructor.newInstance();
+      Method getKindPathMethod = EppHistoryVKey.class.getDeclaredMethod("getKindPath");
+      getKindPathMethod.setAccessible(true);
+      return (String) getKindPathMethod.invoke(instance);
+    } catch (Throwable t) {
+      throw new IllegalStateException(t);
+    }
+  }
+
+  @Override
+  SimpleTranslator<EppHistoryVKey, Key> createTranslator() {
+    return new SimpleTranslator<EppHistoryVKey, Key>() {
+
+      @Nullable
+      @Override
+      public EppHistoryVKey loadValue(@Nullable Key datastoreValue) {
+        if (datastoreValue == null) {
+          return null;
+        } else {
+          com.googlecode.objectify.Key<?> ofyKey =
+              com.googlecode.objectify.Key.create(datastoreValue);
+          String kindPath = EppHistoryVKey.createKindPath(ofyKey);
+          if (kindPathToVKeyClass.containsKey(kindPath)) {
+            Class<? extends EppHistoryVKey> vKeyClass = kindPathToVKeyClass.get(kindPath);
+            try {
+              Method createVKeyMethod =
+                  vKeyClass.getDeclaredMethod("create", com.googlecode.objectify.Key.class);
+              return (EppHistoryVKey) createVKeyMethod.invoke(null, ofyKey);
+            } catch (NoSuchMethodException e) {
+              throw new IllegalStateException(
+                  "Missing static method create(com.googlecode.objectify.Key) on " + vKeyClass);
+            } catch (IllegalAccessException | InvocationTargetException e) {
+              throw new IllegalStateException("Error invoking createVKey on " + vKeyClass, e);
+            }
+          } else {
+            throw new IllegalStateException(
+                "Missing EppHistoryVKey implementation for kind path: " + kindPath);
+          }
+        }
+      }
+
+      @Nullable
+      @Override
+      public Key saveValue(@Nullable EppHistoryVKey pojoValue) {
+        return pojoValue == null ? null : pojoValue.createOfyKey().getRaw();
+      }
+    };
+  }
+}

core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java

@@ -46,7 +46,10 @@ public class UpdateAutoTimestampTranslatorFactory
       /** Save a timestamp, setting it to the current time. */
       @Override
       public Date saveValue(UpdateAutoTimestamp pojoValue) {
-        return tm().getTransactionTime().toDate();
-      }};
+        return UpdateAutoTimestamp.autoUpdateEnabled()
+            ? tm().getTransactionTime().toDate()
+            : pojoValue.getTimestamp().toDate();
+      }
+    };
   }
 }

core/src/main/java/google/registry/persistence/BillingVKey.java

@@ -0,0 +1,134 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+
+import com.googlecode.objectify.Key;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingEvent.OneTime;
+import google.registry.model.billing.BillingEvent.Recurring;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.HistoryEntry;
+import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.MappedSuperclass;
+
+/** Base class for {@link BillingEvent}'s {@link VKey}. */
+@MappedSuperclass
+public abstract class BillingVKey<K> extends EppHistoryVKey<K, DomainBase> {
+  Long billingId;
+
+  // Hibernate requires a default constructor.
+  BillingVKey() {}
+
+  BillingVKey(String repoId, long historyRevisionId, long billingId) {
+    super(repoId, historyRevisionId);
+    this.billingId = billingId;
+  }
+
+  Key<HistoryEntry> createHistoryEntryKey() {
+    return Key.create(Key.create(DomainBase.class, repoId), HistoryEntry.class, historyRevisionId);
+  }
+
+  @Override
+  public Object createSqlKey() {
+    return billingId;
+  }
+
+  /** VKey class for {@link BillingEvent.OneTime} that belongs to a {@link DomainBase} entity. */
+  @Embeddable
+  @AttributeOverrides({
+    @AttributeOverride(name = "repoId", column = @Column(name = "billing_event_domain_repo_id")),
+    @AttributeOverride(
+        name = "historyRevisionId",
+        column = @Column(name = "billing_event_history_id")),
+    @AttributeOverride(name = "billingId", column = @Column(name = "billing_event_id"))
+  })
+  public static class BillingEventVKey extends BillingVKey<OneTime> {
+
+    // Hibernate requires this default constructor
+    private BillingEventVKey() {}
+
+    private BillingEventVKey(String repoId, long historyRevisionId, long billingEventId) {
+      super(repoId, historyRevisionId, billingEventId);
+    }
+
+    @Override
+    public Key<OneTime> createOfyKey() {
+      return Key.create(createHistoryEntryKey(), BillingEvent.OneTime.class, billingId);
+    }
+
+    /** Creates a {@link BillingEventVKey} instance from the given {@link Key} instance. */
+    public static BillingEventVKey create(@Nullable Key<BillingEvent.OneTime> ofyKey) {
+      if (ofyKey == null) {
+        return null;
+      }
+      long billingEventId = ofyKey.getId();
+      long historyRevisionId = ofyKey.getParent().getId();
+      String repoId = ofyKey.getParent().getParent().getName();
+      return new BillingEventVKey(repoId, historyRevisionId, billingEventId);
+    }
+
+    /** Creates a {@link BillingEventVKey} instance from the given {@link VKey} instance. */
+    public static BillingEventVKey create(@Nullable VKey<BillingEvent.OneTime> vKey) {
+      return vKey == null ? null : create(vKey.getOfyKey());
+    }
+  }
+
+  /** VKey class for {@link BillingEvent.Recurring} that belongs to a {@link DomainBase} entity. */
+  @Embeddable
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "repoId",
+        column = @Column(name = "billing_recurrence_domain_repo_id")),
+    @AttributeOverride(
+        name = "historyRevisionId",
+        column = @Column(name = "billing_recurrence_history_id")),
+    @AttributeOverride(name = "billingId", column = @Column(name = "billing_recurrence_id"))
+  })
+  public static class BillingRecurrenceVKey extends BillingVKey<Recurring> {
+
+    // Hibernate requires this default constructor
+    private BillingRecurrenceVKey() {}
+
+    private BillingRecurrenceVKey(String repoId, long historyRevisionId, long billingEventId) {
+      super(repoId, historyRevisionId, billingEventId);
+    }
+
+    @Override
+    public Key<Recurring> createOfyKey() {
+      return Key.create(createHistoryEntryKey(), BillingEvent.Recurring.class, billingId);
+    }
+
+    /** Creates a {@link BillingRecurrenceVKey} instance from the given {@link Key} instance. */
+    public static BillingRecurrenceVKey create(@Nullable Key<BillingEvent.Recurring> ofyKey) {
+      if (ofyKey == null) {
+        return null;
+      }
+      long billingEventId = ofyKey.getId();
+      long historyRevisionId = ofyKey.getParent().getId();
+      String repoId = ofyKey.getParent().getParent().getName();
+      return new BillingRecurrenceVKey(repoId, historyRevisionId, billingEventId);
+    }
+
+    /** Creates a {@link BillingRecurrenceVKey} instance from the given {@link VKey} instance. */
+    public static BillingRecurrenceVKey create(@Nullable VKey<BillingEvent.Recurring> vKey) {
+      return vKey == null ? null : create(vKey.getOfyKey());
+    }
+  }
+}

core/src/main/java/google/registry/persistence/DomainHistoryVKey.java

@@ -18,45 +18,44 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import javax.persistence.Embeddable;
-import javax.persistence.PostLoad;
 
 /** {@link VKey} for {@link HistoryEntry} which parent is {@link DomainBase}. */
 @Embeddable
-public class DomainHistoryVKey extends VKey<HistoryEntry> {
-
-  private String domainRepoId;
-
-  private Long domainHistoryId;
+public class DomainHistoryVKey extends EppHistoryVKey<HistoryEntry, DomainBase> {
 
   // Hibernate requires a default constructor
   private DomainHistoryVKey() {}
 
-  private DomainHistoryVKey(String domainRepoId, long domainHistoryId) {
-    initWith(domainRepoId, domainHistoryId);
+  private DomainHistoryVKey(String repoId, long historyRevisionId) {
+    super(repoId, historyRevisionId);
   }
 
-  @PostLoad
-  void postLoad() {
-    initWith(domainRepoId, domainHistoryId);
+  @Override
+  public Object createSqlKey() {
+    return new DomainHistoryId(repoId, historyRevisionId);
   }
 
-  private void initWith(String domainRepoId, long domainHistoryId) {
-    this.kind = HistoryEntry.class;
-    this.ofyKey =
-        Key.create(Key.create(DomainBase.class, domainRepoId), HistoryEntry.class, domainHistoryId);
-    this.sqlKey = new DomainHistoryId(domainRepoId, domainHistoryId);
-    this.domainRepoId = domainRepoId;
-    this.domainHistoryId = domainHistoryId;
+  @Override
+  public Key<HistoryEntry> createOfyKey() {
+    return Key.create(Key.create(DomainBase.class, repoId), HistoryEntry.class, historyRevisionId);
   }
 
   /** Creates {@link DomainHistoryVKey} from the given {@link Key} instance. */
-  public static DomainHistoryVKey create(Key<HistoryEntry> ofyKey) {
+  public static DomainHistoryVKey create(Key<? extends HistoryEntry> ofyKey) {
     checkArgumentNotNull(ofyKey, "ofyKey must be specified");
-    String domainRepoId = ofyKey.getParent().getName();
-    long domainHistoryId = ofyKey.getId();
-    return new DomainHistoryVKey(domainRepoId, domainHistoryId);
+    String repoId = ofyKey.getParent().getName();
+    long historyRevisionId = ofyKey.getId();
+    return new DomainHistoryVKey(repoId, historyRevisionId);
+  }
+
+  public VKey<? extends HistoryEntry> createDomainHistoryVKey() {
+    return VKey.create(
+        DomainHistory.class,
+        createSqlKey(),
+        Key.create(Key.create(DomainBase.class, repoId), DomainHistory.class, historyRevisionId));
   }
 }

core/src/main/java/google/registry/persistence/EppHistoryVKey.java

@@ -0,0 +1,107 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import com.google.common.base.Joiner;
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.util.TypeUtils.TypeInstantiator;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
+
+/**
+ * Base class for {@link VKey} which ofyKey has a {@link HistoryEntry} key as its parent and a key
+ * for EPP resource as its grandparent.
+ *
+ * <p>For such a {@link VKey}, we need to provide two type parameters to indicate the type of {@link
+ * VKey} itself and the type of EPP resource respectively.
+ *
+ * @param <K> type of the {@link VKey}
+ * @param <E> type of the EPP resource that the key belongs to
+ */
+@MappedSuperclass
+@Access(AccessType.FIELD)
+public abstract class EppHistoryVKey<K, E extends EppResource> extends ImmutableObject
+    implements Serializable {
+
+  private static final long serialVersionUID = -3906580677709539818L;
+
+  String repoId;
+
+  Long historyRevisionId;
+
+  // Hibernate requires a default constructor.
+  EppHistoryVKey() {}
+
+  EppHistoryVKey(String repoId, long historyRevisionId) {
+    this.repoId = repoId;
+    this.historyRevisionId = historyRevisionId;
+  }
+
+  /**
+   * Returns the kind path for the ofyKey in this instance.
+   *
+   * <p>This method is only used reflectively by {@link EppHistoryVKeyTranslatorFactory} to get the
+   * kind path for a given {@link EppHistoryVKey} instance so it is marked as a private method.
+   *
+   * @see #createKindPath(Key)
+   */
+  @SuppressWarnings("unused")
+  private String getKindPath() {
+    String eppKind = Key.getKind(new TypeInstantiator<E>(getClass()) {}.getExactType());
+    String keyKind = Key.getKind(new TypeInstantiator<K>(getClass()) {}.getExactType());
+    if (keyKind.equals(Key.getKind(HistoryEntry.class))) {
+      return createKindPath(eppKind, keyKind);
+    } else {
+      return createKindPath(eppKind, Key.getKind(HistoryEntry.class), keyKind);
+    }
+  }
+
+  /**
+   * Creates the kind path for the given ofyKey}.
+   *
+   * <p>The kind path is a string including all kind names(delimited by slash) of a hierarchical
+   * {@link Key}, e.g., the kind path for BillingEvent.OneTime is "DomainBase/HistoryEntry/OneTime".
+   */
+  @Nullable
+  public static String createKindPath(@Nullable Key<?> ofyKey) {
+    if (ofyKey == null) {
+      return null;
+    } else if (ofyKey.getParent() == null) {
+      return ofyKey.getKind();
+    } else {
+      return createKindPath(createKindPath(ofyKey.getParent()), ofyKey.getKind());
+    }
+  }
+
+  private static String createKindPath(String... kinds) {
+    return Joiner.on("/").join(kinds);
+  }
+
+  /** Creates a {@link VKey} from this instance. */
+  public VKey<K> createVKey() {
+    Class<K> vKeyType = new TypeInstantiator<K>(getClass()) {}.getExactType();
+    return VKey.create(vKeyType, createSqlKey(), createOfyKey());
+  }
+
+  public abstract Object createSqlKey();
+
+  public abstract Key<K> createOfyKey();
+}

core/src/main/java/google/registry/persistence/PersistenceComponent.java

@@ -20,6 +20,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import javax.inject.Singleton;
 import javax.persistence.EntityManagerFactory;
@@ -32,6 +33,7 @@ import javax.persistence.EntityManagerFactory;
       CredentialModule.class,
       KmsModule.class,
       PersistenceModule.class,
+      SecretManagerModule.class,
       UtilsModule.class
     })
 public interface PersistenceComponent {

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -26,6 +26,7 @@ import com.google.api.client.auth.oauth2.Credential;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
@@ -33,6 +34,11 @@ import google.registry.keyring.kms.KmsKeyring;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotId;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
 import google.registry.tools.AuthModule.CloudSqlClientCredential;
 import google.registry.util.Clock;
 import java.lang.annotation.Documented;
@@ -47,6 +53,8 @@ import org.hibernate.cfg.Environment;
 /** Dagger module class for the persistence layer. */
 @Module
 public class PersistenceModule {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   // This name must be the same as the one defined in persistence.xml.
   public static final String PERSISTENCE_UNIT_NAME = "nomulus";
   public static final String HIKARI_CONNECTION_TIMEOUT = "hibernate.hikari.connectionTimeout";
@@ -122,11 +130,17 @@ public class PersistenceModule {
   static JpaTransactionManager provideAppEngineJpaTm(
       @Config("cloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -136,6 +150,7 @@ public class PersistenceModule {
   static JpaTransactionManager provideNomulusToolJpaTm(
       @Config("toolsCloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       @CloudSqlClientCredential Credential credential,
       Clock clock) {
@@ -143,6 +158,11 @@ public class PersistenceModule {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getToolsCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.TOOL),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -150,6 +170,7 @@ public class PersistenceModule {
   @Singleton
   @SocketFactoryJpaTm
   static JpaTransactionManager provideSocketFactoryJpaTm(
+      SqlCredentialStore credentialStore,
       @Config("beamCloudSqlUsername") String username,
       @Config("beamCloudSqlPassword") String password,
       @Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
@@ -159,6 +180,12 @@ public class PersistenceModule {
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, password);
     overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
+    // TODO(b/175700623): consider assigning different logins to pipelines
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -203,6 +230,30 @@ public class PersistenceModule {
     return emf;
   }
 
+  /** Verifies that the credential from the Secret Manager matches the one currently in use.
+   *
+   * <p>This is a helper for the transition to the Secret Manager, and will be removed once data
+   * and permissions are properly set up for all projects.
+   **/
+  private static void validateCredentialStore(
+      SqlCredentialStore credentialStore, SqlUser sqlUser, String login, String password) {
+    try {
+      SqlCredential credential = credentialStore.getCredential(sqlUser);
+      if (!credential.login().equals(login)) {
+        logger.atWarning().log(
+            "Wrong login for %s. Expecting %s, found %s.",
+            sqlUser.geUserName(), login, credential.login());
+        return;
+      }
+      if (!credential.password().equals(password)) {
+        logger.atWarning().log("Wrong password for %s.", sqlUser.geUserName());
+      }
+      logger.atWarning().log("Credentials in the kerying and the secret manager match.");
+    } catch (Throwable e) {
+      logger.atWarning().log(e.getMessage());
+    }
+  }
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
   @Qualifier
   @Documented

core/src/main/java/google/registry/persistence/WithLongVKey.java

@@ -31,4 +31,12 @@ public @interface WithLongVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/WithStringVKey.java

@@ -31,4 +31,12 @@ public @interface WithStringVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/converter/UpdateAutoTimestampConverter.java

@@ -31,7 +31,14 @@ public class UpdateAutoTimestampConverter
 
   @Override
   public Timestamp convertToDatabaseColumn(UpdateAutoTimestamp entity) {
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(jpaTm().getTransactionTime()).toInstant());
+    return Timestamp.from(
+        DateTimeUtils.toZonedDateTime(
+                UpdateAutoTimestamp.autoUpdateEnabled()
+                        || entity == null
+                        || entity.getTimestamp() == null
+                    ? jpaTm().getTransactionTime()
+                    : entity.getTimestamp())
+            .toInstant());
   }
 
   @Override

core/src/main/java/google/registry/persistence/converter/VKeyConverter.java

@@ -14,6 +14,7 @@
 
 package google.registry.persistence.converter;
 
+import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
@@ -29,7 +30,28 @@ public abstract class VKeyConverter<T, C> implements AttributeConverter<VKey<? e
   @Override
   @Nullable
   public VKey<? extends T> convertToEntityAttribute(@Nullable C dbData) {
-    return dbData == null ? null : VKey.createSql(getAttributeClass(), dbData);
+    if (dbData == null) {
+      return null;
+    }
+    Class<? extends T> clazz = getAttributeClass();
+    Key ofyKey = null;
+    if (!hasCompositeOfyKey()) {
+      // If this isn't a composite key, we can create the Ofy key from the SQL key.
+      ofyKey =
+          dbData instanceof String
+              ? Key.create(clazz, (String) dbData)
+              : Key.create(clazz, (Long) dbData);
+      return VKey.create(clazz, dbData, ofyKey);
+    } else {
+      // We don't know how to create the Ofy key and probably don't have everything necessary to do
+      // it anyway, so just create an asymmetric key - the containing object will have to convert it
+      // into a symmetric key.
+      return VKey.createSql(clazz, dbData);
+    }
+  }
+
+  protected boolean hasCompositeOfyKey() {
+    return false;
   }
 
   /** Returns the class of the attribute. */

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -29,6 +29,11 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig;
+import google.registry.model.ImmutableObject;
+import google.registry.model.index.EppResourceIndex;
+import google.registry.model.index.ForeignKeyIndex.ForeignKeyContactIndex;
+import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
+import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
 import google.registry.persistence.JpaRetries;
 import google.registry.persistence.VKey;
 import google.registry.util.Clock;
@@ -56,6 +61,18 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   private static final Retrier retrier = new Retrier(new SystemSleeper(), 3);
 
+  // The entity of classes in this set will be simply ignored when passed to modification
+  // operations, i.e. insert, put, update and delete. This is to help maintain a single code path
+  // when we switch from ofy() to tm() for the database migration as we don't need have a condition
+  // to exclude the Datastore specific entities when the underlying tm() is jpaTm().
+  // TODO(b/176108270): Remove this property after database migration.
+  private static final ImmutableSet<Class<? extends ImmutableObject>> IGNORED_ENTITY_CLASSES =
+      ImmutableSet.of(
+          EppResourceIndex.class,
+          ForeignKeyContactIndex.class,
+          ForeignKeyDomainIndex.class,
+          ForeignKeyHostIndex.class);
+
   // EntityManagerFactory is thread safe.
   private final EntityManagerFactory emf;
   private final Clock clock;
@@ -228,6 +245,9 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   @Override
   public void insert(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
+    if (isEntityOfIgnoredClass(entity)) {
+      return;
+    }
     assertInTransaction();
     getEntityManager().persist(entity);
     transactionInfo.get().addUpdate(entity);
@@ -253,6 +273,9 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   @Override
   public void put(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
+    if (isEntityOfIgnoredClass(entity)) {
+      return;
+    }
     assertInTransaction();
     getEntityManager().merge(entity);
     transactionInfo.get().addUpdate(entity);
@@ -278,6 +301,9 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   @Override
   public void update(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
+    if (isEntityOfIgnoredClass(entity)) {
+      return;
+    }
     assertInTransaction();
     checkArgument(exists(entity), "Given entity does not exist");
     getEntityManager().merge(entity);
@@ -414,6 +440,9 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
   @Override
   public void delete(Object entity) {
     checkArgumentNotNull(entity, "entity must be specified");
+    if (isEntityOfIgnoredClass(entity)) {
+      return;
+    }
     assertInTransaction();
     Object managedEntity = entity;
     if (!getEntityManager().contains(entity)) {
@@ -464,6 +493,10 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     }
   }
 
+  private static boolean isEntityOfIgnoredClass(Object entity) {
+    return IGNORED_ENTITY_CLASSES.contains(entity.getClass());
+  }
+
   private static ImmutableSet<EntityId> getEntityIdsFromEntity(
       EntityType<?> entityType, Object entity) {
     if (entityType.hasSingleIdAttribute()) {

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClient.java

@@ -22,6 +22,9 @@ import java.util.Optional;
 /** A Cloud Secret Manager client for Nomulus, bound to a specific GCP project. */
 public interface SecretManagerClient {
 
+  /** Returns the project name with which this client is associated. */
+  String getProject();
+
   /**
    * Creates a new secret in the Cloud Secret Manager with no data.
    *
@@ -32,6 +35,9 @@ public interface SecretManagerClient {
    */
   void createSecret(String secretId);
 
+  /** Checks if a secret with the given {@code secretId} already exists. */
+  boolean secretExists(String secretId);
+
   /** Returns all secret IDs in the Cloud Secret Manager. */
   Iterable<String> listSecrets();
 
@@ -67,6 +73,24 @@ public interface SecretManagerClient {
    */
   String getSecretData(String secretId, Optional<String> version);
 
+  /**
+   * Enables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void enableSecretVersion(String secretId, String version);
+
+  /**
+   * Disables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void disableSecretVersion(String secretId, String version);
+
   /**
    * Destroys a secret version.
    *

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClientImpl.java

@@ -29,12 +29,11 @@ import com.google.cloud.secretmanager.v1.SecretName;
 import com.google.cloud.secretmanager.v1.SecretPayload;
 import com.google.cloud.secretmanager.v1.SecretVersion;
 import com.google.cloud.secretmanager.v1.SecretVersionName;
-import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
 import com.google.protobuf.ByteString;
 import google.registry.util.Retrier;
 import java.util.Optional;
 import java.util.concurrent.Callable;
-import javax.inject.Inject;
 
 /** Implements {@link SecretManagerClient} on Google Cloud Platform. */
 public class SecretManagerClientImpl implements SecretManagerClient {
@@ -42,13 +41,17 @@ public class SecretManagerClientImpl implements SecretManagerClient {
   private final SecretManagerServiceClient csmClient;
   private final Retrier retrier;
 
-  @Inject
   SecretManagerClientImpl(String project, SecretManagerServiceClient csmClient, Retrier retrier) {
     this.project = project;
     this.csmClient = csmClient;
     this.retrier = retrier;
   }
 
+  @Override
+  public String getProject() {
+    return project;
+  }
+
   @Override
   public void createSecret(String secretId) {
     checkNotNull(secretId, "secretId");
@@ -57,12 +60,25 @@ public class SecretManagerClientImpl implements SecretManagerClient {
         () -> csmClient.createSecret(ProjectName.of(project), secretId, secretSettings));
   }
 
+  @Override
+  public boolean secretExists(String secretId) {
+    checkNotNull(secretId, "secretId");
+    try {
+      callSecretManager(() -> csmClient.getSecret(SecretName.of(project, secretId)));
+      return true;
+    } catch (NoSuchSecretResourceException e) {
+      return false;
+    }
+  }
+
   @Override
   public Iterable<String> listSecrets() {
     ListSecretsPagedResponse response =
         callSecretManager(() -> csmClient.listSecrets(ProjectName.of(project)));
-    return Iterables.transform(
-        response.iterateAll(), secret -> SecretName.parse(secret.getName()).getSecret());
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(secret -> SecretName.parse(secret.getName()).getSecret())
+            .iterator();
   }
 
   @Override
@@ -70,8 +86,10 @@ public class SecretManagerClientImpl implements SecretManagerClient {
     checkNotNull(secretId, "secretId");
     ListSecretVersionsPagedResponse response =
         callSecretManager(() -> csmClient.listSecretVersions(SecretName.of(project, secretId)));
-    return Iterables.transform(
-        response.iterateAll(), SecretManagerClientImpl::toSecretVersionState);
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(SecretManagerClientImpl::toSecretVersionState)
+            .iterator();
   }
 
   private static SecretVersionState toSecretVersionState(SecretVersion secretVersion) {
@@ -108,6 +126,22 @@ public class SecretManagerClientImpl implements SecretManagerClient {
                 .toStringUtf8());
   }
 
+  @Override
+  public void enableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.enableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
+  @Override
+  public void disableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.disableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
   @Override
   public void destroySecretVersion(String secretId, String version) {
     checkNotNull(secretId, "secretId");

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerModule.java

@@ -14,41 +14,47 @@
 
 package google.registry.privileges.secretmanager;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
-import dagger.Component;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
-import google.registry.util.UtilsModule;
 import java.io.IOException;
 import javax.inject.Singleton;
 
 /** Provides bindings for {@link SecretManagerClient}. */
 @Module
-public class SecretManagerModule {
-
-  private final String project;
-
-  public SecretManagerModule(String project) {
-    this.project = checkNotNull(project, "project");
-  }
+public abstract class SecretManagerModule {
 
   @Provides
   @Singleton
-  SecretManagerClient provideSecretManagerClient(Retrier retrier) {
+  static SecretManagerServiceSettings provideSecretManagerSetting(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle) {
     try {
-      return new SecretManagerClientImpl(project, SecretManagerServiceClient.create(), retrier);
+      return SecretManagerServiceSettings.newBuilder()
+          .setCredentialsProvider(() -> credentialsBundle.getGoogleCredentials())
+          .build();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  @Provides
   @Singleton
-  @Component(modules = {ConfigModule.class, SecretManagerModule.class, UtilsModule.class})
-  public interface SecretManagerComponent {
-    SecretManagerClient secretManagerClient();
+  static SecretManagerClient provideSecretManagerClient(
+      SecretManagerServiceSettings serviceSettings,
+      @Config("projectId") String project,
+      Retrier retrier) {
+    try {
+      SecretManagerServiceClient stub = SecretManagerServiceClient.create(serviceSettings);
+      Runtime.getRuntime().addShutdownHook(new Thread(stub::close));
+      return new SecretManagerClientImpl(project, stub, retrier);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
   }
 }

core/src/main/java/google/registry/privileges/secretmanager/SqlCredential.java

@@ -0,0 +1,55 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static avro.shaded.com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.value.AutoValue;
+import java.util.List;
+
+/**
+ * Contains the login name and password of a Cloud SQL user.
+ *
+ * <p>User must take care not to include the {@link #SEPARATOR} in property values.
+ */
+@AutoValue
+public abstract class SqlCredential {
+
+  public static final Character SEPARATOR = ' ';
+
+  public abstract String login();
+
+  public abstract String password();
+
+  @Override
+  public final String toString() {
+    // Use Object.toString(), which does not show object data.
+    return super.toString();
+  }
+
+  public final String toFormattedString() {
+    return String.format("%s%c%s", login(), SEPARATOR, password());
+  }
+
+  public static SqlCredential fromFormattedString(String sqlCredential) {
+    List<String> items = com.google.common.base.Splitter.on(SEPARATOR).splitToList(sqlCredential);
+    checkState(items.size() == 2, "Invalid SqlCredential string.");
+    return of(items.get(0), items.get(1));
+  }
+
+  public static SqlCredential of(String login, String password) {
+    return new AutoValue_SqlCredential(login, password);
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlCredentialStore.java

@@ -0,0 +1,118 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.cloud.secretmanager.v1.SecretVersionName;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Storage of SQL users' login credentials, backed by Cloud Secret Manager.
+ *
+ * <p>A user's credential is stored with one level of indirection using two secret IDs: Each version
+ * of the <em>credential data</em> is stored as follows: its secret ID is determined by {@link
+ * #getCredentialDataSecretId(SqlUser, String dbInstance)}, and the value of each version is a
+ * {@link SqlCredential}, serialized using {@link SqlCredential#toFormattedString}. The 'live'
+ * version of the credential is saved under the 'live pointer' secret explained below.
+ *
+ * <p>The pointer to the 'live' version of the credential data is stored as follows: its secret ID
+ * is determined by {@link #getLiveLabelSecretId(SqlUser, String dbInstance)}; and the value of each
+ * version is a {@link SecretVersionName} in String form, pointing to a version of the credential
+ * data. Only the 'latest' version of this secret should be used. It is guaranteed to be valid.
+ *
+ * <p>The indirection in credential storage makes it easy to handle failures in the credential
+ * change process.
+ */
+public class SqlCredentialStore {
+  private final SecretManagerClient csmClient;
+  private final String dbInstance;
+
+  @Inject
+  SqlCredentialStore(
+      SecretManagerClient csmClient, @Config("cloudSqlDbInstanceName") String dbInstance) {
+    this.csmClient = csmClient;
+    this.dbInstance = dbInstance;
+  }
+
+  public SqlCredential getCredential(SqlUser user) {
+    SecretVersionName credentialName = getLiveCredentialSecretVersion(user);
+    return SqlCredential.fromFormattedString(
+        csmClient.getSecretData(
+            credentialName.getSecret(), Optional.of(credentialName.getSecretVersion())));
+  }
+
+  public void createOrUpdateCredential(SqlUser user, String password) {
+    SecretVersionName dataName = saveCredentialData(user, password);
+    saveLiveLabel(user, dataName);
+  }
+
+  public void deleteCredential(SqlUser user) {
+    try {
+      csmClient.deleteSecret(getCredentialDataSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok
+    }
+    try {
+      csmClient.deleteSecret(getLiveLabelSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok.
+    }
+  }
+
+  private void createSecretIfAbsent(String secretId) {
+    try {
+      csmClient.createSecret(secretId);
+    } catch (SecretAlreadyExistsException ignore) {
+      // Not a problem.
+    }
+  }
+
+  private SecretVersionName saveCredentialData(SqlUser user, String password) {
+    String credentialDataSecretId = getCredentialDataSecretId(user, dbInstance);
+    createSecretIfAbsent(credentialDataSecretId);
+    String credentialVersion =
+        csmClient.addSecretVersion(
+            credentialDataSecretId,
+            SqlCredential.of(createDatabaseLoginName(user), password).toFormattedString());
+    return SecretVersionName.of(csmClient.getProject(), credentialDataSecretId, credentialVersion);
+  }
+
+  private void saveLiveLabel(SqlUser user, SecretVersionName dataVersionName) {
+    String liveLabelSecretId = getLiveLabelSecretId(user, dbInstance);
+    createSecretIfAbsent(liveLabelSecretId);
+    csmClient.addSecretVersion(liveLabelSecretId, dataVersionName.toString());
+  }
+
+  private SecretVersionName getLiveCredentialSecretVersion(SqlUser user) {
+    return SecretVersionName.parse(
+        csmClient.getSecretData(getLiveLabelSecretId(user, dbInstance), Optional.empty()));
+  }
+
+  private static String getLiveLabelSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-live-label-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  private static String getCredentialDataSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-data-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  // WIP: when b/170230882 is complete, login will be versioned.
+  private static String createDatabaseLoginName(SqlUser user) {
+    return user.geUserName();
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlUser.java

@@ -0,0 +1,67 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.common.base.Ascii;
+
+/**
+ * SQL user information for privilege management purposes.
+ *
+ * <p>A {@link RobotUser} represents a software system accessing the database using its own
+ * credential. Robots are well known and enumerated in {@link RobotId}.
+ */
+public abstract class SqlUser {
+
+  private final UserType type;
+  private final String userName;
+
+  protected SqlUser(UserType type, String userName) {
+    this.type = type;
+    this.userName = userName;
+  }
+
+  public UserType getType() {
+    return type;
+  }
+
+  public String geUserName() {
+    return userName;
+  }
+
+  /** Cloud SQL user types. Please see class javadoc of {@link SqlUser} for more information. */
+  enum UserType {
+    // Work in progress. Human user will be added.
+    ROBOT
+  }
+
+  /** Enumerates the {@link RobotUser RobotUsers} in the system. */
+  public enum RobotId {
+    NOMULUS,
+    /**
+     * Credential for RegistryTool. This is temporary, and will be removed when tool users are
+     * assigned their personal credentials.
+     */
+    TOOL;
+  }
+
+  /** Information of a RobotUser for privilege management purposes. */
+  // Work in progress. Eventually will be provided based on configuration.
+  public static class RobotUser extends SqlUser {
+
+    public RobotUser(RobotId robot) {
+      super(UserType.ROBOT, Ascii.toLowerCase(robot.name()));
+    }
+  }
+}

core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java

@@ -14,9 +14,11 @@
 
 package google.registry.reporting.spec11;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
@@ -59,6 +61,24 @@ public class Spec11RegistrarThreatMatchesParser {
     return getFromFile(getGcsFilename(date));
   }
 
+  /** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */
+  public ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
+      throws IOException {
+    if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) {
+      return ImmutableSet.of();
+    }
+    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename);
+        InputStreamReader isr = new InputStreamReader(in, UTF_8)) {
+      // Skip the header at line 0
+      return Splitter.on("\n")
+          .omitEmptyStrings()
+          .splitToStream(CharStreams.toString(isr))
+          .skip(1)
+          .map(this::parseRegistrarThreatMatch)
+          .collect(toImmutableSet());
+    }
+  }
+
   public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
     LocalDate yesterday = date.minusDays(1);
     GcsFilename gcsFilename = getGcsFilename(yesterday);
@@ -82,20 +102,6 @@ public class Spec11RegistrarThreatMatchesParser {
     return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
   }
 
-  private ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
-      throws IOException, JSONException {
-    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
-    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
-      ImmutableList<String> reportLines =
-          ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
-      // Iterate from 1 to size() to skip the header at line 0.
-      for (int i = 1; i < reportLines.size(); i++) {
-        builder.add(parseRegistrarThreatMatch(reportLines.get(i)));
-      }
-      return builder.build();
-    }
-  }
-
   private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
     JSONObject reportJSON = new JSONObject(line);
     String clientId = reportJSON.getString(Spec11Pipeline.REGISTRAR_CLIENT_ID_FIELD);

core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java

@@ -45,10 +45,10 @@ import google.registry.model.domain.DomainBase;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.OneTime entities with duplicate IDs.")
-public class DedupeOneTimeBillingEventIdsCommand extends DedupeEntityIdsCommand<OneTime> {
+public class DedupeOneTimeBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<OneTime> {
 
   @Override
-  void dedupe(OneTime entity) {
+  void process(OneTime entity) {
     Key<BillingEvent> key = Key.create(entity);
     Key<DomainBase> domainKey = getGrandParentAsDomain(key);
     DomainBase domain = ofy().load().key(domainKey).now();

core/src/main/java/google/registry/tools/DedupeRecurringBillingEventIdsCommand.java

@@ -54,11 +54,10 @@ import java.util.Set;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.Recurring entities with duplicate IDs.")
-public class DedupeRecurringBillingEventIdsCommand
-    extends DedupeEntityIdsCommand<BillingEvent.Recurring> {
+public class DedupeRecurringBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<Recurring> {
 
   @Override
-  void dedupe(Recurring recurring) {
+  void process(Recurring recurring) {
     // Loads the associated DomainBase and BillingEvent.OneTime entities that
     // may have reference to this BillingEvent.Recurring entity.
     Key<DomainBase> domainKey = getGrandParentAsDomain(Key.create(recurring));

core/src/main/java/google/registry/tools/DeleteAllocationTokensCommand.java

@@ -14,19 +14,19 @@
 
 package google.registry.tools;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Iterables.partition;
 import static com.google.common.collect.Streams.stream;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.token.AllocationToken;
+import google.registry.persistence.VKey;
 import java.util.List;
 
 /**
@@ -48,7 +48,7 @@ final class DeleteAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
   private static final int BATCH_SIZE = 20;
   private static final Joiner JOINER = Joiner.on(", ");
 
-  private ImmutableSet<Key<AllocationToken>> tokensToDelete;
+  private ImmutableSet<VKey<AllocationToken>> tokensToDelete;
 
   @Override
   public void init() {
@@ -71,26 +71,24 @@ final class DeleteAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
   }
 
   /** Deletes a (filtered) batch of AllocationTokens and returns how many were deleted. */
-  private long deleteBatch(List<Key<AllocationToken>> batch) {
+  private long deleteBatch(List<VKey<AllocationToken>> batch) {
     // Load the tokens in the same transaction as they are deleted to verify they weren't redeemed
     // since the query ran. This also filters out per-domain tokens if they're not to be deleted.
-    ImmutableSet<AllocationToken> tokensToDelete =
-        ofy().load().keys(batch).values().stream()
-            .filter(t -> withDomains || !t.getDomainName().isPresent())
+    ImmutableSet<VKey<AllocationToken>> tokensToDelete =
+        tm().load(batch).values().stream()
+            .filter(t -> withDomains || t.getDomainName().isEmpty())
             .filter(t -> SINGLE_USE.equals(t.getTokenType()))
             .filter(t -> !t.isRedeemed())
+            .map(AllocationToken::createVKey)
             .collect(toImmutableSet());
     if (!dryRun) {
-      ofy().delete().entities(tokensToDelete);
+      tm().delete(tokensToDelete);
     }
     System.out.printf(
         "%s tokens: %s\n",
         dryRun ? "Would delete" : "Deleted",
         JOINER.join(
-            tokensToDelete.stream()
-                .map(AllocationToken::getToken)
-                .sorted()
-                .collect(toImmutableSet())));
+            tokensToDelete.stream().map(VKey::getSqlKey).sorted().collect(toImmutableList())));
     return tokensToDelete.size();
   }
 }

core/src/main/java/google/registry/tools/GenerateAllocationTokensCommand.java

@@ -20,8 +20,8 @@ import static com.google.common.collect.Queues.newArrayDeque;
 import static com.google.common.collect.Sets.difference;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.StringGenerator.DEFAULT_PASSWORD_LENGTH;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -38,10 +38,10 @@ import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Streams;
 import com.google.common.io.Files;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationToken.TokenType;
+import google.registry.persistence.VKey;
 import google.registry.tools.params.TransitionListParameter.TokenStatusTransitions;
 import google.registry.util.CollectionUtils;
 import google.registry.util.NonFinalForTesting;
@@ -258,8 +258,13 @@ class GenerateAllocationTokensCommand implements CommandWithRemoteApi {
 
   @VisibleForTesting
   int saveTokens(final ImmutableSet<AllocationToken> tokens) {
-    Collection<AllocationToken> savedTokens =
-        dryRun ? tokens : tm().transact(() -> ofy().save().entities(tokens).now().values());
+    Collection<AllocationToken> savedTokens;
+    if (dryRun) {
+      savedTokens = tokens;
+    } else {
+      transactIfJpaTm(() -> tm().transact(() -> tm().putAll(tokens)));
+      savedTokens = tm().transact(() -> tm().loadAll(tokens));
+    }
     savedTokens.forEach(
         t -> System.out.println(SKIP_NULLS.join(t.getDomainName().orElse(null), t.getToken())));
     return savedTokens.size();
@@ -282,12 +287,14 @@ class GenerateAllocationTokensCommand implements CommandWithRemoteApi {
   }
 
   private ImmutableSet<String> getExistingTokenStrings(ImmutableSet<String> candidates) {
-    ImmutableSet<Key<AllocationToken>> existingTokenKeys =
+    ImmutableSet<VKey<AllocationToken>> existingTokenKeys =
         candidates.stream()
-            .map(input -> Key.create(AllocationToken.class, input))
+            .map(input -> VKey.create(AllocationToken.class, input))
             .collect(toImmutableSet());
-    return ofy().load().keys(existingTokenKeys).values().stream()
-        .map(AllocationToken::getToken)
-        .collect(toImmutableSet());
+    return transactIfJpaTm(
+        () ->
+            tm().load(existingTokenKeys).values().stream()
+                .map(AllocationToken::getToken)
+                .collect(toImmutableSet()));
   }
 }

core/src/main/java/google/registry/tools/GetAllocationTokenCommand.java

@@ -15,7 +15,6 @@
 package google.registry.tools;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.beust.jcommander.Parameter;
@@ -26,6 +25,7 @@ import com.google.common.collect.Lists;
 import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken;
+import google.registry.persistence.VKey;
 import java.util.Collection;
 import java.util.List;
 import java.util.Optional;
@@ -45,22 +45,26 @@ final class GetAllocationTokenCommand implements CommandWithRemoteApi {
   public void run() {
     ImmutableMap.Builder<String, AllocationToken> builder = new ImmutableMap.Builder<>();
     for (List<String> tokens : Lists.partition(mainParameters, BATCH_SIZE)) {
-      ImmutableList<Key<AllocationToken>> tokenKeys =
-          tokens.stream().map(t -> Key.create(AllocationToken.class, t)).collect(toImmutableList());
-      ofy().load().keys(tokenKeys).forEach((k, v) -> builder.put(k.getName(), v));
+      ImmutableList<VKey<AllocationToken>> tokenKeys =
+          tokens.stream()
+              .map(t -> VKey.create(AllocationToken.class, t))
+              .collect(toImmutableList());
+      tm().load(tokenKeys).forEach((k, v) -> builder.put(k.getSqlKey().toString(), v));
     }
     ImmutableMap<String, AllocationToken> loadedTokens = builder.build();
-    ImmutableMap<Key<DomainBase>, DomainBase> domains = loadRedeemedDomains(loadedTokens.values());
+    ImmutableMap<VKey<DomainBase>, DomainBase> domains = loadRedeemedDomains(loadedTokens.values());
 
     for (String token : mainParameters) {
       if (loadedTokens.containsKey(token)) {
         AllocationToken loadedToken = loadedTokens.get(token);
         System.out.println(loadedToken.toString());
-        if (!loadedToken.getRedemptionHistoryEntry().isPresent()) {
+        if (loadedToken.getRedemptionHistoryEntry().isEmpty()) {
           System.out.printf("Token %s was not redeemed.\n", token);
         } else {
+          Key<DomainBase> domainOfyKey =
+              loadedToken.getRedemptionHistoryEntry().get().getOfyKey().getParent();
           DomainBase domain =
-              domains.get(loadedToken.getRedemptionHistoryEntry().get().getOfyKey().getParent());
+              domains.get(VKey.create(DomainBase.class, domainOfyKey.getName(), domainOfyKey));
           if (domain == null) {
             System.out.printf("ERROR: Token %s was redeemed but domain can't be loaded.\n", token);
           } else {
@@ -76,19 +80,23 @@ final class GetAllocationTokenCommand implements CommandWithRemoteApi {
     }
   }
 
-  private static ImmutableMap<Key<DomainBase>, DomainBase> loadRedeemedDomains(
+  @SuppressWarnings("unchecked")
+  private static ImmutableMap<VKey<DomainBase>, DomainBase> loadRedeemedDomains(
       Collection<AllocationToken> tokens) {
-    ImmutableList<Key<DomainBase>> domainKeys =
+    ImmutableList<VKey<DomainBase>> domainKeys =
         tokens.stream()
             .map(AllocationToken::getRedemptionHistoryEntry)
             .filter(Optional::isPresent)
             .map(Optional::get)
             .map(key -> tm().load(key))
             .map(he -> (Key<DomainBase>) he.getParent())
+            .map(key -> VKey.create(DomainBase.class, key.getName(), key))
             .collect(toImmutableList());
-    ImmutableMap.Builder<Key<DomainBase>, DomainBase> domainsBuilder = new ImmutableMap.Builder<>();
-    for (List<Key<DomainBase>> keys : Lists.partition(domainKeys, BATCH_SIZE)) {
-      domainsBuilder.putAll(ofy().load().keys(keys));
+    ImmutableMap.Builder<VKey<DomainBase>, DomainBase> domainsBuilder =
+        new ImmutableMap.Builder<>();
+    for (List<VKey<DomainBase>> keys : Lists.partition(domainKeys, BATCH_SIZE)) {
+      tm().load(ImmutableList.copyOf(keys))
+          .forEach((k, v) -> domainsBuilder.put((VKey<DomainBase>) k, v));
     }
     return domainsBuilder.build();
   }

core/src/main/java/google/registry/tools/GetSqlCredentialCommand.java

@@ -0,0 +1,73 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.io.FileOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to get a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(separators = " =", commandDescription = "Get the Cloud SQL Credential for a given user")
+public class GetSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"-o", "--output"},
+      description = "Name of output file for key data.",
+      validateWith = PathParameter.OutputFile.class)
+  private Path outputPath = null;
+
+  @Inject
+  GetSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+
+    SqlCredential credential;
+    try {
+      credential = store.getCredential(sqlUser);
+    } catch (SecretManagerException e) {
+      System.out.println(e.getMessage());
+      return;
+    }
+
+    if (outputPath == null) {
+      System.out.printf("[%s]\n", credential.toFormattedString());
+      return;
+    }
+    try (FileOutputStream out = new FileOutputStream(outputPath.toFile())) {
+      out.write(credential.toFormattedString().getBytes(StandardCharsets.UTF_8));
+    }
+  }
+}

core/src/main/java/google/registry/tools/ReadEntityFromKeyPathCommand.java

@@ -33,8 +33,13 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.List;
 
-/** Base Command to dedupe entities with duplicate IDs. */
-abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
+/**
+ * Base Command to read entities from Datastore by their key paths retrieved from BigQuery.
+ *
+ * <p>The key path is the value of column __key__.path of the entity's BigQuery table. Its value is
+ * converted from the entity's key.
+ */
+abstract class ReadEntityFromKeyPathCommand<T> extends MutatingCommand {
 
   @Parameter(
       names = "--key_paths_file",
@@ -48,7 +53,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
 
   private StringBuilder changeMessage = new StringBuilder();
 
-  abstract void dedupe(T entity);
+  abstract void process(T entity);
 
   @Override
   protected void init() throws Exception {
@@ -69,7 +74,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
       }
       Class<T> clazz = new TypeInstantiator<T>(getClass()) {}.getExactType();
       if (clazz.isInstance(entity)) {
-        dedupe((T) entity);
+        process((T) entity);
       } else {
         throw new IllegalArgumentException("Unsupported entity key: " + untypedKey);
       }

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
+import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 
@@ -32,6 +33,7 @@ public final class RegistryTool {
       new ImmutableMap.Builder<String, Class<? extends Command>>()
           .put("ack_poll_messages", AckPollMessagesCommand.class)
           .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
+          .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
@@ -78,6 +80,7 @@ public final class RegistryTool {
           .put("get_routing_map", GetRoutingMapCommand.class)
           .put("get_schema", GetSchemaCommand.class)
           .put("get_schema_tree", GetSchemaTreeCommand.class)
+          .put("get_sql_credential", GetSqlCredentialCommand.class)
           .put("get_tld", GetTldCommand.class)
           .put("ghostryde", GhostrydeCommand.class)
           .put("hash_certificate", HashCertificateCommand.class)
@@ -99,12 +102,15 @@ public final class RegistryTool {
           .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)
           .put("remove_ip_address", RemoveIpAddressCommand.class)
+          .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class)
           .put("renew_domain", RenewDomainCommand.class)
           .put("resave_entities", ResaveEntitiesCommand.class)
           .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
           .put("resave_epp_resource", ResaveEppResourceCommand.class)
+          .put("save_sql_credential", SaveSqlCredentialCommand.class)
           .put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class)
           .put("set_num_instances", SetNumInstancesCommand.class)
+          .put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class)
           .put("setup_ote", SetupOteCommand.class)
           .put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class)
           .put("unlock_domain", UnlockDomainCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -34,6 +34,7 @@ import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.NomulusToolJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
@@ -74,6 +75,7 @@ import javax.inject.Singleton;
       PersistenceModule.class,
       RdeModule.class,
       RequestFactoryModule.class,
+      SecretManagerModule.class,
       URLFetchServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
@@ -118,6 +120,8 @@ interface RegistryToolComponent {
 
   void inject(GetOperationStatusCommand command);
 
+  void inject(GetSqlCredentialCommand command);
+
   void inject(GhostrydeCommand command);
 
   void inject(ImportDatastoreCommand command);
@@ -138,10 +142,14 @@ interface RegistryToolComponent {
 
   void inject(RenewDomainCommand command);
 
+  void inject(SaveSqlCredentialCommand command);
+
   void inject(SendEscrowReportToIcannCommand command);
 
   void inject(SetNumInstancesCommand command);
 
+  void inject(SetSqlReplayCheckpointCommand command);
+
   void inject(SetupOteCommand command);
 
   void inject(UnlockDomainCommand command);

core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameters;
+import com.googlecode.objectify.Key;
+import google.registry.model.domain.DomainBase;
+import google.registry.persistence.VKey;
+import javax.annotation.Nullable;
+import org.joda.time.DateTime;
+
+/** Command to remove the Registry 1.0 key in {@link DomainBase} entity. */
+@Parameters(separators = " =", commandDescription = "Remove .")
+public class RemoveRegistryOneKeyCommand extends ReadEntityFromKeyPathCommand<DomainBase> {
+
+  @Override
+  void process(DomainBase entity) {
+    // Assert that the DomainBase entity must be deleted before 2017-08-01(most of the problematic
+    // entities were deleted before 2017, though there are still a few entities deleted in 2017-07).
+    // This is because we finished the Registry 2.0 migration in 2017 and should not generate any
+    // Registry 1.0 key after it.
+    if (!entity.getDeletionTime().isBefore(DateTime.parse("2017-08-01T00:00:00Z"))) {
+      throw new IllegalStateException(
+          String.format(
+              "Entity's deletion time %s is not before 2017-08-01T00:00:00Z",
+              entity.getDeletionTime()));
+    }
+    boolean hasChange = false;
+    DomainBase.Builder domainBuilder = entity.asBuilder();
+    // We only found the registry 1.0 key existed in fields autorenewBillingEvent,
+    // autorenewPollMessage and deletePollMessage so we just need to check these fields for each
+    // entity.
+    if (isRegistryOneKey(entity.getAutorenewBillingEvent())) {
+      domainBuilder.setAutorenewBillingEvent(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getAutorenewPollMessage())) {
+      domainBuilder.setAutorenewPollMessage(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getDeletePollMessage())) {
+      domainBuilder.setDeletePollMessage(null);
+      hasChange = true;
+    }
+    if (hasChange) {
+      stageEntityChange(entity, domainBuilder.build());
+    }
+  }
+
+  private static boolean isRegistryOneKey(@Nullable VKey<?> vKey) {
+    if (vKey == null || vKey.getOfyKey() == null || vKey.getOfyKey().getParent() == null) {
+      return false;
+    }
+    Key<?> parentKey = vKey.getOfyKey().getParent();
+    return parentKey.getKind().equals("EntityGroupRoot") && parentKey.getName().equals("per-tld");
+  }
+}

core/src/main/java/google/registry/tools/SaveSqlCredentialCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to create or update a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(
+    separators = " =",
+    commandDescription = "Create or update the Cloud SQL Credential for a given user")
+public class SaveSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"--input"},
+      description =
+          "Name of input file for the password. If absent, command will prompt for "
+              + "password in console.",
+      validateWith = PathParameter.InputFile.class)
+  private Path inputPath = null;
+
+  @Inject
+  SaveSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    String password = getPassword();
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+    store.createOrUpdateCredential(sqlUser, password);
+    System.out.printf("\nDone:[%s]\n", password);
+  }
+
+  private String getPassword() throws Exception {
+    if (inputPath != null) {
+      return Files.readAllLines(inputPath, StandardCharsets.UTF_8).get(0);
+    }
+    return System.console().readLine("Please enter the password: ").trim();
+  }
+}

core/src/main/java/google/registry/tools/SetSqlReplayCheckpointCommand.java

@@ -0,0 +1,48 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.Iterables;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import java.util.List;
+import org.joda.time.DateTime;
+
+/** Command to set {@link SqlReplayCheckpoint} to a particular, post-initial-population time. */
+@Parameters(separators = " =", commandDescription = "Set SqlReplayCheckpoint to a particular time")
+public class SetSqlReplayCheckpointCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  @Parameter(description = "Time to which SqlReplayCheckpoint will be set", required = true)
+  List<DateTime> mainParameters;
+
+  @Override
+  protected String prompt() {
+    checkArgument(mainParameters.size() == 1, "Must provide exactly one DateTime to set");
+    return String.format(
+        "Set SqlReplayCheckpoint to %s?", Iterables.getOnlyElement(mainParameters));
+  }
+
+  @Override
+  protected String execute() {
+    DateTime dateTime = Iterables.getOnlyElement(mainParameters);
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime));
+    return String.format("Set SqlReplayCheckpoint time to %s", dateTime);
+  }
+}

core/src/main/java/google/registry/tools/UpdateAllocationTokensCommand.java

@@ -18,8 +18,8 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Iterables.partition;
 import static com.google.common.collect.Streams.stream;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -105,13 +105,17 @@ final class UpdateAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
     }
 
     tokensToSave =
-        ofy().load().keys(getTokenKeys()).values().stream()
-            .collect(toImmutableMap(Function.identity(), this::updateToken))
-            .entrySet()
-            .stream()
-            .filter(entry -> !entry.getKey().equals(entry.getValue())) // only update changed tokens
-            .map(Map.Entry::getValue)
-            .collect(toImmutableSet());
+        transactIfJpaTm(
+            () ->
+                tm().load(getTokenKeys()).values().stream()
+                    .collect(toImmutableMap(Function.identity(), this::updateToken))
+                    .entrySet()
+                    .stream()
+                    .filter(
+                        entry ->
+                            !entry.getKey().equals(entry.getValue())) // only update changed tokens
+                    .map(Map.Entry::getValue)
+                    .collect(toImmutableSet()));
   }
 
   @Override
@@ -123,7 +127,7 @@ final class UpdateAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
   protected String execute() {
     long numUpdated =
         stream(partition(tokensToSave, BATCH_SIZE))
-            .mapToLong(batch -> tm().transact(() -> saveBatch(batch)))
+            .mapToLong(batch -> tm().transact(() -> saveBatch(ImmutableList.copyOf(batch))))
             .sum();
     return String.format("Updated %d tokens in total.", numUpdated);
   }
@@ -141,9 +145,9 @@ final class UpdateAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
     return builder.build();
   }
 
-  private long saveBatch(List<AllocationToken> batch) {
+  private long saveBatch(ImmutableList<AllocationToken> batch) {
     if (!dryRun) {
-      ofy().save().entities(batch);
+      tm().putAll(batch);
     }
     System.out.printf(
         "%s tokens: %s\n",

core/src/main/java/google/registry/tools/UpdateOrDeleteAllocationTokensCommand.java

@@ -15,13 +15,15 @@
 package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 
 import com.beust.jcommander.Parameter;
 import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
 import google.registry.model.domain.token.AllocationToken;
+import google.registry.persistence.VKey;
 import java.util.List;
 
 /** Shared base class for commands to update or delete allocation tokens. */
@@ -47,19 +49,28 @@ abstract class UpdateOrDeleteAllocationTokensCommand extends ConfirmingCommand
       description = "Do not actually update or delete the tokens; defaults to false")
   protected boolean dryRun;
 
-  protected ImmutableSet<Key<AllocationToken>> getTokenKeys() {
+  protected ImmutableSet<VKey<AllocationToken>> getTokenKeys() {
     checkArgument(
         tokens == null ^ prefix == null,
         "Must provide one of --tokens or --prefix, not both / neither");
     if (tokens != null) {
-      return tokens.stream()
-          .map(token -> Key.create(AllocationToken.class, token))
-          .collect(toImmutableSet());
+      ImmutableSet<VKey<AllocationToken>> keys =
+          tokens.stream()
+              .map(token -> VKey.create(AllocationToken.class, token))
+              .collect(toImmutableSet());
+      ImmutableSet<VKey<AllocationToken>> nonexistentKeys =
+          transactIfJpaTm(
+              () -> keys.stream().filter(key -> !tm().exists(key)).collect(toImmutableSet()));
+      checkState(nonexistentKeys.isEmpty(), "Tokens with keys %s did not exist.", nonexistentKeys);
+      return keys;
     } else {
       checkArgument(!prefix.isEmpty(), "Provided prefix should not be blank");
-      return ofy().load().type(AllocationToken.class).keys().list().stream()
-          .filter(key -> key.getName().startsWith(prefix))
-          .collect(toImmutableSet());
+      return transactIfJpaTm(
+          () ->
+              tm().loadAll(AllocationToken.class).stream()
+                  .filter(token -> token.getToken().startsWith(prefix))
+                  .map(AllocationToken::createVKey)
+                  .collect(toImmutableSet()));
     }
   }
 }

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -0,0 +1,216 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.beam.spec11.ThreatMatch;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.model.reporting.Spec11ThreatMatchDao;
+import google.registry.reporting.spec11.RegistrarThreatMatches;
+import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.Clock;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+import javax.inject.Inject;
+import org.joda.time.LocalDate;
+
+/**
+ * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
+ *
+ * <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
+ * rough estimate of when we started using this format) and convert those RegistrarThreatMatches
+ * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
+ *
+ * <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
+ * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
+ * or if the domain objects were not active at the time of the scan.
+ */
+@Parameters(
+    commandDescription =
+        "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
+            + "Cloud SQL database.")
+public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
+
+  @Parameter(
+      names = {"-o", "--overwrite_existing_dates"},
+      description =
+          "Whether the command will overwrite data that already exists for dates that exist in the "
+              + "GCS bucket. Defaults to false.")
+  private boolean overrideExistingDates;
+
+  @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
+  // Inject the clock for testing purposes
+  @Inject Clock clock;
+
+  @Override
+  protected String prompt() {
+    return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
+  }
+
+  @Override
+  protected String execute() {
+    ImmutableList<LocalDate> dates = getDatesToBackfill();
+    ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
+        new ImmutableListMultimap.Builder<>();
+    for (LocalDate date : dates) {
+      try {
+        // It's OK if the file doesn't exist for a particular date; the result will be empty.
+        threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
+      } catch (IOException e) {
+        throw new RuntimeException(
+            String.format("Error parsing through file with date %s.", date), e);
+      }
+    }
+    ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
+        threatMatchesBuilder.build();
+    // Look up all possible DomainBases for these domain names, any of which can be in the past
+    ImmutableListMultimap<String, DomainBase> domainsByDomainName =
+        getDomainsByDomainName(threatMatches);
+
+    // For each date, convert all threat matches with the proper domain repo ID
+    int totalNumThreats = 0;
+    for (LocalDate date : threatMatches.keySet()) {
+      ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
+      for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
+        rtm.threatMatches().stream()
+            .map(
+                threatMatch ->
+                    threatMatchToCloudSqlObject(
+                        threatMatch, date, rtm.clientId(), domainsByDomainName))
+            .forEach(spec11ThreatsBuilder::add);
+      }
+      ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
+      jpaTm()
+          .transact(
+              () -> {
+                Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
+                jpaTm().putAll(spec11Threats);
+              });
+      totalNumThreats += spec11Threats.size();
+    }
+    return String.format(
+        "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
+  }
+
+  /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
+  private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
+      ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
+    return threatMatchesByDate.values().stream()
+        .map(RegistrarThreatMatches::threatMatches)
+        .flatMap(ImmutableList::stream)
+        .map(ThreatMatch::fullyQualifiedDomainName)
+        .distinct()
+        .collect(
+            flatteningToImmutableListMultimap(
+                Function.identity(),
+                (domainName) -> {
+                  List<DomainBase> domains =
+                      ofy()
+                          .load()
+                          .type(DomainBase.class)
+                          .filter("fullyQualifiedDomainName", domainName)
+                          .list();
+                  domains.sort(Comparator.comparing(DomainBase::getCreationTime).reversed());
+                  checkState(
+                      !domains.isEmpty(),
+                      "Domain name %s had no associated DomainBase objects.",
+                      domainName);
+                  return domains.stream();
+                }));
+  }
+
+  /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
+  private Spec11ThreatMatch threatMatchToCloudSqlObject(
+      ThreatMatch threatMatch,
+      LocalDate date,
+      String registrarId,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    DomainBase domain =
+        findDomainAsOfDateOrThrow(
+            threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
+    return new Spec11ThreatMatch.Builder()
+        .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
+        .setCheckDate(date)
+        .setRegistrarId(registrarId)
+        .setDomainName(threatMatch.fullyQualifiedDomainName())
+        .setDomainRepoId(domain.getRepoId())
+        .build();
+  }
+
+  /** Returns the DomainBase object as of the particular date, which is likely in the past. */
+  private DomainBase findDomainAsOfDateOrThrow(
+      String domainName,
+      LocalDate date,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
+    for (DomainBase domain : domains) {
+      // We only know the date (not datetime) of the threat scan, so we approximate
+      LocalDate creationDate = domain.getCreationTime().toLocalDate();
+      LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
+      if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
+        return domain;
+      }
+    }
+    throw new IllegalStateException(
+        String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
+  }
+
+  /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
+  private ImmutableList<LocalDate> getDatesToBackfill() {
+    ImmutableSet<LocalDate> datesToSkip =
+        overrideExistingDates ? ImmutableSet.of() : getExistingDates();
+    ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
+    LocalDate endDate = clock.nowUtc().toLocalDate();
+    for (LocalDate currentDate = START_DATE;
+        !currentDate.isAfter(endDate);
+        currentDate = currentDate.plusDays(1)) {
+      if (!datesToSkip.contains(currentDate)) {
+        result.add(currentDate);
+      }
+    }
+    return result.build();
+  }
+
+  private ImmutableSet<LocalDate> getExistingDates() {
+    return jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .getEntityManager()
+                    .createQuery(
+                        "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
+                    .getResultStream()
+                    .collect(toImmutableSet()));
+  }
+}

core/src/test/java/google/registry/batch/ResaveAllEppResourcesActionTest.java

@@ -55,6 +55,19 @@ class ResaveAllEppResourcesActionTest extends MapreduceTestCase<ResaveAllEppReso
         .isGreaterThan(creationTime);
   }
 
+  @Test
+  void test_fastMode_doesNotResaveEntityWithNoChanges() throws Exception {
+    ContactResource contact = persistActiveContact("test123");
+    DateTime creationTime = contact.getUpdateTimestamp().getTimestamp();
+    assertThat(ofy().load().entity(contact).now().getUpdateTimestamp().getTimestamp())
+        .isEqualTo(creationTime);
+    ofy().clearSessionCache();
+    action.isFast = true;
+    runMapreduce();
+    assertThat(ofy().load().entity(contact).now().getUpdateTimestamp().getTimestamp())
+        .isEqualTo(creationTime);
+  }
+
   @Test
   void test_mapreduceResolvesPendingTransfer() throws Exception {
     DateTime now = DateTime.now(UTC);

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -20,7 +20,6 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.withSettings;
 
@@ -285,8 +284,7 @@ class Spec11PipelineTest {
             .build();
 
     verify(mockJpaTm).transact(any(Runnable.class));
-    verify(mockJpaTm).insert(expected);
-    verifyNoMoreInteractions(mockJpaTm);
+    verify(mockJpaTm).putAll(ImmutableList.of(expected));
   }
 
   /**

core/src/test/java/google/registry/flows/FlowTestCase.java

@@ -100,7 +100,11 @@ public abstract class FlowTestCase<F extends Flow> {
 
   @RegisterExtension
   final AppEngineExtension appEngine =
-      AppEngineExtension.builder().withDatastoreAndCloudSql().withTaskQueue().build();
+      AppEngineExtension.builder()
+          .withClock(clock)
+          .withDatastoreAndCloudSql()
+          .withTaskQueue()
+          .build();
 
   @BeforeEach
   public void beforeEachFlowTestCase() {
@@ -288,7 +292,7 @@ public abstract class FlowTestCase<F extends Flow> {
           e);
     }
     // Clear the cache so that we don't see stale results in tests.
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     return output;
   }
 

core/src/test/java/google/registry/flows/ResourceFlowTestCase.java

@@ -19,6 +19,8 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.tmch.ClaimsListShardTest.createTestClaimsListShard;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.LogsSubject.assertAboutLogs;
 import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
@@ -72,7 +74,7 @@ public abstract class ResourceFlowTestCase<F extends Flow, R extends EppResource
   protected R reloadResourceByForeignKey(DateTime now) throws Exception {
     // Force the session to be cleared so that when we read it back, we read from Datastore and not
     // from the transaction's session cache.
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     return loadByForeignKey(getResourceClass(), getUniqueIdFromCommand(), now).orElse(null);
   }
 
@@ -83,9 +85,9 @@ public abstract class ResourceFlowTestCase<F extends Flow, R extends EppResource
 
   protected <T extends EppResource> T reloadResourceAndCloneAtTime(T resource, DateTime now) {
     // Force the session to be cleared.
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     @SuppressWarnings("unchecked")
-    T refreshedResource = (T) ofy().load().entity(resource).now().cloneProjectedAtTime(now);
+    T refreshedResource = (T) transactIfJpaTm(() -> tm().load(resource)).cloneProjectedAtTime(now);
     return refreshedResource;
   }
 

core/src/test/java/google/registry/flows/domain/DomainCheckFlowTest.java

@@ -71,7 +71,6 @@ import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.model.registry.label.ReservedList;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -169,7 +168,7 @@ class DomainCheckFlowTest extends ResourceCheckFlowTestCase<DomainCheckFlow, Dom
         new AllocationToken.Builder()
             .setToken("abc123")
             .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
             .build());
     doCheckTest(
         create(false, "example1.tld", "In use"),

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -162,7 +162,6 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.monitoring.whitebox.EppMetric;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.math.BigDecimal;
@@ -505,7 +504,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
         new AllocationToken.Builder()
             .setToken("abc123")
             .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
             .build());
     clock.advanceOneMilli();
     EppException thrown =
@@ -528,7 +527,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     HistoryEntry historyEntry =
         ofy().load().type(HistoryEntry.class).ancestor(reloadResourceByForeignKey()).first().now();
     assertThat(ofy().load().entity(token).now().getRedemptionHistoryEntry())
-        .hasValue(DomainHistoryVKey.create(Key.create(historyEntry)));
+        .hasValue(HistoryEntry.createVKey(Key.create(historyEntry)));
   }
 
   @Test
@@ -1275,7 +1274,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     assertThat(reloadedToken.isRedeemed()).isTrue();
     assertThat(reloadedToken.getRedemptionHistoryEntry())
         .hasValue(
-            DomainHistoryVKey.create(
+            HistoryEntry.createVKey(
                 Key.create(getHistoryEntries(reloadResourceByForeignKey()).get(0))));
   }
 

core/src/test/java/google/registry/flows/domain/token/AllocationTokenFlowUtilsTest.java

@@ -49,7 +49,6 @@ import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.testing.AppEngineExtension;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
@@ -198,7 +197,7 @@ class AllocationTokenFlowUtilsTest {
         new AllocationToken.Builder()
             .setToken("tokeN")
             .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
             .build());
     assertThat(
             flowUtils

core/src/test/java/google/registry/flows/host/HostCheckFlowTest.java

@@ -24,16 +24,18 @@ import google.registry.flows.EppException;
 import google.registry.flows.ResourceCheckFlowTestCase;
 import google.registry.flows.exceptions.TooManyResourceChecksException;
 import google.registry.model.host.HostResource;
-import org.junit.jupiter.api.Test;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 
 /** Unit tests for {@link HostCheckFlow}. */
+@DualDatabaseTest
 class HostCheckFlowTest extends ResourceCheckFlowTestCase<HostCheckFlow, HostResource> {
 
   HostCheckFlowTest() {
     setEppInput("host_check.xml");
   }
 
-  @Test
+  @TestOfyAndSql
   void testNothingExists() throws Exception {
     // These ids come from the check xml.
     doCheckTest(
@@ -42,7 +44,7 @@ class HostCheckFlowTest extends ResourceCheckFlowTestCase<HostCheckFlow, HostRes
         create(true, "ns3.example.tld", null));
   }
 
-  @Test
+  @TestOfyAndSql
   void testOneExists() throws Exception {
     persistActiveHost("ns1.example.tld");
     // These ids come from the check xml.
@@ -52,7 +54,7 @@ class HostCheckFlowTest extends ResourceCheckFlowTestCase<HostCheckFlow, HostRes
         create(true, "ns3.example.tld", null));
   }
 
-  @Test
+  @TestOfyAndSql
   void testOneExistsButWasDeleted() throws Exception {
     persistDeletedHost("ns1.example.tld", clock.nowUtc().minusDays(1));
     // These ids come from the check xml.
@@ -62,27 +64,27 @@ class HostCheckFlowTest extends ResourceCheckFlowTestCase<HostCheckFlow, HostRes
         create(true, "ns3.example.tld", null));
   }
 
-  @Test
+  @TestOfyAndSql
   void testXmlMatches() throws Exception {
     persistActiveHost("ns2.example.tld");
     runFlowAssertResponse(loadFile("host_check_response.xml"));
   }
 
-  @Test
+  @TestOfyAndSql
   void test50IdsAllowed() throws Exception {
     // Make sure we don't have a regression that reduces the number of allowed checks.
     setEppInput("host_check_50.xml");
     runFlow();
   }
 
-  @Test
+  @TestOfyAndSql
   void testTooManyIds() {
     setEppInput("host_check_51.xml");
     EppException thrown = assertThrows(TooManyResourceChecksException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testIcannActivityReportField_getsLogged() throws Exception {
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-host-check");

core/src/test/java/google/registry/flows/host/HostCreateFlowTest.java

@@ -16,6 +16,7 @@ package google.registry.flows.host;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.assertNoBillingEvents;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.createTlds;
@@ -53,10 +54,12 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import org.joda.time.DateTime;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link HostCreateFlow}. */
+@DualDatabaseTest
 class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResource> {
 
   private void setEppHostCreateInput(String hostName, String hostAddrs) {
@@ -90,7 +93,9 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
         .hasOnlyOneHistoryEntryWhich()
         .hasType(HistoryEntry.Type.HOST_CREATE);
     assertNoBillingEvents();
-    assertEppResourceIndexEntityFor(reloadResourceByForeignKey());
+    if (tm().isOfy()) {
+      assertEppResourceIndexEntityFor(reloadResourceByForeignKey());
+    }
   }
 
   private void doSuccessfulInternalTest(String tld) throws Exception {
@@ -100,19 +105,19 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     doSuccessfulTest();
   }
 
-  @Test
+  @TestOfyAndSql
   void testDryRun() throws Exception {
     dryRunFlowAssertResponse(loadFile("host_create_response.xml"));
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_externalNeverExisted() throws Exception {
     doSuccessfulTest();
     assertAboutHosts().that(reloadResourceByForeignKey()).hasSuperordinateDomain(null);
     assertNoDnsTasksEnqueued();
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_internalNeverExisted() throws Exception {
     doSuccessfulInternalTest("tld");
     HostResource host = reloadResourceByForeignKey();
@@ -123,7 +128,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertDnsTasksEnqueued("ns1.example.tld");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_multipartTLDsAndInvalidHost() {
     createTlds("bar.tld", "tld");
 
@@ -132,7 +137,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_externalExistedButWasDeleted() throws Exception {
     persistDeletedHost(getUniqueIdFromCommand(), clock.nowUtc().minusDays(1));
     doSuccessfulTest();
@@ -140,7 +145,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertNoDnsTasksEnqueued();
   }
 
-  @Test
+  @TestOfyAndSql
   void testSuccess_internalExistedButWasDeleted() throws Exception {
     persistDeletedHost(getUniqueIdFromCommand(), clock.nowUtc().minusDays(1));
     doSuccessfulInternalTest("tld");
@@ -152,7 +157,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertDnsTasksEnqueued("ns1.example.tld");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_subordinateNeedsIps() {
     setEppHostCreateInput("ns1.example.tld", null);
     createTld("tld");
@@ -161,7 +166,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_externalMustNotHaveIps() {
     setEppHostCreateInputWithIps("ns1.example.external");
     createTld("tld");
@@ -170,7 +175,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_superordinateMissing() {
     setEppHostCreateInput("ns1.example.tld", null);
     createTld("tld");
@@ -179,7 +184,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertThat(thrown).hasMessageThat().contains("(example.tld)");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_superordinateInPendingDelete() {
     setEppHostCreateInputWithIps("ns1.example.tld");
     createTld("tld");
@@ -197,7 +202,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
         .contains("Superordinate domain for this hostname is in pending delete");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_alreadyExists() throws Exception {
     setEppHostCreateInput("ns1.example.tld", null);
     persistActiveHost(getUniqueIdFromCommand());
@@ -209,7 +214,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
             String.format("Object with given ID (%s) already exists", getUniqueIdFromCommand()));
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_resourceContention() throws Exception {
     setEppHostCreateInput("ns1.example.tld", null);
     String targetId = getUniqueIdFromCommand();
@@ -226,14 +231,14 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_nonLowerCaseHostname() {
     setEppHostCreateInput("ns1.EXAMPLE.tld", null);
     EppException thrown = assertThrows(HostNameNotLowerCaseException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_nonPunyCodedHostname() {
     setEppHostCreateInput("ns1.çauçalito.みんな", null);
     HostNameNotPunyCodedException thrown =
@@ -241,21 +246,21 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertThat(thrown).hasMessageThat().contains("expected ns1.xn--aualito-txac.xn--q9jyb4c");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_nonCanonicalHostname() {
     setEppHostCreateInput("ns1.example.tld.", null);
     EppException thrown = assertThrows(HostNameNotNormalizedException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_longHostName() {
     setEppHostCreateInputWithIps("a" + Strings.repeat(".labelpart", 25) + ".tld");
     EppException thrown = assertThrows(HostNameTooLongException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_ip4AddressWithIp6Declaration() {
     setEppHostCreateInput(
         "ns1.example.tld",
@@ -272,37 +277,37 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_badCharacter() {
     doFailingHostNameTest("foo bar", InvalidHostNameException.class);
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_tooShallowPublicSuffix() {
     doFailingHostNameTest("example.tld", HostNameTooShallowException.class);
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_tooShallowCcTld() {
     doFailingHostNameTest("foo.co.uk", HostNameTooShallowException.class);
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_barePublicSuffix() {
     doFailingHostNameTest("com", HostNameTooShallowException.class);
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_bareCcTld() {
     doFailingHostNameTest("co.uk", HostNameTooShallowException.class);
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_tooShallowNewTld() {
     doFailingHostNameTest("example.lol", HostNameTooShallowException.class);
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_ccTldInBailiwick() {
     createTld("co.uk");
     setEppHostCreateInputWithIps("foo.co.uk");
@@ -310,7 +315,7 @@ class HostCreateFlowTest extends ResourceFlowTestCase<HostCreateFlow, HostResour
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
-  @Test
+  @TestOfyAndSql
   void testIcannActivityReportField_getsLogged() throws Exception {
     runFlow();
     assertIcannReportingActivityFieldLogged("srs-host-create");

core/src/test/java/google/registry/flows/host/HostFlowUtilsTest.java

@@ -26,66 +26,68 @@ import google.registry.flows.host.HostFlowUtils.HostNameTooLongException;
 import google.registry.flows.host.HostFlowUtils.HostNameTooShallowException;
 import google.registry.flows.host.HostFlowUtils.InvalidHostNameException;
 import google.registry.testing.AppEngineExtension;
-import org.junit.jupiter.api.Test;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link HostFlowUtils}. */
+@DualDatabaseTest
 class HostFlowUtilsTest {
 
   @RegisterExtension
   final AppEngineExtension appEngine =
       AppEngineExtension.builder().withDatastoreAndCloudSql().build();
 
-  @Test
+  @TestOfyAndSql
   void test_validExternalHostName_validates() throws Exception {
     assertThat(validateHostName("host.example.com").toString()).isEqualTo("host.example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validExternalHostNameOnRegistrySuffixList_validates() throws Exception {
     assertThat(validateHostName("host.blogspot.com").toString()).isEqualTo("host.blogspot.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validExternalHostNameOnRegistrySuffixList_multipartTLD_validates() throws Exception {
     assertThat(validateHostName("ns1.host.co.uk").toString()).isEqualTo("ns1.host.co.uk");
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validExternalHostNameOnRegistrySuffixList_multipartTLD_tooShallow() {
     assertThrows(
         HostNameTooShallowException.class, () -> validateHostName("host.co.uk").toString());
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validateHostName_hostNameTooLong() {
     assertThrows(
         HostNameTooLongException.class,
         () -> validateHostName(Strings.repeat("na", 200) + ".wat.man"));
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validateHostName_hostNameNotLowerCase() {
     assertThrows(HostNameNotLowerCaseException.class, () -> validateHostName("NA.CAPS.TLD"));
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validateHostName_hostNameNotPunyCoded() {
     assertThrows(
         HostNameNotPunyCodedException.class, () -> validateHostName("motörhead.death.metal"));
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validateHostName_hostNameNotNormalized() {
     assertThrows(HostNameNotNormalizedException.class, () -> validateHostName("root.node.yeah."));
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validateHostName_hostNameHasLeadingHyphen() {
     assertThrows(InvalidHostNameException.class, () -> validateHostName("-giga.mega.tld"));
   }
 
-  @Test
+  @TestOfyAndSql
   void test_validateHostName_hostNameTooShallow() {
     assertThrows(HostNameTooShallowException.class, () -> validateHostName("domain.tld"));
   }

core/src/test/java/google/registry/model/EntityTestCase.java

@@ -28,6 +28,7 @@ import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.annotation.Serialize;
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.ofy.Ofy;
+import google.registry.persistence.EppHistoryVKey;
 import google.registry.persistence.VKey;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
@@ -173,7 +174,8 @@ public abstract class EntityTestCase {
           }
           // Descend into persisted ImmutableObject classes, but not anything else.
           if (ImmutableObject.class.isAssignableFrom(fieldClass)
-              && !VKey.class.isAssignableFrom(fieldClass)) {
+              && !VKey.class.isAssignableFrom(fieldClass)
+              && !EppHistoryVKey.class.isAssignableFrom(fieldClass)) {
             getAllPotentiallyIndexedFieldPaths(fieldClass).stream()
                 .map(subfield -> field.getName() + "." + subfield)
                 .distinct()

core/src/test/java/google/registry/model/EppResourceUtilsTest.java

@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.EppResourceUtils.loadAtPointInTime;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.newHostResource;
+import static google.registry.testing.DatabaseHelper.persistNewRegistrars;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.DatabaseHelper.persistResourceWithCommitLog;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -26,16 +27,19 @@ import static org.joda.time.DateTimeZone.UTC;
 import google.registry.model.host.HostResource;
 import google.registry.model.ofy.Ofy;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Tests for {@link EppResourceUtils}. */
-public class EppResourceUtilsTest {
+@DualDatabaseTest
+class EppResourceUtilsTest {
 
   @RegisterExtension
   public final AppEngineExtension appEngine =
@@ -51,7 +55,7 @@ public class EppResourceUtilsTest {
     inject.setStaticField(Ofy.class, "clock", clock);
   }
 
-  @Test
+  @TestOfyAndSql
   void testLoadAtPointInTime_beforeCreated_returnsNull() {
     clock.advanceOneMilli();
     // Don't save a commit log, we shouldn't need one.
@@ -62,7 +66,7 @@ public class EppResourceUtilsTest {
     assertThat(loadAtPointInTime(host, clock.nowUtc().minus(Duration.millis(1))).now()).isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testLoadAtPointInTime_atOrAfterLastAutoUpdateTime_returnsResource() {
     clock.advanceOneMilli();
     // Don't save a commit log, we shouldn't need one.
@@ -73,8 +77,9 @@ public class EppResourceUtilsTest {
     assertThat(loadAtPointInTime(host, clock.nowUtc()).now()).isEqualTo(host);
   }
 
-  @Test
+  @TestOfyOnly
   void testLoadAtPointInTime_usingIntactRevisionHistory_returnsMutationValue() {
+    persistNewRegistrars("OLD", "NEW");
     clock.advanceOneMilli();
     // Save resource with a commit log that we can read in later as a revisions map value.
     HostResource oldHost = persistResourceWithCommitLog(
@@ -94,7 +99,7 @@ public class EppResourceUtilsTest {
         .isEqualTo(oldHost);
   }
 
-  @Test
+  @TestOfyOnly
   void testLoadAtPointInTime_brokenRevisionHistory_returnsResourceAsIs() {
     // Don't save a commit log since we want to test the handling of a broken revisions key.
     HostResource oldHost = persistResource(
@@ -114,7 +119,7 @@ public class EppResourceUtilsTest {
     assertThat(loadAtPointInTime(host, clock.nowUtc().minusMillis(1)).now()).isEqualTo(host);
   }
 
-  @Test
+  @TestOfyOnly
   void testLoadAtPointInTime_fallback_returnsMutationValueForOldestRevision() {
     clock.advanceOneMilli();
     // Save a commit log that we can fall back to.
@@ -136,7 +141,7 @@ public class EppResourceUtilsTest {
         .isEqualTo(oldHost);
   }
 
-  @Test
+  @TestOfyOnly
   void testLoadAtPointInTime_ultimateFallback_onlyOneRevision_returnsCurrentResource() {
     clock.advanceOneMilli();
     // Don't save a commit log; we want to test that we load from the current resource.
@@ -151,7 +156,7 @@ public class EppResourceUtilsTest {
     assertThat(loadAtPointInTime(host, clock.nowUtc().minusMillis(1)).now()).isEqualTo(host);
   }
 
-  @Test
+  @TestOfyOnly
   void testLoadAtPointInTime_moreThanThirtyDaysInPast_historyIsPurged() {
     clock.advanceOneMilli();
     HostResource host =

core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java

@@ -15,64 +15,106 @@
 package google.registry.model;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.joda.time.DateTimeZone.UTC;
 
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.persistence.VKey;
 import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.FakeClock;
+import google.registry.testing.TestOfyAndSql;
 import org.joda.time.DateTime;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link UpdateAutoTimestamp}. */
+@DualDatabaseTest
 public class UpdateAutoTimestampTest {
 
+  FakeClock clock = new FakeClock();
+
   @RegisterExtension
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder()
           .withDatastoreAndCloudSql()
+          .withJpaUnitTestEntities(UpdateAutoTimestampTestObject.class)
           .withOfyTestEntities(UpdateAutoTimestampTestObject.class)
+          .withClock(clock)
           .build();
 
   /** Timestamped class. */
   @Entity(name = "UatTestEntity")
+  @javax.persistence.Entity
   @EntityForTesting
   public static class UpdateAutoTimestampTestObject extends CrossTldSingleton {
+    @Ignore @javax.persistence.Id long id = SINGLETON_ID;
     UpdateAutoTimestamp updateTime = UpdateAutoTimestamp.create(null);
   }
 
   private UpdateAutoTimestampTestObject reload() {
-    return ofy().load().entity(new UpdateAutoTimestampTestObject()).now();
+    return tm().transact(
+            () ->
+                tm().load(
+                        VKey.create(
+                            UpdateAutoTimestampTestObject.class,
+                            1L,
+                            Key.create(new UpdateAutoTimestampTestObject()))));
   }
 
-  @Test
+  @TestOfyAndSql
   void testSaveSetsTime() {
     DateTime transactionTime =
         tm().transact(
                 () -> {
                   UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                   assertThat(object.updateTime.timestamp).isNull();
-                  ofy().save().entity(object);
+                  tm().insert(object);
                   return tm().getTransactionTime();
                 });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
   }
 
-  @Test
+  @TestOfyAndSql
+  void testDisabledUpdates() throws Exception {
+    DateTime initialTime =
+        tm().transact(
+                () -> {
+                  tm().insert(new UpdateAutoTimestampTestObject());
+                  return tm().getTransactionTime();
+                });
+
+    UpdateAutoTimestampTestObject object = reload();
+    clock.advanceOneMilli();
+
+    try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+        new UpdateAutoTimestamp.DisableAutoUpdateResource()) {
+      DateTime secondTransactionTime =
+          tm().transact(
+                  () -> {
+                    tm().put(object);
+                    return tm().getTransactionTime();
+                  });
+      assertThat(secondTransactionTime).isGreaterThan(initialTime);
+    }
+    assertThat(reload().updateTime.timestamp).isEqualTo(initialTime);
+  }
+
+  @TestOfyAndSql
   void testResavingOverwritesOriginalTime() {
     DateTime transactionTime =
         tm().transact(
                 () -> {
                   UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                   object.updateTime = UpdateAutoTimestamp.create(DateTime.now(UTC).minusDays(1));
-                  ofy().save().entity(object);
+                  tm().insert(object);
                   return tm().getTransactionTime();
                 });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
   }
 }

core/src/test/java/google/registry/model/domain/DomainBaseSqlTest.java

@@ -84,10 +84,10 @@ public class DomainBaseSqlTest {
     saveRegistrar("registrar1");
     saveRegistrar("registrar2");
     saveRegistrar("registrar3");
-    contactKey = VKey.createSql(ContactResource.class, "contact_id1");
-    contact2Key = VKey.createSql(ContactResource.class, "contact_id2");
+    contactKey = createKey(ContactResource.class, "contact_id1");
+    contact2Key = createKey(ContactResource.class, "contact_id2");
 
-    host1VKey = VKey.createSql(HostResource.class, "host1");
+    host1VKey = createKey(HostResource.class, "host1");
 
     domain =
         new DomainBase.Builder()
@@ -633,14 +633,13 @@ public class DomainBaseSqlTest {
                           "4-COM",
                           END_OF_TIME,
                           "registrar1",
-                          createLegacyVKey(
-                              BillingEvent.OneTime.class, oneTimeBillingEvent.getId())),
+                          oneTimeBillingEvent.createVKey()),
                       GracePeriod.createForRecurring(
                           GracePeriodStatus.AUTO_RENEW,
                           "4-COM",
                           END_OF_TIME,
                           "registrar1",
-                          createLegacyVKey(BillingEvent.Recurring.class, billEvent.getId())));
+                          billEvent.createVKey()));
 
               jpaTm().insert(contact);
               jpaTm().insert(contact2);
@@ -696,6 +695,10 @@ public class DomainBaseSqlTest {
     assertThat(domain.getGracePeriods()).isEqualTo(gracePeriods);
   }
 
+  private <T> VKey<T> createKey(Class<T> clazz, String name) {
+    return VKey.create(clazz, name, Key.create(clazz, name));
+  }
+
   private <T> VKey<T> createLegacyVKey(Class<T> clazz, long id) {
     return VKey.create(
         clazz, id, Key.create(Key.create(EntityGroupRoot.class, "per-tld"), clazz, id));

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -875,16 +875,13 @@ public class DomainBaseTest extends EntityTestCase {
     ImmutableSet<BillEventInfo> historyIds =
         domain.getGracePeriods().stream()
             .map(
-                gp ->
-                    new BillEventInfo(
-                        gp.getRecurringBillingEvent(), gp.billingEventRecurringHistoryId,
-                        gp.getOneTimeBillingEvent(), gp.billingEventOneTimeHistoryId))
+                gp -> new BillEventInfo(gp.getRecurringBillingEvent(), gp.getOneTimeBillingEvent()))
             .collect(toImmutableSet());
     assertThat(historyIds)
         .isEqualTo(
             ImmutableSet.of(
-                new BillEventInfo(null, null, oneTimeBillKey, historyEntryKey.getId()),
-                new BillEventInfo(recurringBillKey, historyEntryKey.getId(), null, null)));
+                new BillEventInfo(null, oneTimeBillKey),
+                new BillEventInfo(recurringBillKey, null)));
   }
 
   static class BillEventInfo extends ImmutableObject {
@@ -895,13 +892,9 @@ public class DomainBaseTest extends EntityTestCase {
 
     BillEventInfo(
         VKey<BillingEvent.Recurring> billingEventRecurring,
-        Long billingEventRecurringHistoryId,
-        VKey<BillingEvent.OneTime> billingEventOneTime,
-        Long billingEventOneTimeHistoryId) {
+        VKey<BillingEvent.OneTime> billingEventOneTime) {
       this.billingEventRecurring = billingEventRecurring;
-      this.billingEventRecurringHistoryId = billingEventRecurringHistoryId;
       this.billingEventOneTime = billingEventOneTime;
-      this.billingEventOneTimeHistoryId = billingEventOneTimeHistoryId;
     }
   }
 }

core/src/test/java/google/registry/model/domain/GracePeriodTest.java

@@ -63,11 +63,11 @@ public class GracePeriodTest {
     recurringKey =
         VKey.create(
             Recurring.class,
-            12345,
+            12345L,
             Key.create(
                 Key.create(Key.create(DomainBase.class, "1-TEST"), HistoryEntry.class, 343L),
                 Recurring.class,
-                12345));
+                12345L));
   }
 
   @Test
@@ -80,8 +80,6 @@ public class GracePeriodTest {
     assertThat(gracePeriod.getClientId()).isEqualTo("TheRegistrar");
     assertThat(gracePeriod.getExpirationTime()).isEqualTo(now.plusDays(1));
     assertThat(gracePeriod.hasBillingEvent()).isTrue();
-    assertThat(gracePeriod.billingEventOneTimeHistoryId).isEqualTo(12345L);
-    assertThat(gracePeriod.billingEventRecurringHistoryId).isNull();
   }
 
   @Test
@@ -96,8 +94,6 @@ public class GracePeriodTest {
     assertThat(gracePeriod.getClientId()).isEqualTo("TheRegistrar");
     assertThat(gracePeriod.getExpirationTime()).isEqualTo(now.plusDays(1));
     assertThat(gracePeriod.hasBillingEvent()).isTrue();
-    assertThat(gracePeriod.billingEventOneTimeHistoryId).isNull();
-    assertThat(gracePeriod.billingEventRecurringHistoryId).isEqualTo(343L);
   }
 
   @Test

core/src/test/java/google/registry/model/domain/token/AllocationTokenTest.java

@@ -16,15 +16,14 @@ package google.registry.model.domain.token;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
-import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.model.domain.token.AllocationToken.TokenStatus.CANCELLED;
 import static google.registry.model.domain.token.AllocationToken.TokenStatus.ENDED;
 import static google.registry.model.domain.token.AllocationToken.TokenStatus.NOT_STARTED;
 import static google.registry.model.domain.token.AllocationToken.TokenStatus.VALID;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
-import static google.registry.model.ofy.ObjectifyService.ofy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
 import static google.registry.testing.DatabaseHelper.persistResource;
@@ -40,13 +39,14 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
-import google.registry.persistence.VKey;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link AllocationToken}. */
+@DualDatabaseTest
 public class AllocationTokenTest extends EntityTestCase {
 
   public AllocationTokenTest() {
@@ -58,7 +58,7 @@ public class AllocationTokenTest extends EntityTestCase {
     createTld("foo");
   }
 
-  @Test
+  @TestOfyAndSql
   void testPersistence() {
     AllocationToken unlimitedUseToken =
         persistResource(
@@ -78,7 +78,7 @@ public class AllocationTokenTest extends EntityTestCase {
                         .put(DateTime.now(UTC).plusWeeks(8), TokenStatus.ENDED)
                         .build())
                 .build());
-    assertThat(ofy().load().entity(unlimitedUseToken).now()).isEqualTo(unlimitedUseToken);
+    assertThat(transactIfJpaTm(() -> tm().load(unlimitedUseToken))).isEqualTo(unlimitedUseToken);
 
     DomainBase domain = persistActiveDomain("example.foo");
     Key<HistoryEntry> historyEntryKey = Key.create(Key.create(domain), HistoryEntry.class, 1);
@@ -86,37 +86,15 @@ public class AllocationTokenTest extends EntityTestCase {
         persistResource(
             new AllocationToken.Builder()
                 .setToken("abc123Single")
-                .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+                .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
                 .setDomainName("example.foo")
                 .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
                 .setTokenType(SINGLE_USE)
                 .build());
-    assertThat(ofy().load().entity(singleUseToken).now()).isEqualTo(singleUseToken);
-
-    jpaTm()
-        .transact(
-            () -> {
-              jpaTm().insert(unlimitedUseToken);
-              jpaTm().insert(singleUseToken);
-            });
-    jpaTm()
-        .transact(
-            () -> {
-              assertAboutImmutableObjects()
-                  .that(jpaTm().load(VKey.createSql(AllocationToken.class, "abc123Unlimited")))
-                  .isEqualExceptFields(
-                      unlimitedUseToken,
-                      "creationTime",
-                      "updateTimestamp",
-                      "redemptionHistoryEntry");
-              assertAboutImmutableObjects()
-                  .that(jpaTm().load(VKey.createSql(AllocationToken.class, "abc123Single")))
-                  .isEqualExceptFields(
-                      singleUseToken, "creationTime", "updateTimestamp", "redemptionHistoryEntry");
-            });
+    assertThat(transactIfJpaTm(() -> tm().load(singleUseToken))).isEqualTo(singleUseToken);
   }
 
-  @Test
+  @TestOfyOnly
   void testIndexing() throws Exception {
     DomainBase domain = persistActiveDomain("blahdomain.foo");
     Key<HistoryEntry> historyEntryKey = Key.create(Key.create(domain), HistoryEntry.class, 1);
@@ -125,7 +103,7 @@ public class AllocationTokenTest extends EntityTestCase {
             new AllocationToken.Builder()
                 .setToken("abc123")
                 .setTokenType(SINGLE_USE)
-                .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+                .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
                 .setDomainName("blahdomain.foo")
                 .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
                 .build()),
@@ -134,7 +112,7 @@ public class AllocationTokenTest extends EntityTestCase {
         "domainName");
   }
 
-  @Test
+  @TestOfyAndSql
   void testCreationTime_autoPopulates() {
     AllocationToken tokenBeforePersisting =
         new AllocationToken.Builder().setToken("abc123").setTokenType(SINGLE_USE).build();
@@ -143,7 +121,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(tokenAfterPersisting.getCreationTime()).hasValue(fakeClock.nowUtc());
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetCreationTime_cantCallMoreThanOnce() {
     AllocationToken.Builder builder =
         new AllocationToken.Builder()
@@ -157,7 +135,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Creation time can only be set once");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetToken_cantCallMoreThanOnce() {
     AllocationToken.Builder builder = new AllocationToken.Builder().setToken("foobar");
     IllegalStateException thrown =
@@ -165,7 +143,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Token can only be set once");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetTokenType_cantCallMoreThanOnce() {
     AllocationToken.Builder builder =
         new AllocationToken.Builder().setTokenType(TokenType.UNLIMITED_USE);
@@ -174,7 +152,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Token type can only be set once");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_DomainNameWithLessThanTwoParts() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -192,7 +170,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Invalid domain name: example");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_invalidTld() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -210,7 +188,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Invalid domain name: example.nosuchtld");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_domainNameOnlyOnSingleUse() {
     AllocationToken.Builder builder =
         new AllocationToken.Builder()
@@ -223,7 +201,7 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("Domain name can only be specified for SINGLE_USE tokens");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_redemptionHistoryEntryOnlyInSingleUse() {
     DomainBase domain = persistActiveDomain("blahdomain.foo");
     Key<HistoryEntry> historyEntryKey = Key.create(Key.create(domain), HistoryEntry.class, 1);
@@ -231,14 +209,14 @@ public class AllocationTokenTest extends EntityTestCase {
         new AllocationToken.Builder()
             .setToken("foobar")
             .setTokenType(TokenType.UNLIMITED_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey));
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey));
     IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, builder::build);
     assertThat(thrown)
         .hasMessageThat()
         .isEqualTo("Redemption history entry can only be specified for SINGLE_USE tokens");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetTransitions_notStartOfTime() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -256,7 +234,7 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("tokenStatusTransitions map must start at START_OF_TIME.");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetTransitions_badInitialValue() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -273,14 +251,14 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("tokenStatusTransitions must start with NOT_STARTED");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetTransitions_invalidInitialTransitions() {
     // NOT_STARTED can only go to VALID or CANCELLED
     assertBadInitialTransition(NOT_STARTED);
     assertBadInitialTransition(ENDED);
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetTransitions_badTransitionsFromValid() {
     // VALID can only go to ENDED or CANCELLED
     assertBadTransition(
@@ -301,14 +279,14 @@ public class AllocationTokenTest extends EntityTestCase {
         NOT_STARTED);
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetTransitions_terminalTransitions() {
     // both ENDED and CANCELLED are terminal
     assertTerminal(ENDED);
     assertTerminal(CANCELLED);
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetDiscountFractionTooHigh() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -319,7 +297,7 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("Discount fraction must be between 0 and 1 inclusive");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetDiscountFractionTooLow() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -330,7 +308,7 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("Discount fraction must be between 0 and 1 inclusive");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetDiscountYearsTooHigh() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -341,7 +319,7 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("Discount years must be between 1 and 10 inclusive");
   }
 
-  @Test
+  @TestOfyAndSql
   void testSetDiscountYearsTooLow() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -352,7 +330,7 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("Discount years must be between 1 and 10 inclusive");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_noTokenType() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -361,7 +339,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Token type must be specified");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_noToken() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -370,7 +348,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Token must not be null or empty");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_emptyToken() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -379,7 +357,7 @@ public class AllocationTokenTest extends EntityTestCase {
     assertThat(thrown).hasMessageThat().isEqualTo("Token must not be blank");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_discountPremiumsRequiresDiscountFraction() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -395,7 +373,7 @@ public class AllocationTokenTest extends EntityTestCase {
         .isEqualTo("Discount premiums can only be specified along with a discount fraction");
   }
 
-  @Test
+  @TestOfyAndSql
   void testBuild_discountYearsRequiresDiscountFraction() {
     IllegalArgumentException thrown =
         assertThrows(

core/src/test/java/google/registry/model/history/DomainHistoryTest.java

@@ -14,7 +14,6 @@
 
 package google.registry.model.history;
 
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
@@ -92,10 +91,7 @@ public class DomainHistoryTest extends EntityTestCase {
               assertDomainHistoriesEqual(fromDatabase, domainHistory);
               assertThat(fromDatabase.getParentVKey()).isEqualTo(domainHistory.getParentVKey());
               assertThat(fromDatabase.getNsHosts())
-                  .containsExactlyElementsIn(
-                      domainHistory.getNsHosts().stream()
-                          .map(key -> VKey.createSql(HostResource.class, key.getSqlKey()))
-                          .collect(toImmutableSet()));
+                  .containsExactlyElementsIn(domainHistory.getNsHosts());
             });
   }
 

core/src/test/java/google/registry/model/host/HostResourceTest.java

@@ -17,13 +17,17 @@ package google.registry.model.host;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
+import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.cloneAndSetAutoTimestamps;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistNewRegistrars;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.HostResourceSubject.assertAboutHosts;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
 import google.registry.model.EntityTestCase;
@@ -32,11 +36,14 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link HostResource}. */
+@DualDatabaseTest
 class HostResourceTest extends EntityTestCase {
 
   private final DateTime day3 = fakeClock.nowUtc();
@@ -49,6 +56,7 @@ class HostResourceTest extends EntityTestCase {
   @BeforeEach
   void setUp() {
     createTld("com");
+    persistNewRegistrars("gaining", "losing", "thisRegistrar", "thatRegistrar");
     // Set up a new persisted registrar entity.
     domain =
         persistResource(
@@ -71,9 +79,9 @@ class HostResourceTest extends EntityTestCase {
                 new HostResource.Builder()
                     .setRepoId("DEADBEEF-COM")
                     .setHostName("ns1.example.com")
-                    .setCreationClientId("a registrar")
+                    .setCreationClientId("thisRegistrar")
                     .setLastEppUpdateTime(fakeClock.nowUtc())
-                    .setLastEppUpdateClientId("another registrar")
+                    .setLastEppUpdateClientId("thatRegistrar")
                     .setLastTransferTime(fakeClock.nowUtc())
                     .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
                     .setStatusValues(ImmutableSet.of(StatusValue.OK))
@@ -81,13 +89,22 @@ class HostResourceTest extends EntityTestCase {
                     .build()));
   }
 
-  @Test
+  @TestOfyAndSql
   void testPersistence() {
+    HostResource newHost = host.asBuilder().setRepoId("NEWHOST").build();
+    tm().transact(() -> tm().insert(newHost));
+    assertThat(ImmutableList.of(tm().transact(() -> tm().load(newHost.createVKey()))))
+        .comparingElementsUsing(immutableObjectCorrespondence("revisions"))
+        .containsExactly(newHost);
+  }
+
+  @TestOfyOnly
+  void testLoadingByForeignKey() {
     assertThat(loadByForeignKey(HostResource.class, host.getForeignKey(), fakeClock.nowUtc()))
         .hasValue(host);
   }
 
-  @Test
+  @TestOfyOnly
   void testIndexing() throws Exception {
     // Clone it and save it before running the indexing test so that its transferData fields are
     // populated from the superordinate domain.
@@ -100,7 +117,7 @@ class HostResourceTest extends EntityTestCase {
         "currentSponsorClientId");
   }
 
-  @Test
+  @TestOfyAndSql
   void testEmptyStringsBecomeNull() {
     assertThat(
             new HostResource.Builder()
@@ -122,7 +139,7 @@ class HostResourceTest extends EntityTestCase {
         .isNotNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testEmptySetsBecomeNull() {
     assertThat(new HostResource.Builder().setInetAddresses(null).build().inetAddresses).isNull();
     assertThat(new HostResource.Builder().setInetAddresses(ImmutableSet.of()).build().inetAddresses)
@@ -135,7 +152,7 @@ class HostResourceTest extends EntityTestCase {
         .isNotNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testImplicitStatusValues() {
     // OK is implicit if there's no other statuses.
     assertAboutHosts()
@@ -157,13 +174,13 @@ class HostResourceTest extends EntityTestCase {
         .hasExactlyStatusValues(StatusValue.CLIENT_HOLD);
   }
 
-  @Test
+  @TestOfyAndSql
   void testToHydratedString_notCircular() {
     // If there are circular references, this will overflow the stack.
     host.toHydratedString();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_uppercaseHostName() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -173,7 +190,7 @@ class HostResourceTest extends EntityTestCase {
         .contains("Host name must be in puny-coded, lower-case form");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_utf8HostName() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -183,14 +200,14 @@ class HostResourceTest extends EntityTestCase {
         .contains("Host name must be in puny-coded, lower-case form");
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostNeverSwitchedDomains_domainWasNeverTransferred() {
     domain = domain.asBuilder().setLastTransferTime(null).build();
     host = host.asBuilder().setLastTransferTime(null).setLastSuperordinateChange(null).build();
     assertThat(host.computeLastTransferTime(domain)).isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostNeverSwitchedDomains_domainWasTransferred() {
     // Host was created on Day 1.
     // Domain was transferred on Day 2.
@@ -205,7 +222,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isEqualTo(day2);
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostCreatedAfterDomainWasTransferred() {
     // Domain was transferred on Day 1.
     // Host was created subordinate to domain on Day 2.
@@ -217,9 +234,9 @@ class HostResourceTest extends EntityTestCase {
                     .setCreationTime(day2)
                     .setRepoId("DEADBEEF-COM")
                     .setHostName("ns1.example.com")
-                    .setCreationClientId("a registrar")
+                    .setCreationClientId("thisRegistrar")
                     .setLastEppUpdateTime(fakeClock.nowUtc())
-                    .setLastEppUpdateClientId("another registrar")
+                    .setLastEppUpdateClientId("thatRegistrar")
                     .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
                     .setStatusValues(ImmutableSet.of(StatusValue.OK))
                     .setSuperordinateDomain(domain.createVKey())
@@ -227,7 +244,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostWasTransferred_domainWasNeverTransferred() {
     // Host was transferred on Day 1.
     // Host was made subordinate to domain on Day 2.
@@ -237,7 +254,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isEqualTo(day1);
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_domainWasTransferredBeforeHostBecameSubordinate() {
     // Host was transferred on Day 1.
     // Domain was transferred on Day 2.
@@ -247,7 +264,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isEqualTo(day1);
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_domainWasTransferredAfterHostBecameSubordinate() {
     // Host was transferred on Day 1.
     // Host was made subordinate to domain on Day 2.

core/src/test/java/google/registry/model/index/ForeignKeyIndexTest.java

@@ -20,6 +20,7 @@ import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.deleteResource;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static google.registry.testing.DatabaseHelper.persistActiveHost;
 import static google.registry.testing.DatabaseHelper.persistDeletedHost;
@@ -29,16 +30,21 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import com.google.common.collect.ImmutableList;
 import google.registry.model.EntityTestCase;
 import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.model.index.ForeignKeyIndex.ForeignKeyHostIndex;
+import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.TestCacheExtension;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
+import google.registry.testing.TestSqlOnly;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link ForeignKeyIndex}. */
-public class ForeignKeyIndexTest extends EntityTestCase {
+@DualDatabaseTest
+class ForeignKeyIndexTest extends EntityTestCase {
 
   @RegisterExtension
   public final TestCacheExtension testCacheExtension =
@@ -49,7 +55,17 @@ public class ForeignKeyIndexTest extends EntityTestCase {
     createTld("com");
   }
 
-  @Test
+  @TestSqlOnly
+  void testModifyForeignKeyIndex_notThrowExceptionInSql() {
+    DomainBase domainBase = newDomainBase("test.com");
+    ForeignKeyIndex<DomainBase> fki = ForeignKeyIndex.create(domainBase, fakeClock.nowUtc());
+    tm().transact(() -> tm().insert(fki));
+    tm().transact(() -> tm().put(fki));
+    tm().transact(() -> tm().delete(fki));
+    tm().transact(() -> tm().update(fki));
+  }
+
+  @TestOfyOnly
   void testPersistence() {
     // Persist a host and implicitly persist a ForeignKeyIndex for it.
     HostResource host = persistActiveHost("ns1.example.com");
@@ -59,7 +75,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
     assertThat(fki.getDeletionTime()).isEqualTo(END_OF_TIME);
   }
 
-  @Test
+  @TestOfyOnly
   void testIndexing() throws Exception {
     // Persist a host and implicitly persist a ForeignKeyIndex for it.
     persistActiveHost("ns1.example.com");
@@ -68,38 +84,50 @@ public class ForeignKeyIndexTest extends EntityTestCase {
         "deletionTime");
   }
 
-  @Test
+  @TestOfyAndSql
   void testLoadForNonexistentForeignKey_returnsNull() {
     assertThat(ForeignKeyIndex.load(HostResource.class, "ns1.example.com", fakeClock.nowUtc()))
         .isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testLoadForDeletedForeignKey_returnsNull() {
     HostResource host = persistActiveHost("ns1.example.com");
-    persistResource(ForeignKeyIndex.create(host, fakeClock.nowUtc().minusDays(1)));
+    if (tm().isOfy()) {
+      persistResource(ForeignKeyIndex.create(host, fakeClock.nowUtc().minusDays(1)));
+    } else {
+      persistResource(host.asBuilder().setDeletionTime(fakeClock.nowUtc().minusDays(1)).build());
+    }
     assertThat(ForeignKeyIndex.load(HostResource.class, "ns1.example.com", fakeClock.nowUtc()))
         .isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testLoad_newerKeyHasBeenSoftDeleted() {
     HostResource host1 = persistActiveHost("ns1.example.com");
     fakeClock.advanceOneMilli();
-    ForeignKeyHostIndex fki = new ForeignKeyHostIndex();
-    fki.foreignKey = "ns1.example.com";
-    fki.topReference = host1.createVKey();
-    fki.deletionTime = fakeClock.nowUtc();
-    persistResource(fki);
+    if (tm().isOfy()) {
+      ForeignKeyHostIndex fki = new ForeignKeyHostIndex();
+      fki.foreignKey = "ns1.example.com";
+      fki.topReference = host1.createVKey();
+      fki.deletionTime = fakeClock.nowUtc();
+      persistResource(fki);
+    } else {
+      persistResource(host1.asBuilder().setDeletionTime(fakeClock.nowUtc()).build());
+    }
     assertThat(ForeignKeyIndex.load(HostResource.class, "ns1.example.com", fakeClock.nowUtc()))
         .isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testBatchLoad_skipsDeletedAndNonexistent() {
     persistActiveHost("ns1.example.com");
     HostResource host = persistActiveHost("ns2.example.com");
-    persistResource(ForeignKeyIndex.create(host, fakeClock.nowUtc().minusDays(1)));
+    if (tm().isOfy()) {
+      persistResource(ForeignKeyIndex.create(host, fakeClock.nowUtc().minusDays(1)));
+    } else {
+      persistResource(host.asBuilder().setDeletionTime(fakeClock.nowUtc().minusDays(1)).build());
+    }
     assertThat(
             ForeignKeyIndex.load(
                     HostResource.class,
@@ -109,7 +137,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
         .containsExactly("ns1.example.com");
   }
 
-  @Test
+  @TestOfyAndSql
   void testDeadCodeThatDeletedScrapCommandsReference() {
     persistActiveHost("omg");
     assertThat(ForeignKeyIndex.load(HostResource.class, "omg", fakeClock.nowUtc()).getForeignKey())
@@ -124,7 +152,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
     return ForeignKeyIndex.load(ContactResource.class, contactId, fakeClock.nowUtc());
   }
 
-  @Test
+  @TestOfyOnly
   void test_loadCached_cachesNonexistenceOfHosts() {
     assertThat(
             ForeignKeyIndex.loadCached(
@@ -144,7 +172,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
         .containsExactly("ns4.example.com", loadHostFki("ns4.example.com"));
   }
 
-  @Test
+  @TestOfyOnly
   void test_loadCached_cachesExistenceOfHosts() {
     HostResource host1 = persistActiveHost("ns1.example.com");
     HostResource host2 = persistActiveHost("ns2.example.com");
@@ -172,7 +200,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
             "ns3.example.com", loadHostFki("ns3.example.com"));
   }
 
-  @Test
+  @TestOfyOnly
   void test_loadCached_doesntSeeHostChangesWhileCacheIsValid() {
     HostResource originalHost = persistActiveHost("ns1.example.com");
     ForeignKeyIndex<HostResource> originalFki = loadHostFki("ns1.example.com");
@@ -195,7 +223,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
         .containsExactly("ns1.example.com", originalFki);
   }
 
-  @Test
+  @TestOfyOnly
   void test_loadCached_filtersOutSoftDeletedHosts() {
     persistActiveHost("ns1.example.com");
     persistDeletedHost("ns2.example.com", fakeClock.nowUtc().minusDays(1));
@@ -207,7 +235,7 @@ public class ForeignKeyIndexTest extends EntityTestCase {
         .containsExactly("ns1.example.com", loadHostFki("ns1.example.com"));
   }
 
-  @Test
+  @TestOfyOnly
   void test_loadCached_cachesContactFkis() {
     persistActiveContact("contactid1");
     ForeignKeyIndex<ContactResource> fki1 = loadContactFki("contactid1");

core/src/test/java/google/registry/model/smd/SignedMarkRevocationListDaoTest.java

@@ -50,21 +50,6 @@ public class SignedMarkRevocationListDaoTest {
     assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
   }
 
-  @Test
-  void trySave_failureIsSwallowed() {
-    SignedMarkRevocationList list =
-        SignedMarkRevocationList.create(
-            fakeClock.nowUtc(), ImmutableMap.of("mark", fakeClock.nowUtc().minusHours(1)));
-    SignedMarkRevocationListDao.trySave(list);
-    SignedMarkRevocationList fromDb = SignedMarkRevocationListDao.getLatestRevision().get();
-    assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
-
-    // This should throw an exception, which is swallowed and nothing changed
-    SignedMarkRevocationListDao.trySave(list);
-    SignedMarkRevocationList secondFromDb = SignedMarkRevocationListDao.getLatestRevision().get();
-    assertAboutImmutableObjects().that(secondFromDb).isEqualExceptFields(fromDb);
-  }
-
   @Test
   void testRetrieval_notPresent() {
     assertThat(SignedMarkRevocationListDao.getLatestRevision().isPresent()).isFalse();

core/src/test/java/google/registry/model/smd/SignedMarkRevocationListTest.java

@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.smd.SignedMarkRevocationList.SHARD_SIZE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.time.Duration.standardDays;
@@ -138,6 +139,33 @@ public class SignedMarkRevocationListTest {
         .isEqualTo(DateTime.parse("2000-01-01T00:00:00Z"));
   }
 
+  @Test
+  void test_getCreationTime_missingInCloudSQL() {
+    clock.setTo(DateTime.parse("2000-01-01T00:00:00Z"));
+    createSaveGetHelper(1);
+    jpaTm().transact(() -> jpaTm().delete(SignedMarkRevocationListDao.getLatestRevision().get()));
+    RuntimeException thrown =
+        assertThrows(RuntimeException.class, () -> SignedMarkRevocationList.get());
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("Signed mark revocation list in Cloud SQL is empty.");
+  }
+
+  @Test
+  void test_getCreationTime_unequalListsInDatabases() {
+    clock.setTo(DateTime.parse("2000-01-01T00:00:00Z"));
+    createSaveGetHelper(1);
+    ImmutableMap.Builder<String, DateTime> revokes = new ImmutableMap.Builder<>();
+    for (int i = 0; i < 3; i++) {
+      revokes.put(Integer.toString(i), clock.nowUtc());
+    }
+    SignedMarkRevocationListDao.trySave(
+        SignedMarkRevocationList.create(clock.nowUtc(), revokes.build()));
+    RuntimeException thrown =
+        assertThrows(RuntimeException.class, () -> SignedMarkRevocationList.get());
+    assertThat(thrown).hasMessageThat().contains("Unequal SM revocation lists detected:");
+  }
+
   @Test
   void test_isSmdRevoked_present() {
     final int rows = SHARD_SIZE + 1;

core/src/test/java/google/registry/model/translators/EppHistoryVKeyTranslatorFactoryTest.java

@@ -0,0 +1,36 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.translators;
+
+import static google.registry.model.translators.EppHistoryVKeyTranslatorFactory.kindPathToVKeyClass;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import org.junit.jupiter.api.Test;
+
+/** Unit test for {@link EppHistoryVKeyTranslatorFactory}. */
+class EppHistoryVKeyTranslatorFactoryTest {
+
+  @Test
+  void assertAllVKeyClassesHavingCreateFromOfyKeyMethod() {
+    kindPathToVKeyClass.forEach(
+        (kindPath, vKeyClass) -> {
+          try {
+            vKeyClass.getDeclaredMethod("create", com.googlecode.objectify.Key.class);
+          } catch (NoSuchMethodException e) {
+            fail("Missing static method create(com.googlecode.objectify.Key) on " + vKeyClass, e);
+          }
+        });
+  }
+}

core/src/test/java/google/registry/persistence/BillingVKeyTest.java

@@ -0,0 +1,116 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Parent;
+import google.registry.model.ImmutableObject;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.EntityGroupRoot;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
+import javax.persistence.Transient;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit test for {@link BillingVKey}. */
+@DualDatabaseTest
+class BillingVKeyTest {
+  @RegisterExtension
+  final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withDatastoreAndCloudSql()
+          .withOfyTestEntities(BillingVKeyTestEntity.class)
+          .withJpaUnitTestEntities(BillingVKeyTestEntity.class)
+          .build();
+
+  @TestOfyAndSql
+  void testRestoreSymmetricVKey() {
+    Key<HistoryEntry> domainHistoryKey =
+        Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L);
+
+    Key<BillingEvent.OneTime> oneTimeOfyKey =
+        Key.create(domainHistoryKey, BillingEvent.OneTime.class, 100L);
+    VKey<BillingEvent.OneTime> oneTimeVKey =
+        VKey.create(BillingEvent.OneTime.class, 100L, oneTimeOfyKey);
+
+    Key<BillingEvent.Recurring> recurringOfyKey =
+        Key.create(domainHistoryKey, BillingEvent.Recurring.class, 200L);
+    VKey<BillingEvent.Recurring> recurringVKey =
+        VKey.create(BillingEvent.Recurring.class, 200L, recurringOfyKey);
+
+    BillingVKeyTestEntity original = new BillingVKeyTestEntity(oneTimeVKey, recurringVKey);
+    tm().transact(() -> tm().insert(original));
+    BillingVKeyTestEntity persisted = tm().transact(() -> tm().load(original.createVKey()));
+
+    assertThat(persisted).isEqualTo(original);
+    assertThat(persisted.getBillingEventVKey()).isEqualTo(oneTimeVKey);
+    assertThat(persisted.getBillingRecurrenceVKey()).isEqualTo(recurringVKey);
+  }
+
+  @TestOfyAndSql
+  void testHandleNullVKeyCorrectly() {
+    BillingVKeyTestEntity original = new BillingVKeyTestEntity(null, null);
+    tm().transact(() -> tm().insert(original));
+    BillingVKeyTestEntity persisted = tm().transact(() -> tm().load(original.createVKey()));
+
+    assertThat(persisted).isEqualTo(original);
+  }
+
+  @EntityForTesting
+  @Entity
+  @javax.persistence.Entity
+  private static class BillingVKeyTestEntity extends ImmutableObject {
+    @Transient @Parent Key<EntityGroupRoot> parent = getCrossTldKey();
+
+    @Id @javax.persistence.Id String id = "id";
+
+    BillingEventVKey billingEventVKey;
+
+    BillingRecurrenceVKey billingRecurrenceVKey;
+
+    BillingVKeyTestEntity() {}
+
+    BillingVKeyTestEntity(
+        VKey<BillingEvent.OneTime> onetime, VKey<BillingEvent.Recurring> recurring) {
+      this.billingEventVKey = BillingEventVKey.create(onetime);
+      this.billingRecurrenceVKey = BillingRecurrenceVKey.create(recurring);
+    }
+
+    VKey<BillingEvent.OneTime> getBillingEventVKey() {
+      return billingEventVKey.createVKey();
+    }
+
+    VKey<BillingEvent.Recurring> getBillingRecurrenceVKey() {
+      return billingRecurrenceVKey.createVKey();
+    }
+
+    VKey<BillingVKeyTestEntity> createVKey() {
+      return VKey.create(
+          BillingVKeyTestEntity.class, id, Key.create(parent, BillingVKeyTestEntity.class, id));
+    }
+  }
+}

core/src/test/java/google/registry/persistence/DomainHistoryVKeyTest.java

@@ -56,12 +56,17 @@ class DomainHistoryVKeyTest {
     TestEntity persisted = tm().transact(() -> tm().load(original.createVKey()));
     assertThat(persisted).isEqualTo(original);
     // Double check that the persisted.domainHistoryVKey is a symmetric VKey
-    assertThat(persisted.domainHistoryVKey.getKind()).isEqualTo(HistoryEntry.class);
-    assertThat(persisted.domainHistoryVKey.getOfyKey())
+    assertThat(persisted.domainHistoryVKey.createOfyKey())
         .isEqualTo(
             Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L));
-    assertThat(persisted.domainHistoryVKey.getSqlKey())
+    assertThat(persisted.domainHistoryVKey.createSqlKey())
         .isEqualTo(new DomainHistoryId("domainRepoId", 10L));
+    assertThat(persisted.domainHistoryVKey.createVKey())
+        .isEqualTo(
+            VKey.create(
+                HistoryEntry.class,
+                new DomainHistoryId("domainRepoId", 10L),
+                Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L)));
   }
 
   @TestOfyAndSql
@@ -69,9 +74,12 @@ class DomainHistoryVKeyTest {
     Key<HistoryEntry> ofyKey =
         Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L);
     DomainHistoryVKey domainHistoryVKey = DomainHistoryVKey.create(ofyKey);
-    assertThat(domainHistoryVKey.getKind()).isEqualTo(HistoryEntry.class);
-    assertThat(domainHistoryVKey.getOfyKey()).isEqualTo(ofyKey);
-    assertThat(domainHistoryVKey.getSqlKey()).isEqualTo(new DomainHistoryId("domainRepoId", 10L));
+    assertThat(domainHistoryVKey.createOfyKey()).isEqualTo(ofyKey);
+    assertThat(domainHistoryVKey.createSqlKey())
+        .isEqualTo(new DomainHistoryId("domainRepoId", 10L));
+    assertThat(domainHistoryVKey.createVKey())
+        .isEqualTo(
+            VKey.create(HistoryEntry.class, new DomainHistoryId("domainRepoId", 10L), ofyKey));
   }
 
   @EntityForTesting

core/src/test/java/google/registry/persistence/converter/LongVKeyConverterTest.java

@@ -19,8 +19,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
-import google.registry.persistence.transaction.JpaTestRules;
-import google.registry.persistence.transaction.JpaTestRules.JpaUnitTestExtension;
+import google.registry.testing.AppEngineExtension;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.junit.jupiter.api.Test;
@@ -30,33 +29,56 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 public class LongVKeyConverterTest {
 
   @RegisterExtension
-  public final JpaUnitTestExtension jpaExtension =
-      new JpaTestRules.Builder()
-          .withEntityClass(TestEntity.class, VKeyConverter_LongType.class)
-          .buildUnitTestRule();
+  public final AppEngineExtension appEngineExtension =
+      new AppEngineExtension.Builder()
+          .withDatastoreAndCloudSql()
+          .withoutCannedData()
+          .withJpaUnitTestEntities(
+              TestLongEntity.class,
+              VKeyConverter_LongType.class,
+              VKeyConverter_CompositeLongType.class)
+          .withOfyTestEntities(TestLongEntity.class, CompositeKeyTestLongEntity.class)
+          .build();
 
   @Test
   void testRoundTrip() {
-    TestEntity original = new TestEntity(VKey.createSql(TestEntity.class, 10L));
+    TestLongEntity original =
+        new TestLongEntity(
+            VKey.createSql(TestLongEntity.class, 10L),
+            VKey.createSql(CompositeKeyTestLongEntity.class, 20L));
     jpaTm().transact(() -> jpaTm().getEntityManager().persist(original));
 
-    TestEntity retrieved =
-        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "id"));
+    TestLongEntity retrieved =
+        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestLongEntity.class, "id"));
     assertThat(retrieved.number.getSqlKey()).isEqualTo(10L);
+    assertThat(retrieved.number.getOfyKey().getId()).isEqualTo(10L);
+
+    assertThat(retrieved.composite.getSqlKey()).isEqualTo(20L);
+    assertThat(retrieved.composite.maybeGetOfyKey().isPresent()).isFalse();
   }
 
-  @Entity(name = "TestEntity")
+  @Entity(name = "TestLongEntity")
+  @com.googlecode.objectify.annotation.Entity
   @WithLongVKey(classNameSuffix = "LongType")
-  static class TestEntity {
-    @Id String id = "id";
+  static class TestLongEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id = "id";
 
-    VKey<TestEntity> number;
+    VKey<TestLongEntity> number;
+    VKey<CompositeKeyTestLongEntity> composite;
 
-    TestEntity(VKey<TestEntity> number) {
+    TestLongEntity(VKey<TestLongEntity> number, VKey<CompositeKeyTestLongEntity> composite) {
       this.number = number;
+      this.composite = composite;
     }
 
     /** Default constructor, needed for hibernate. */
-    public TestEntity() {}
+    public TestLongEntity() {}
+  }
+
+  @Entity(name = "CompositeKeyTestLongEntity")
+  @com.googlecode.objectify.annotation.Entity
+  @WithLongVKey(classNameSuffix = "CompositeLongType", compositeKey = true)
+  static class CompositeKeyTestLongEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id = "id";
   }
 }

core/src/test/java/google/registry/persistence/converter/StringVKeyConverterTest.java

@@ -19,8 +19,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
-import google.registry.persistence.transaction.JpaTestRules;
-import google.registry.persistence.transaction.JpaTestRules.JpaUnitTestExtension;
+import google.registry.testing.AppEngineExtension;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.junit.jupiter.api.Test;
@@ -30,36 +29,61 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 public class StringVKeyConverterTest {
 
   @RegisterExtension
-  public final JpaUnitTestExtension jpaExtension =
-      new JpaTestRules.Builder()
-          .withEntityClass(TestEntity.class, VKeyConverter_StringType.class)
-          .buildUnitTestRule();
+  public final AppEngineExtension appEngineExtension =
+      new AppEngineExtension.Builder()
+          .withDatastoreAndCloudSql()
+          .withoutCannedData()
+          .withJpaUnitTestEntities(
+              TestStringEntity.class,
+              VKeyConverter_StringType.class,
+              VKeyConverter_CompositeStringType.class)
+          .withOfyTestEntities(TestStringEntity.class, CompositeKeyTestStringEntity.class)
+          .build();
 
   @Test
   void testRoundTrip() {
-    TestEntity original =
-        new TestEntity("TheRealSpartacus", VKey.createSql(TestEntity.class, "ImSpartacus!"));
+    TestStringEntity original =
+        new TestStringEntity(
+            "TheRealSpartacus",
+            VKey.createSql(TestStringEntity.class, "ImSpartacus!"),
+            VKey.createSql(CompositeKeyTestStringEntity.class, "NoImSpartacus!"));
     jpaTm().transact(() -> jpaTm().getEntityManager().persist(original));
 
-    TestEntity retrieved =
+    TestStringEntity retrieved =
         jpaTm()
-            .transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "TheRealSpartacus"));
+            .transact(
+                () -> jpaTm().getEntityManager().find(TestStringEntity.class, "TheRealSpartacus"));
     assertThat(retrieved.other.getSqlKey()).isEqualTo("ImSpartacus!");
+    assertThat(retrieved.other.getOfyKey().getName()).isEqualTo("ImSpartacus!");
+
+    assertThat(retrieved.composite.getSqlKey()).isEqualTo("NoImSpartacus!");
+    assertThat(retrieved.composite.maybeGetOfyKey().isPresent()).isFalse();
   }
 
-  @Entity(name = "TestEntity")
+  @Entity(name = "TestStringEntity")
+  @com.googlecode.objectify.annotation.Entity
   @WithStringVKey(classNameSuffix = "StringType")
-  static class TestEntity {
-    @Id String id;
+  static class TestStringEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id;
 
-    VKey<TestEntity> other;
+    VKey<TestStringEntity> other;
+    VKey<CompositeKeyTestStringEntity> composite;
 
-    TestEntity(String id, VKey<TestEntity> other) {
+    TestStringEntity(
+        String id, VKey<TestStringEntity> other, VKey<CompositeKeyTestStringEntity> composite) {
       this.id = id;
       this.other = other;
+      this.composite = composite;
     }
 
     /** Default constructor, needed for hibernate. */
-    public TestEntity() {}
+    public TestStringEntity() {}
+  }
+
+  @Entity(name = "CompositeKeyTestStringEntity")
+  @com.googlecode.objectify.annotation.Entity
+  @WithStringVKey(classNameSuffix = "CompositeStringType", compositeKey = true)
+  static class CompositeKeyTestStringEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id = "id";
   }
 }

core/src/test/java/google/registry/privileges/secretmanager/FakeSecretManagerClient.java

@@ -32,6 +32,11 @@ public class FakeSecretManagerClient implements SecretManagerClient {
   @Inject
   FakeSecretManagerClient() {}
 
+  @Override
+  public String getProject() {
+    return "fake_project";
+  }
+
   @Override
   public void createSecret(String secretId) {
     checkNotNull(secretId, "secretId");
@@ -41,6 +46,12 @@ public class FakeSecretManagerClient implements SecretManagerClient {
     secrets.put(secretId, new SecretEntry(secretId));
   }
 
+  @Override
+  public boolean secretExists(String secretId) {
+    checkNotNull(secretId, "secretId");
+    return secrets.containsKey(secretId);
+  }
+
   @Override
   public Iterable<String> listSecrets() {
     return ImmutableSet.copyOf(secrets.keySet());
@@ -78,6 +89,28 @@ public class FakeSecretManagerClient implements SecretManagerClient {
     return secretEntry.getVersion(version).getData();
   }
 
+  @Override
+  public void enableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    secretEntry.enableVersion(version);
+  }
+
+  @Override
+  public void disableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    secretEntry.disableVersion(version);
+  }
+
   @Override
   public void destroySecretVersion(String secretId, String version) {
     checkNotNull(secretId, "secretId");
@@ -118,6 +151,20 @@ public class FakeSecretManagerClient implements SecretManagerClient {
       return state;
     }
 
+    void enable() {
+      if (state.equals(State.DESTROYED)) {
+        throw new SecretManagerException(null);
+      }
+      state = State.ENABLED;
+    }
+
+    void disable() {
+      if (state.equals(State.DESTROYED)) {
+        throw new SecretManagerException(null);
+      }
+      state = State.DISABLED;
+    }
+
     void destroy() {
       data = null;
       state = State.DESTROYED;
@@ -145,6 +192,8 @@ public class FakeSecretManagerClient implements SecretManagerClient {
         return versions.get(index);
       } catch (NumberFormatException e) {
         throw new IllegalArgumentException("Invalid version " + version.get());
+      } catch (ArrayIndexOutOfBoundsException e) {
+        throw new NoSuchSecretResourceException(null);
       }
     }
 
@@ -156,15 +205,16 @@ public class FakeSecretManagerClient implements SecretManagerClient {
       return builder.build();
     }
 
+    void enableVersion(String version) {
+      getVersion(Optional.of(version)).enable();
+    }
+
+    void disableVersion(String version) {
+      getVersion(Optional.of(version)).disable();
+    }
+
     void destroyVersion(String version) {
-      try {
-        int index = Integer.valueOf(version);
-        versions.get(index).destroy();
-      } catch (NumberFormatException e) {
-        throw new IllegalArgumentException("Invalid version " + version);
-      } catch (ArrayIndexOutOfBoundsException e) {
-        throw new NoSuchSecretResourceException(null);
-      }
+      getVersion(Optional.of(version)).destroy();
     }
   }
 }

core/src/test/java/google/registry/privileges/secretmanager/SecretManagerClientTest.java

@@ -17,13 +17,20 @@ package google.registry.privileges.secretmanager;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import com.google.cloud.secretmanager.v1.SecretVersion.State;
+import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
 import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
 import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
+import google.registry.util.Retrier;
+import google.registry.util.SystemSleeper;
+import java.io.IOException;
 import java.util.Optional;
 import java.util.UUID;
-import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -43,52 +50,53 @@ public class SecretManagerClientTest {
   private static final String SECRET_ID_PREFIX = "TEST_" + UUID.randomUUID() + "_";
   // Used for unique secret id generation.
   private static int seqno = 0;
-
   private static SecretManagerClient secretManagerClient;
   private static boolean isUnitTest = true;
 
-  private static String nextSecretId() {
-    return SECRET_ID_PREFIX + seqno++;
-  }
+  private String secretId;
 
   @BeforeAll
-  static void beforeAll() {
+  static void beforeAll() throws IOException {
     String environmentName = System.getProperty("test.gcp_integration.env");
     if (environmentName != null) {
       secretManagerClient =
-          DaggerSecretManagerModule_SecretManagerComponent.builder()
-              .secretManagerModule(
-                  new SecretManagerModule(String.format("domain-registry-%s", environmentName)))
-              .build()
-              .secretManagerClient();
+          SecretManagerModule.provideSecretManagerClient(
+              SecretManagerServiceSettings.newBuilder()
+                  .setCredentialsProvider(() -> GoogleCredentials.getApplicationDefault())
+                  .build(),
+              String.format("domain-registry-%s", environmentName),
+              new Retrier(new SystemSleeper(), 1));
       isUnitTest = false;
     } else {
       secretManagerClient = new FakeSecretManagerClient();
     }
   }
 
-  @AfterAll
-  static void afterAll() {
+  @BeforeEach
+  void beforeEach() {
+    secretId = SECRET_ID_PREFIX + seqno++;
+  }
+
+  @AfterEach
+  void afterEach() throws IOException {
     if (isUnitTest) {
       return;
     }
-    for (String secretId : secretManagerClient.listSecrets()) {
-      if (secretId.startsWith(SECRET_ID_PREFIX)) {
-        secretManagerClient.deleteSecret(secretId);
-      }
+    try {
+      secretManagerClient.deleteSecret(secretId);
+    } catch (NoSuchSecretResourceException e) {
+      // deleteSecret() deleted it already.
     }
   }
 
   @Test
   void createSecret_success() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     assertThat(secretManagerClient.listSecrets()).contains(secretId);
   }
 
   @Test
   void createSecret_duplicate() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     assertThrows(
         SecretAlreadyExistsException.class, () -> secretManagerClient.createSecret(secretId));
@@ -96,16 +104,25 @@ public class SecretManagerClientTest {
 
   @Test
   void addSecretVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     String version = secretManagerClient.addSecretVersion(secretId, "mydata");
     assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED))
         .containsExactly(version);
   }
 
+  @Test
+  void secretExists_true() {
+    secretManagerClient.createSecret(secretId);
+    assertThat(secretManagerClient.secretExists(secretId)).isTrue();
+  }
+
+  @Test
+  void secretExists_False() {
+    assertThat(secretManagerClient.secretExists(secretId)).isFalse();
+  }
+
   @Test
   void getSecretData_byVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     String version = secretManagerClient.addSecretVersion(secretId, "mydata");
     assertThat(secretManagerClient.getSecretData(secretId, Optional.of(version)))
@@ -114,15 +131,70 @@ public class SecretManagerClientTest {
 
   @Test
   void getSecretData_latestVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     secretManagerClient.addSecretVersion(secretId, "mydata");
     assertThat(secretManagerClient.getSecretData(secretId, Optional.empty())).isEqualTo("mydata");
   }
 
+  @Test
+  void disableSecretVersion() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.disableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
+  }
+
+  @Test
+  void disableSecretVersion_ignoreAlreadyDisabled() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.disableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
+    secretManagerClient.disableSecretVersion(secretId, version);
+  }
+
+  @Test
+  void disableSecretVersion_destroyed() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.destroySecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DESTROYED)).contains(version);
+    assertThrows(
+        SecretManagerException.class,
+        () -> secretManagerClient.disableSecretVersion(secretId, version));
+  }
+
+  @Test
+  void enableSecretVersion_ignoreAlreadyEnabled() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED)).contains(version);
+    secretManagerClient.enableSecretVersion(secretId, version);
+  }
+
+  @Test
+  void enableSecretVersion() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.disableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
+    secretManagerClient.enableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED)).contains(version);
+  }
+
+  @Test
+  void enableSecretVersion_destroyed() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.destroySecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DESTROYED)).contains(version);
+    assertThrows(
+        SecretManagerException.class,
+        () -> secretManagerClient.enableSecretVersion(secretId, version));
+  }
+
   @Test
   void destroySecretVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     String version = secretManagerClient.addSecretVersion(secretId, "mydata");
     secretManagerClient.destroySecretVersion(secretId, version);
@@ -134,7 +206,6 @@ public class SecretManagerClientTest {
 
   @Test
   void deleteSecret() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     assertThat(secretManagerClient.listSecrets()).contains(secretId);
     secretManagerClient.deleteSecret(secretId);

core/src/test/java/google/registry/privileges/secretmanager/SqlCredentialStoreTest.java

@@ -0,0 +1,63 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.cloud.secretmanager.v1.SecretVersionName;
+import google.registry.privileges.secretmanager.SqlUser.RobotId;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link SqlCredentialStore}. */
+public class SqlCredentialStoreTest {
+
+  private final SecretManagerClient client = new FakeSecretManagerClient();
+  private final SqlCredentialStore credentialStore = new SqlCredentialStore(client, "db");
+  private SqlUser user = new RobotUser(RobotId.NOMULUS);
+
+  @Test
+  void createSecret() {
+    credentialStore.createOrUpdateCredential(user, "password");
+    assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isTrue();
+    assertThat(
+            SecretVersionName.parse(
+                    client.getSecretData("sql-cred-live-label-nomulus-db", Optional.empty()))
+                .getSecret())
+        .isEqualTo("sql-cred-data-nomulus-db");
+    assertThat(client.secretExists("sql-cred-data-nomulus-db")).isTrue();
+    assertThat(client.getSecretData("sql-cred-data-nomulus-db", Optional.empty()))
+        .isEqualTo("nomulus password");
+  }
+
+  @Test
+  void getCredential() {
+    credentialStore.createOrUpdateCredential(user, "password");
+    SqlCredential credential = credentialStore.getCredential(user);
+    assertThat(credential.login()).isEqualTo("nomulus");
+    assertThat(credential.password()).isEqualTo("password");
+  }
+
+  @Test
+  void deleteCredential() {
+    credentialStore.createOrUpdateCredential(user, "password");
+    assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isTrue();
+    assertThat(client.secretExists("sql-cred-data-nomulus-db")).isTrue();
+    credentialStore.deleteCredential(user);
+    assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isFalse();
+    assertThat(client.secretExists("sql-cred-data-nomulus-db")).isFalse();
+  }
+}

core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java

@@ -34,7 +34,7 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link Spec11RegistrarThreatMatchesParser}. */
-class Spec11RegistrarThreatMatchesParserTest {
+public class Spec11RegistrarThreatMatchesParserTest {
 
   private static final String TODAY = "2018-07-21";
   private static final String YESTERDAY = "2018-07-20";
@@ -66,6 +66,12 @@ class Spec11RegistrarThreatMatchesParserTest {
     assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))).isEmpty();
   }
 
+  @Test
+  void testNonexistent_returnsEmpty() throws Exception {
+    assertThat(parser.getRegistrarThreatMatches(LocalDate.parse(YESTERDAY).minusYears(1)))
+        .isEmpty();
+  }
+
   @Test
   void testFindPrevious_olderThanYesterdayFound() throws Exception {
     setupFile("spec11_fake_report_previous_day", "2018-07-14");
@@ -95,7 +101,7 @@ class Spec11RegistrarThreatMatchesParserTest {
   }
 
   /** The expected contents of the sample spec11 report file */
-  static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
+  public static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
     return ImmutableSet.of(getMatchA(), getMatchB());
   }
 

core/src/test/java/google/registry/testing/AppEngineExtension.java

@@ -454,10 +454,6 @@ public final class AppEngineExtension implements BeforeEachCallback, AfterEachCa
           .setEnvIsAdmin(userInfo.isAdmin());
     }
 
-    if (clock != null) {
-      helper.setClock(() -> clock.nowUtc().getMillis());
-    }
-
     if (withLocalModules) {
       helper.setEnvInstance("0");
     }

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -750,6 +750,20 @@ public class DatabaseHelper {
             .build());
   }
 
+  /** Persists and returns a {@link Registrar} with the specified registrarId. */
+  public static Registrar persistNewRegistrar(String registrarId) {
+    return persistNewRegistrar(registrarId, registrarId + " name", Registrar.Type.REAL, 100L);
+  }
+
+  /** Persists and returns a list of {@link Registrar}s with the specified registrarIds. */
+  public static ImmutableList<Registrar> persistNewRegistrars(String... registrarIds) {
+    ImmutableList.Builder<Registrar> newRegistrars = new ImmutableList.Builder<>();
+    for (String registrarId : registrarIds) {
+      newRegistrars.add(persistNewRegistrar(registrarId));
+    }
+    return newRegistrars.build();
+  }
+
   private static Iterable<BillingEvent> getBillingEvents() {
     return transactIfJpaTm(
         () ->

core/src/test/java/google/registry/testing/DualDatabaseTestInvocationContextProvider.java

@@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import google.registry.persistence.transaction.TransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import java.lang.reflect.Field;
@@ -160,6 +161,14 @@ class DualDatabaseTestInvocationContextProvider implements TestTemplateInvocatio
 
   private static boolean isDualDatabaseTest(ExtensionContext context) {
     Object testInstance = context.getTestInstance().orElseThrow(RuntimeException::new);
-    return testInstance.getClass().isAnnotationPresent(DualDatabaseTest.class);
+    // If the test method is declared in its parent class,
+    // e.g. google.registry.flows.ResourceFlowTestCase.testRequiresLogin,
+    // we don't consider it is a DualDatabaseTest. This is because there may exist some subclasses
+    // that have not been migrated to DualDatabaseTest.
+    boolean isDeclaredTestMethod =
+        ImmutableSet.copyOf(testInstance.getClass().getDeclaredMethods())
+            .contains(context.getTestMethod().orElseThrow(RuntimeException::new));
+    return testInstance.getClass().isAnnotationPresent(DualDatabaseTest.class)
+        && isDeclaredTestMethod;
   }
 }

core/src/test/java/google/registry/tools/CommandTestCase.java

@@ -17,7 +17,8 @@ package google.registry.tools;
 import static com.google.common.collect.Iterables.concat;
 import static com.google.common.collect.Iterables.toArray;
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.joda.time.DateTimeZone.UTC;
 
@@ -116,7 +117,7 @@ public abstract class CommandTestCase<C extends Command> {
     } finally {
       // Clear the session cache so that subsequent reads for verification purposes hit Datastore.
       // This primarily matters for AutoTimestamp fields, which otherwise won't have updated values.
-      ofy().clearSessionCache();
+      tm().clearSessionCache();
       // Reset back to UNITTEST environment.
       RegistryToolEnvironment.UNITTEST.setup(systemPropertyExtension);
     }
@@ -185,12 +186,12 @@ public abstract class CommandTestCase<C extends Command> {
 
   /** Reloads the given resource from Datastore. */
   <T> T reloadResource(T resource) {
-    return ofy().load().entity(resource).now();
+    return transactIfJpaTm(() -> tm().load(resource));
   }
 
   /** Returns count of all poll messages in Datastore. */
   int getPollMessageCount() {
-    return ofy().load().type(PollMessage.class).count();
+    return transactIfJpaTm(() -> tm().loadAll(PollMessage.class).size());
   }
 
   /**