Add BillingVKey to restore symmetric VKey in GracePeriodBase (#902)

* Use PollMessageVKey to replace VKey<PollMessage> in DomainBase

* Revert changes to DomainContent

* Use BillingVKey in GracePeriodBase to restore symmetric vkey

* Rebase on HEAD

config/presubmits.py

@@ -65,7 +65,7 @@ class PresubmitCheck:
     for pattern in self.skipped_patterns:
       if pattern in file:
         return False
-    with open(file, "r") as f:
+    with open(file, "r", encoding='utf8') as f:
       file_content = f.read()
       matches = re.match(self.regex, file_content, re.DOTALL)
       if self.regex_type == FORBIDDEN:
@@ -241,7 +241,7 @@ def verify_flyway_index():
 
   # Remove the sequence numbers and compare against the index file contents.
   files = [filename[1] for filename in sorted(files)]
-  with open('db/src/main/resources/sql/flyway.txt') as index:
+  with open('db/src/main/resources/sql/flyway.txt', encoding='utf8') as index:
     indexed_files = index.read().splitlines()
   if files != indexed_files:
     unindexed = set(files) - set(indexed_files)

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -15,10 +15,10 @@
 package google.registry.backup;
 
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.BackupUtils.createDeserializingIterator;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Streams;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -31,7 +31,6 @@ import java.io.InputStream;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.util.Iterator;
-import java.util.stream.Stream;
 
 /**
  * Helpers for reading CommitLog records from a file.
@@ -43,6 +42,56 @@ public final class CommitLogImports {
 
   private CommitLogImports() {}
 
+  /**
+   * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link
+   * ImmutableList} of {@link ImmutableList}s of {@link VersionedEntity} records where the inner
+   * lists each consist of one transaction. Upon completion the {@code inputStream} is closed.
+   *
+   * <p>The returned list may be empty, since CommitLogs are written at fixed intervals regardless
+   * if actual changes exist. Each sublist, however, will not be empty.
+   *
+   * <p>A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated)
+   * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence
+   * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
+   * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
+   */
+  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+      InputStream inputStream) {
+    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
+        InputStream input = new BufferedInputStream(inputStream)) {
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+      checkState(commitLogs.hasNext());
+      checkState(commitLogs.next() instanceof CommitLogCheckpoint);
+
+      ImmutableList.Builder<ImmutableList<VersionedEntity>> resultBuilder =
+          new ImmutableList.Builder<>();
+      ImmutableList.Builder<VersionedEntity> currentTransactionBuilder =
+          new ImmutableList.Builder<>();
+
+      while (commitLogs.hasNext()) {
+        ImmutableObject currentObject = commitLogs.next();
+        if (currentObject instanceof CommitLogManifest) {
+          // CommitLogManifest means we are starting a new transaction
+          addIfNonempty(resultBuilder, currentTransactionBuilder);
+          currentTransactionBuilder = new ImmutableList.Builder<>();
+          VersionedEntity.fromManifest((CommitLogManifest) currentObject)
+              .forEach(currentTransactionBuilder::add);
+        } else if (currentObject instanceof CommitLogMutation) {
+          currentTransactionBuilder.add(
+              VersionedEntity.fromMutation((CommitLogMutation) currentObject));
+        } else {
+          throw new IllegalStateException(
+              String.format("Unknown entity type %s in commit logs", currentObject.getClass()));
+        }
+      }
+      // Add the last transaction in (if it's not empty)
+      addIfNonempty(resultBuilder, currentTransactionBuilder);
+      return resultBuilder.build();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   /**
    * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link
    * ImmutableList} of {@link VersionedEntity} records. Upon completion the {@code inputStream} is
@@ -57,23 +106,9 @@ public final class CommitLogImports {
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
   public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
-    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
-        InputStream input = new BufferedInputStream(inputStream)) {
-      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
-      checkState(commitLogs.hasNext());
-      checkState(commitLogs.next() instanceof CommitLogCheckpoint);
-
-      return Streams.stream(commitLogs)
-          .map(
-              e ->
-                  e instanceof CommitLogManifest
-                      ? VersionedEntity.fromManifest((CommitLogManifest) e)
-                      : Stream.of(VersionedEntity.fromMutation((CommitLogMutation) e)))
-          .flatMap(s -> s)
-          .collect(ImmutableList.toImmutableList());
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
+    return loadEntitiesByTransaction(inputStream).stream()
+        .flatMap(ImmutableList::stream)
+        .collect(toImmutableList());
   }
 
   /** Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link File}. */
@@ -92,4 +127,13 @@ public final class CommitLogImports {
   public static ImmutableList<VersionedEntity> loadEntities(ReadableByteChannel channel) {
     return loadEntities(Channels.newInputStream(channel));
   }
+
+  private static void addIfNonempty(
+      ImmutableList.Builder<ImmutableList<VersionedEntity>> resultBuilder,
+      ImmutableList.Builder<VersionedEntity> currentTransactionBuilder) {
+    ImmutableList<VersionedEntity> currentTransaction = currentTransactionBuilder.build();
+    if (!currentTransaction.isEmpty()) {
+      resultBuilder.add(currentTransaction);
+    }
+  }
 }

core/src/main/java/google/registry/backup/GcsDiffFileLister.java

@@ -122,7 +122,7 @@ class GcsDiffFileLister {
 
     // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
     // last file that we found) and finding its previous file until we either run out of files or
-    // get to one that preceeds "fromTime".
+    // get to one that precedes "fromTime".
     //
     // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
     // metadata of a file is sufficient to identify the preceding file, so if we start from the

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -0,0 +1,188 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
+import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static org.joda.time.Duration.standardHours;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.Key;
+import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig;
+import google.registry.model.server.Lock;
+import google.registry.model.translators.VKeyTranslatorFactory;
+import google.registry.persistence.VKey;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
+import google.registry.schema.replay.NonReplicatedEntity;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import google.registry.util.RequestStatusChecker;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletResponse;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/** Action that replays commit logs to Cloud SQL to keep it up to date. */
+@Action(
+    service = Action.Service.BACKEND,
+    path = ReplayCommitLogsToSqlAction.PATH,
+    method = Action.Method.POST,
+    automaticallyPrintOk = true,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class ReplayCommitLogsToSqlAction implements Runnable {
+
+  static final String PATH = "/_dr/task/replayCommitLogsToSql";
+
+  private static final int BLOCK_SIZE =
+      1024 * 1024; // Buffer 1mb at a time, for no particular reason.
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final Duration LEASE_LENGTH = standardHours(1);
+
+  @Inject GcsService gcsService;
+  @Inject Response response;
+  @Inject RequestStatusChecker requestStatusChecker;
+  @Inject GcsDiffFileLister diffLister;
+
+  @Inject
+  ReplayCommitLogsToSqlAction() {}
+
+  @Override
+  public void run() {
+    if (!RegistryConfig.getCloudSqlReplayCommitLogs()) {
+      String message = "ReplayCommitLogsToSqlAction was called but disabled in the config.";
+      logger.atWarning().log(message);
+      // App Engine will retry on any non-2xx status code, which we don't want in this case.
+      response.setStatus(SC_NO_CONTENT);
+      response.setPayload(message);
+      return;
+    }
+    Optional<Lock> lock =
+        Lock.acquire(
+            this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
+    if (lock.isEmpty()) {
+      String message = "Can't acquire SQL commit log replay lock, aborting.";
+      logger.atSevere().log(message);
+      // App Engine will retry on any non-2xx status code, which we don't want in this case.
+      // Let the next run after the next export happen naturally.
+      response.setStatus(SC_NO_CONTENT);
+      response.setPayload(message);
+      return;
+    }
+    try {
+      replayFiles();
+      response.setStatus(HttpServletResponse.SC_OK);
+      logger.atInfo().log("ReplayCommitLogsToSqlAction completed successfully.");
+    } finally {
+      lock.ifPresent(Lock::release);
+    }
+  }
+
+  private void replayFiles() {
+    // Start at the first millisecond we haven't seen yet
+    DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    // If there's an inconsistent file set, this will throw IllegalStateException and the job
+    // will try later -- this is likely because an export hasn't finished yet.
+    ImmutableList<GcsFileMetadata> commitLogFiles =
+        diffLister.listDiffFiles(fromTime, /* current time */ null);
+    for (GcsFileMetadata metadata : commitLogFiles) {
+      // One transaction per GCS file
+      jpaTm().transact(() -> processFile(metadata));
+    }
+    logger.atInfo().log("Replayed %d commit log files to SQL successfully.", commitLogFiles.size());
+  }
+
+  private void processFile(GcsFileMetadata metadata) {
+    try (InputStream input =
+        Channels.newInputStream(
+            gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+      // Load and process the Datastore transactions one at a time
+      ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
+          CommitLogImports.loadEntitiesByTransaction(input);
+      allTransactions.forEach(this::replayTransaction);
+      // if we succeeded, set the last-seen time
+      DateTime checkpoint =
+          DateTime.parse(
+              metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
+      SqlReplayCheckpoint.set(checkpoint);
+      logger.atInfo().log("Replayed %d transactions from commit log file.", allTransactions.size());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void replayTransaction(ImmutableList<VersionedEntity> transaction) {
+    transaction.stream()
+        .sorted(ReplayCommitLogsToSqlAction::compareByWeight)
+        .forEach(
+            versionedEntity ->
+                versionedEntity
+                    .getEntity()
+                    .ifPresentOrElse(
+                        this::handleEntityPut, () -> handleEntityDelete(versionedEntity)));
+  }
+
+  private void handleEntityPut(Entity entity) {
+    Object ofyPojo = ofy().toPojo(entity);
+    if (ofyPojo instanceof DatastoreEntity) {
+      DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
+      datastoreEntity.toSqlEntity().ifPresent(jpaTm()::put);
+    } else {
+      // this should never happen, but we shouldn't fail on it
+      logger.atSevere().log(
+          "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
+          ofyPojo.getClass());
+    }
+  }
+
+  private void handleEntityDelete(VersionedEntity entityToDelete) {
+    Key key = entityToDelete.key();
+    VKey<?> entityVKey;
+    try {
+      entityVKey = VKeyTranslatorFactory.createVKey(key);
+    } catch (RuntimeException e) {
+      // This means that the key wasn't convertible to VKey through the standard methods or via
+      // a createVKey method. This means that the object isn't persisted in SQL so we ignore it.
+      logger.atInfo().log(
+          "Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
+      return;
+    }
+    Class<?> entityClass = entityVKey.getKind();
+    // Delete the key iff the class represents a JPA entity that is replicated
+    if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
+        && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
+        && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
+      jpaTm().delete(entityVKey);
+    }
+  }
+
+  private static int compareByWeight(VersionedEntity a, VersionedEntity b) {
+    return getEntityPriority(a.key().getKind(), a.getEntity().isEmpty())
+        - getEntityPriority(b.key().getKind(), b.getEntity().isEmpty());
+  }
+}

core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java

@@ -50,11 +50,21 @@ public class ResaveAllEppResourcesAction implements Runnable {
   @Inject Response response;
   @Inject ResaveAllEppResourcesAction() {}
 
+  /**
+   * The number of shards to run the map-only mapreduce on.
+   *
+   * <p>This is less than the default of 100 because we only run this action monthly and can afford
+   * it being slower, but we don't want to write out lots of large commit logs in a short period of
+   * time because they make the Cloud SQL migration tougher.
+   */
+  private static final int NUM_SHARDS = 10;
+
   @Override
   public void run() {
     mrRunner
         .setJobName("Re-save all EPP resources")
         .setModuleName("backend")
+        .setDefaultMapShards(NUM_SHARDS)
         .runMapOnly(
             new ResaveAllEppResourcesActionMapper(),
             ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))

core/src/main/java/google/registry/beam/initsql/BeamJpaModule.java

@@ -31,6 +31,7 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.JdbcJpaTm;
 import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -168,6 +169,7 @@ public class BeamJpaModule {
         BeamJpaModule.class,
         KmsModule.class,
         PersistenceModule.class,
+        SecretManagerModule.class,
         UtilsModule.class
       })
   public interface JpaTransactionManagerComponent {

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Throwables.throwIfUnchecked;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.JpaRetries.isFailedTxnRetriable;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.setJpaTm;
@@ -68,6 +69,7 @@ import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -264,9 +266,9 @@ public final class Transforms {
   }
 
   /**
-   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
-   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
-   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   * Returns a {@link PTransform} that writes a {@link PCollection} of {@link VersionedEntity}s to a
+   * SQL database. and outputs an empty {@code PCollection<Void>}. This allows other operations to
+   * {@link org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
    *
    * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
    * job, we will not add features unless proven necessary.
@@ -282,16 +284,53 @@ public final class Transforms {
       int maxWriters,
       int batchSize,
       SerializableSupplier<JpaTransactionManager> jpaSupplier) {
-    return new PTransform<PCollection<VersionedEntity>, PCollection<Void>>() {
+    return writeToSql(
+        transformId,
+        maxWriters,
+        batchSize,
+        jpaSupplier,
+        (e) -> ofy().toPojo(e.getEntity().get()),
+        TypeDescriptor.of(VersionedEntity.class));
+  }
+
+  /**
+   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
+   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
+   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   *
+   * <p>The converter and type descriptor are generics so that we can convert any type of entity to
+   * an object to be placed in SQL.
+   *
+   * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
+   * job, we will not add features unless proven necessary.
+   *
+   * @param transformId a unique ID for an instance of the returned transform
+   * @param maxWriters the max number of concurrent writes to SQL, which also determines the max
+   *     number of connection pools created
+   * @param batchSize the number of entities to write in each operation
+   * @param jpaSupplier supplier of a {@link JpaTransactionManager}
+   * @param jpaConverter the function that converts the input object to a JPA entity
+   * @param objectDescriptor the type descriptor of the input object
+   */
+  public static <T> PTransform<PCollection<T>, PCollection<Void>> writeToSql(
+      String transformId,
+      int maxWriters,
+      int batchSize,
+      SerializableSupplier<JpaTransactionManager> jpaSupplier,
+      SerializableFunction<T, Object> jpaConverter,
+      TypeDescriptor<T> objectDescriptor) {
+    return new PTransform<PCollection<T>, PCollection<Void>>() {
       @Override
-      public PCollection<Void> expand(PCollection<VersionedEntity> input) {
+      public PCollection<Void> expand(PCollection<T> input) {
         return input
             .apply(
                 "Shard data for " + transformId,
-                MapElements.into(kvs(integers(), TypeDescriptor.of(VersionedEntity.class)))
+                MapElements.into(kvs(integers(), objectDescriptor))
                     .via(ve -> KV.of(ThreadLocalRandom.current().nextInt(maxWriters), ve)))
             .apply("Batch output by shard " + transformId, GroupIntoBatches.ofSize(batchSize))
-            .apply("Write in batch for " + transformId, ParDo.of(new SqlBatchWriter(jpaSupplier)));
+            .apply(
+                "Write in batch for " + transformId,
+                ParDo.of(new SqlBatchWriter<T>(jpaSupplier, jpaConverter)));
       }
     };
   }
@@ -385,18 +424,22 @@ public final class Transforms {
    * to hold the {@code JpaTransactionManager} instance, we must ensure that JpaTransactionManager
    * is not changed or torn down while being used by some instance.
    */
-  private static class SqlBatchWriter extends DoFn<KV<Integer, Iterable<VersionedEntity>>, Void> {
+  private static class SqlBatchWriter<T> extends DoFn<KV<Integer, Iterable<T>>, Void> {
 
     private static int instanceCount = 0;
     private static JpaTransactionManager originalJpa;
 
     private final SerializableSupplier<JpaTransactionManager> jpaSupplier;
+    private final SerializableFunction<T, Object> jpaConverter;
 
     private transient Ofy ofy;
     private transient SystemSleeper sleeper;
 
-    SqlBatchWriter(SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+    SqlBatchWriter(
+        SerializableSupplier<JpaTransactionManager> jpaSupplier,
+        SerializableFunction<T, Object> jpaConverter) {
       this.jpaSupplier = jpaSupplier;
+      this.jpaConverter = jpaConverter;
     }
 
     @Setup
@@ -429,13 +472,11 @@ public final class Transforms {
     }
 
     @ProcessElement
-    public void processElement(@Element KV<Integer, Iterable<VersionedEntity>> kv) {
+    public void processElement(@Element KV<Integer, Iterable<T>> kv) {
       try (AppEngineEnvironment env = new AppEngineEnvironment()) {
         ImmutableList<Object> ofyEntities =
             Streams.stream(kv.getValue())
-                .map(VersionedEntity::getEntity)
-                .map(Optional::get)
-                .map(ofy::toPojo)
+                .map(this.jpaConverter::apply)
                 .collect(ImmutableList.toImmutableList());
         retry(() -> jpaTm().transact(() -> jpaTm().putAll(ofyEntities)));
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -20,7 +20,7 @@ import static google.registry.beam.BeamUtils.getQueryFromFile;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
-import google.registry.backup.AppEngineEnvironment;
+import google.registry.beam.initsql.Transforms;
 import google.registry.beam.initsql.Transforms.SerializableSupplier;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
 import google.registry.config.CredentialModule.LocalCredential;
@@ -43,7 +43,6 @@ import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
-import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -191,34 +190,27 @@ public class Spec11Pipeline implements Serializable {
       PCollection<Subdomain> domains,
       EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
       ValueProvider<String> dateProvider) {
-
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsSql =
         domains.apply("Run through SafeBrowsing API", ParDo.of(evaluateSafeBrowsingFn));
-    /* Store ThreatMatch objects in SQL. */
+    TypeDescriptor<KV<Subdomain, ThreatMatch>> descriptor =
+        new TypeDescriptor<KV<Subdomain, ThreatMatch>>() {};
     subdomainsSql.apply(
-        ParDo.of(
-            new DoFn<KV<Subdomain, ThreatMatch>, Void>() {
-              @ProcessElement
-              public void processElement(ProcessContext context) {
-                // create the Spec11ThreatMatch from Subdomain and ThreatMatch
-                try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-                  Subdomain subdomain = context.element().getKey();
-                  Spec11ThreatMatch threatMatch =
-                      new Spec11ThreatMatch.Builder()
-                          .setThreatTypes(
-                              ImmutableSet.of(
-                                  ThreatType.valueOf(context.element().getValue().threatType())))
-                          .setCheckDate(
-                              LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
-                          .setDomainName(subdomain.domainName())
-                          .setDomainRepoId(subdomain.domainRepoId())
-                          .setRegistrarId(subdomain.registrarId())
-                          .build();
-                  JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
-                  jpaTransactionManager.transact(() -> jpaTransactionManager.insert(threatMatch));
-                }
-              }
-            }));
+        Transforms.writeToSql(
+            "Spec11ThreatMatch",
+            4,
+            4,
+            jpaSupplierFactory,
+            (kv) -> {
+              Subdomain subdomain = kv.getKey();
+              return new Spec11ThreatMatch.Builder()
+                  .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
+                  .setCheckDate(LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
+                  .setDomainName(subdomain.domainName())
+                  .setDomainRepoId(subdomain.domainRepoId())
+                  .setRegistrarId(subdomain.registrarId())
+                  .build();
+            },
+            descriptor));
 
     /* Store ThreatMatch objects in JSON. */
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -415,6 +415,14 @@ public final class RegistryConfig {
       return config.cloudSql.instanceConnectionName;
     }
 
+    @Provides
+    @Config("cloudSqlDbInstanceName")
+    public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
+      // Format of instanceConnectionName: project-id:region:instance-name
+      int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
+      return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);
+    }
+
     @Provides
     @Config("cloudDnsRootUrl")
     public static Optional<String> getCloudDnsRootUrl(RegistryConfigSettings config) {
@@ -1592,6 +1600,22 @@ public final class RegistryConfig {
     CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
   }
 
+  /**
+   * Returns whether or not to replay commit logs to the SQL database after export to GCS.
+   *
+   * <p>If true, we will trigger the {@link google.registry.backup.ReplayCommitLogsToSqlAction}
+   * after the {@link google.registry.backup.ExportCommitLogDiffAction} to load the commit logs and
+   * replay them to SQL.
+   */
+  public static boolean getCloudSqlReplayCommitLogs() {
+    return CONFIG_SETTINGS.get().cloudSql.replayCommitLogs;
+  }
+
+  @VisibleForTesting
+  public static void overrideCloudSqlReplayCommitLogs(boolean replayCommitLogs) {
+    CONFIG_SETTINGS.get().cloudSql.replayCommitLogs = replayCommitLogs;
+  }
+
   /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
   public static String getContactAndHostRoidSuffix() {
     return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -126,6 +126,7 @@ public class RegistryConfigSettings {
     public String username;
     public String instanceConnectionName;
     public boolean replicateTransactions;
+    public boolean replayCommitLogs;
   }
 
   /** Configuration for Apache Beam (Cloud Dataflow). */

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -233,6 +233,8 @@ cloudSql:
   # Set this to true to replicate cloud SQL transactions to datastore in the
   # background.
   replicateTransactions: false
+  # Set this to true to enable replay of commit logs to SQL
+  replayCommitLogs: false
 
 cloudDns:
   # Set both properties to null in Production.

core/src/main/java/google/registry/env/common/default/WEB-INF/queue.xml

@@ -208,6 +208,12 @@
     <max-concurrent-requests>5</max-concurrent-requests>
   </queue>
 
+  <!-- Queue for replaying commit logs to SQL during the transition from Datastore -> SQL. -->
+  <queue>
+    <name>replay-commit-logs-to-sql</name>
+    <rate>1/s</rate>
+  </queue>
+
   <!-- The load[0-9] queues are used for load-testing, and can be safely deleted
        in any environment that doesn't require load-testing. -->
   <queue>

core/src/main/java/google/registry/flows/EppRequestHandler.java

@@ -74,6 +74,14 @@ public class EppRequestHandler {
           && eppOutput.getResponse().getResult().getCode() == SUCCESS_AND_CLOSE) {
         response.setHeader("Epp-Session", "close");
       }
+      // If a login request returns a success, a logged-in header is added to the response to inform
+      // the proxy that it is no longer necessary to send the full client certificate to the backend
+      // for this connection.
+      if (eppOutput.isResponse()
+          && eppOutput.getResponse().isLoginResponse()
+          && eppOutput.isSuccess()) {
+        response.setHeader("Logged-In", "true");
+      }
     } catch (Exception e) {
       logger.atWarning().withCause(e).log("handleEppCommand general exception");
       response.setStatus(SC_BAD_REQUEST);

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -110,7 +110,6 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.persistence.VKey;
 import google.registry.tmch.LordnTaskUtils;
 import java.util.Optional;
@@ -372,7 +371,7 @@ public class DomainCreateFlow implements TransactionalFlow {
         && TokenType.SINGLE_USE.equals(allocationToken.get().getTokenType())) {
       entitiesToSave.add(
           allocationTokenFlowUtils.redeemToken(
-              allocationToken.get(), DomainHistoryVKey.create(Key.create(historyEntry))));
+              allocationToken.get(), HistoryEntry.createVKey(Key.create(historyEntry))));
     }
     enqueueTasks(newDomain, hasSignedMarks, hasClaimsNotice);
 

core/src/main/java/google/registry/flows/domain/token/AllocationTokenFlowUtils.java

@@ -32,7 +32,8 @@ import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.registry.Registry;
-import google.registry.persistence.DomainHistoryVKey;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -107,7 +108,7 @@ public class AllocationTokenFlowUtils {
 
   /** Redeems a SINGLE_USE {@link AllocationToken}, returning the redeemed copy. */
   public AllocationToken redeemToken(
-      AllocationToken token, DomainHistoryVKey redemptionHistoryEntry) {
+      AllocationToken token, VKey<? extends HistoryEntry> redemptionHistoryEntry) {
     checkArgument(
         TokenType.SINGLE_USE.equals(token.getTokenType()),
         "Only SINGLE_USE tokens can be marked as redeemed");

core/src/main/java/google/registry/flows/session/LoginFlow.java

@@ -141,7 +141,7 @@ public class LoginFlow implements Flow {
     sessionMetadata.resetFailedLoginAttempts();
     sessionMetadata.setClientId(login.getClientId());
     sessionMetadata.setServiceExtensionUris(serviceExtensionUrisBuilder.build());
-    return responseBuilder.build();
+    return responseBuilder.setIsLoginResponse().build();
   }
 
   /** Registrar with this client ID could not be found. */

core/src/main/java/google/registry/model/DatabaseMigrationUtils.java

@@ -0,0 +1,38 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryEnvironment;
+
+/** Utility methods related to migrating dual-read/dual-write entities. */
+public class DatabaseMigrationUtils {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Throws exceptions only in unit tests, otherwise only logs exceptions. */
+  public static void suppressExceptionUnlessInTest(Runnable work, String message) {
+    try {
+      work.run();
+    } catch (Exception e) {
+      if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)) {
+        throw e;
+      }
+      logger.atWarning().withCause(e).log(message);
+    }
+  }
+
+  private DatabaseMigrationUtils() {}
+}

core/src/main/java/google/registry/model/OteAccountBuilder.java

@@ -211,11 +211,6 @@ public final class OteAccountBuilder {
     return transformRegistrars(builder -> builder.setPassword(password));
   }
 
-  /** Sets the client certificate hash to all the OT&E Registrars. */
-  public OteAccountBuilder setCertificateHash(String certHash) {
-    return transformRegistrars(builder -> builder.setClientCertificateHash(certHash));
-  }
-
   /** Sets the client certificate to all the OT&E Registrars. */
   public OteAccountBuilder setCertificate(String asciiCert, DateTime now) {
     return transformRegistrars(builder -> builder.setClientCertificate(asciiCert, now));

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -28,6 +28,11 @@ import org.joda.time.DateTime;
  */
 public class UpdateAutoTimestamp extends ImmutableObject {
 
+  // When set to true, database converters/translators should do tha auto update.  When set to
+  // false, auto update should be suspended (this exists to allow us to preserve the original value
+  // during a replay).
+  static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
+
   DateTime timestamp;
 
   /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
@@ -40,4 +45,30 @@ public class UpdateAutoTimestamp extends ImmutableObject {
     instance.timestamp = timestamp;
     return instance;
   }
+
+  // TODO(b/175610935): Remove the auto-update disabling code below after migration.
+
+  /** Class to allow us to safely disable auto-update in a try-with-resources block. */
+  public static class DisableAutoUpdateResource implements AutoCloseable {
+    DisableAutoUpdateResource() {
+      autoUpdateEnabled.set(false);
+    }
+
+    @Override
+    public void close() {
+      autoUpdateEnabled.set(true);
+    }
+  }
+
+  /**
+   * Resturns a resource that disables auto-updates on all {@link UpdateAutoTimestamp}s in the
+   * current thread, suitable for use with in a try-with-resources block.
+   */
+  public static DisableAutoUpdateResource disableAutoUpdate() {
+    return new DisableAutoUpdateResource();
+  }
+
+  public static boolean autoUpdateEnabled() {
+    return autoUpdateEnabled.get();
+  }
 }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -290,7 +290,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "allocationToken")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
@@ -464,7 +464,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {
 
     /**
@@ -559,7 +559,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "billingTime")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billing time of the charge that is being cancelled. */
@@ -680,7 +680,7 @@ public abstract class BillingEvent extends ImmutableObject
   /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Modification extends BillingEvent implements DatastoreOnlyEntity {
 
     /** The change in cost that should be applied to the original billing event. */

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,7 +14,6 @@
 
 package google.registry.model.contact;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -111,8 +110,8 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
 
   // In Datastore, save as a HistoryEntry object regardless of this object's type
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(asHistoryEntry());
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(asHistoryEntry());
   }
 
   /** Class to represent the composite primary key of {@link ContactHistory} entity. */

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -17,7 +17,6 @@ package google.registry.model.domain;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
@@ -250,8 +249,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
 
   // In Datastore, save as a HistoryEntry object regardless of this object's type
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(asHistoryEntry());
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(asHistoryEntry());
   }
 
   /** Class to represent the composite primary key of {@link DomainHistory} entity. */

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -23,6 +23,8 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.ofy.ObjectifyService;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import javax.annotation.Nullable;
@@ -82,10 +84,8 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
     instance.domainRepoId = checkArgumentNotNull(domainRepoId);
     instance.expirationTime = checkArgumentNotNull(expirationTime);
     instance.clientId = checkArgumentNotNull(clientId);
-    instance.billingEventOneTime = billingEventOneTime;
-    instance.billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
-    instance.billingEventRecurring = billingEventRecurring;
-    instance.billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+    instance.billingEventOneTime = BillingEventVKey.create(billingEventOneTime);
+    instance.billingEventRecurring = BillingRecurrenceVKey.create(billingEventRecurring);
     return instance;
   }
 
@@ -178,7 +178,6 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
   public GracePeriod cloneAfterOfyLoad(String domainRepoId) {
     GracePeriod clone = clone(this);
     clone.domainRepoId = checkArgumentNotNull(domainRepoId);
-    clone.restoreHistoryIds();
     return clone;
   }
 
@@ -190,7 +189,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
    */
   public GracePeriod cloneWithRecurringBillingEvent(VKey<BillingEvent.Recurring> recurring) {
     GracePeriod clone = clone(this);
-    clone.billingEventRecurring = recurring;
+    clone.billingEventRecurring = BillingRecurrenceVKey.create(recurring);
     return clone;
   }
 
@@ -232,9 +231,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
       instance.expirationTime = gracePeriod.expirationTime;
       instance.clientId = gracePeriod.clientId;
       instance.billingEventOneTime = gracePeriod.billingEventOneTime;
-      instance.billingEventOneTimeHistoryId = gracePeriod.billingEventOneTimeHistoryId;
       instance.billingEventRecurring = gracePeriod.billingEventRecurring;
-      instance.billingEventRecurringHistoryId = gracePeriod.billingEventRecurringHistoryId;
       return instance;
     }
   }

core/src/main/java/google/registry/model/domain/GracePeriodBase.java

@@ -14,14 +14,14 @@
 
 package google.registry.model.domain;
 
-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ModelUtils;
 import google.registry.model.billing.BillingEvent;
-import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import java.lang.reflect.Field;
 import java.util.LinkedHashMap;
@@ -68,24 +68,16 @@ public class GracePeriodBase extends ImmutableObject {
    * billingEventRecurring}) or for redemption grace periods (since deletes have no cost).
    */
   // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  @Column(name = "billing_event_id")
-  VKey<OneTime> billingEventOneTime = null;
-
-  @Ignore
-  @Column(name = "billing_event_history_id")
-  Long billingEventOneTimeHistoryId;
+  @Access(AccessType.FIELD)
+  BillingEventVKey billingEventOneTime = null;
 
   /**
    * The recurring billing event corresponding to the action that triggered this grace period, if
    * applicable - i.e. if the action was an autorenew - or null in all other cases.
    */
   // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  @Column(name = "billing_recurrence_id")
-  VKey<BillingEvent.Recurring> billingEventRecurring = null;
-
-  @Ignore
-  @Column(name = "billing_recurrence_history_id")
-  Long billingEventRecurringHistoryId;
+  @Access(AccessType.FIELD)
+  BillingRecurrenceVKey billingEventRecurring = null;
 
   public long getGracePeriodId() {
     return gracePeriodId;
@@ -123,8 +115,7 @@ public class GracePeriodBase extends ImmutableObject {
    * period is not AUTO_RENEW.
    */
   public VKey<BillingEvent.OneTime> getOneTimeBillingEvent() {
-    restoreOfyKeys();
-    return billingEventOneTime;
+    return billingEventOneTime == null ? null : billingEventOneTime.createVKey();
   }
 
   /**
@@ -132,18 +123,7 @@ public class GracePeriodBase extends ImmutableObject {
    * period is AUTO_RENEW.
    */
   public VKey<BillingEvent.Recurring> getRecurringBillingEvent() {
-    restoreOfyKeys();
-    return billingEventRecurring;
-  }
-
-  /**
-   * Restores history ids for composite VKeys after a load from datastore.
-   *
-   * <p>For use by DomainContent.load() ONLY.
-   */
-  protected void restoreHistoryIds() {
-    billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
-    billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+    return billingEventRecurring == null ? null : billingEventRecurring.createVKey();
   }
 
   /**
@@ -152,7 +132,6 @@ public class GracePeriodBase extends ImmutableObject {
    */
   @Override
   protected Map<Field, Object> getSignificantFields() {
-    restoreOfyKeys();
     // Can't use streams or ImmutableMap because we can have null values.
     Map<Field, Object> result = new LinkedHashMap();
     for (Map.Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
@@ -162,33 +141,4 @@ public class GracePeriodBase extends ImmutableObject {
     }
     return result;
   }
-
-  /**
-   * Restores Ofy keys in the billing events.
-   *
-   * <p>This must be called by all methods that access the one time or recurring billing event keys.
-   * When the billing event keys are loaded from SQL, they are loaded as asymmetric keys because the
-   * database columns that we load them from do not contain all of the information necessary to
-   * reconsitute the Ofy side of the key. In other cases, we restore the Ofy key during the
-   * hibernate {@link javax.persistence.PostLoad} method from the other fields of the object, but we
-   * have been unable to make this work with hibernate's internal persistence model in this case
-   * because the {@link GracePeriod}'s hash code is evaluated prior to these calls, and would be
-   * invalidated by changing the fields.
-   */
-  private final synchronized void restoreOfyKeys() {
-    if (billingEventOneTime != null && !billingEventOneTime.maybeGetOfyKey().isPresent()) {
-      billingEventOneTime =
-          DomainBase.restoreOfyFrom(
-              Key.create(DomainBase.class, domainRepoId),
-              billingEventOneTime,
-              billingEventOneTimeHistoryId);
-    }
-    if (billingEventRecurring != null && !billingEventRecurring.maybeGetOfyKey().isPresent()) {
-      billingEventRecurring =
-          DomainBase.restoreOfyFrom(
-              Key.create(DomainBase.class, domainRepoId),
-              billingEventRecurring,
-              billingEventRecurringHistoryId);
-    }
-  }
 }

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java

@@ -14,11 +14,11 @@
 
 package google.registry.model.domain.secdns;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.ofy.ObjectifyService;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import java.util.Optional;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
@@ -86,7 +86,7 @@ public class DomainDsDataHistory extends DomainDsDataBase implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not persisted in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore
   }
 }

core/src/main/java/google/registry/model/domain/token/AllocationToken.java

@@ -111,9 +111,9 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
   @Nullable
   @Index
   @AttributeOverrides({
-    @AttributeOverride(name = "domainRepoId", column = @Column(name = "redemption_domain_repo_id")),
+    @AttributeOverride(name = "repoId", column = @Column(name = "redemption_domain_repo_id")),
     @AttributeOverride(
-        name = "domainHistoryId",
+        name = "historyRevisionId",
         column = @Column(name = "redemption_domain_history_id"))
   })
   DomainHistoryVKey redemptionHistoryEntry;
@@ -192,8 +192,9 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
     return token;
   }
 
-  public Optional<VKey<HistoryEntry>> getRedemptionHistoryEntry() {
-    return Optional.ofNullable(redemptionHistoryEntry);
+  public Optional<VKey<? extends HistoryEntry>> getRedemptionHistoryEntry() {
+    return Optional.ofNullable(
+        redemptionHistoryEntry == null ? null : redemptionHistoryEntry.createDomainHistoryVKey());
   }
 
   public boolean isRedeemed() {
@@ -291,9 +292,10 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
       return this;
     }
 
-    public Builder setRedemptionHistoryEntry(DomainHistoryVKey redemptionHistoryEntry) {
+    public Builder setRedemptionHistoryEntry(VKey<? extends HistoryEntry> redemptionHistoryEntry) {
+      checkArgumentNotNull(redemptionHistoryEntry, "Redemption history entry must not be null");
       getInstance().redemptionHistoryEntry =
-          checkArgumentNotNull(redemptionHistoryEntry, "Redemption history entry must not be null");
+          DomainHistoryVKey.create(redemptionHistoryEntry.getOfyKey());
       return this;
     }
 

core/src/main/java/google/registry/model/eppoutput/EppResponse.java

@@ -65,6 +65,7 @@ import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlElementRef;
 import javax.xml.bind.annotation.XmlElementRefs;
 import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.XmlType;
 
 /**
@@ -87,6 +88,9 @@ public class EppResponse extends ImmutableObject implements ResponseOrGreeting {
   /** The command result. The RFC allows multiple failure results, but we always return one. */
   Result result;
 
+  /** Indicates if this response is for a login request. */
+  @XmlTransient boolean isLoginResponse = false;
+
   /**
    * Information about messages queued for retrieval. This may appear in response to any EPP message
    * (if messages are queued), but in practice this will only be set in response to a poll request.
@@ -178,6 +182,10 @@ public class EppResponse extends ImmutableObject implements ResponseOrGreeting {
     return result;
   }
 
+  public boolean isLoginResponse() {
+    return isLoginResponse;
+  }
+
   /** Marker interface for types that can go in the {@link #resData} field. */
   public interface ResponseData {}
 
@@ -222,5 +230,10 @@ public class EppResponse extends ImmutableObject implements ResponseOrGreeting {
       getInstance().extensions = forceEmptyToNull(extensions);
       return this;
     }
+
+    public Builder setIsLoginResponse() {
+      getInstance().isLoginResponse = true;
+      return this;
+    }
   }
 }

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,7 +14,6 @@
 
 package google.registry.model.host;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -112,8 +111,8 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
 
   // In Datastore, save as a HistoryEntry object regardless of this object's type
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(asHistoryEntry());
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(asHistoryEntry());
   }
 
   /** Class to represent the composite primary key of {@link HostHistory} entity. */

core/src/main/java/google/registry/model/ofy/EntityWritePriorities.java

@@ -0,0 +1,60 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.ofy;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Contains the mapping from class names to SQL-replay-write priorities.
+ *
+ * <p>When replaying Datastore commit logs to SQL (asynchronous replication), in order to avoid
+ * issues with foreign keys, we should replay entity writes so that foreign key references are
+ * always written after the entity that they reference. This class represents that DAG, where lower
+ * values represent an earlier write (and later delete). Higher-valued classes can have foreign keys
+ * on lower-valued classes, but not vice versa.
+ */
+public class EntityWritePriorities {
+
+  /**
+   * Mapping from class name to "priority".
+   *
+   * <p>Here, "priority" means the order in which the class should be inserted / updated in a
+   * transaction with respect to instances of other classes. By default, all classes have a priority
+   * number of zero.
+   *
+   * <p>For each transaction, classes should be written in priority order from the lowest number to
+   * the highest, in order to maintain foreign-key write consistency. For the same reason, deletes
+   * should happen after all writes.
+   */
+  static final ImmutableMap<String, Integer> CLASS_PRIORITIES =
+      ImmutableMap.of(
+          "HistoryEntry", -10,
+          "AllocationToken", -9,
+          "ContactResource", 5,
+          "DomainBase", 10);
+
+  // The beginning of the range of priority numbers reserved for delete.  This must be greater than
+  // any of the values in CLASS_PRIORITIES by enough overhead to accommodate  any negative values in
+  // it. Note: by design, deletions will happen in the opposite order of insertions, which is
+  // necessary to make sure foreign keys aren't violated during deletion.
+  @VisibleForTesting static final int DELETE_RANGE = Integer.MAX_VALUE / 2;
+
+  /** Returns the priority of the entity type in the map entry. */
+  public static int getEntityPriority(String kind, boolean isDelete) {
+    int priority = CLASS_PRIORITIES.getOrDefault(kind, 0);
+    return isDelete ? DELETE_RANGE - priority : priority;
+  }
+}

core/src/main/java/google/registry/model/ofy/ObjectifyService.java

@@ -42,8 +42,8 @@ import google.registry.model.translators.CidrAddressBlockTranslatorFactory;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
 import google.registry.model.translators.CreateAutoTimestampTranslatorFactory;
 import google.registry.model.translators.CurrencyUnitTranslatorFactory;
-import google.registry.model.translators.DomainHistoryVKeyTranslatorFactory;
 import google.registry.model.translators.DurationTranslatorFactory;
+import google.registry.model.translators.EppHistoryVKeyTranslatorFactory;
 import google.registry.model.translators.InetAddressTranslatorFactory;
 import google.registry.model.translators.ReadableInstantUtcTranslatorFactory;
 import google.registry.model.translators.UpdateAutoTimestampTranslatorFactory;
@@ -128,7 +128,7 @@ public class ObjectifyService {
             new CreateAutoTimestampTranslatorFactory(),
             new CurrencyUnitTranslatorFactory(),
             new DurationTranslatorFactory(),
-            new DomainHistoryVKeyTranslatorFactory(),
+            new EppHistoryVKeyTranslatorFactory(),
             new InetAddressTranslatorFactory(),
             new MoneyStringTranslatorFactory(),
             new ReadableInstantUtcTranslatorFactory(),

core/src/main/java/google/registry/model/ofy/TransactionInfo.java

@@ -20,6 +20,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Maps.filterValues;
 import static com.google.common.collect.Maps.toMap;
 import static google.registry.model.ofy.CommitLogBucket.getArbitraryBucketId;
+import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
@@ -29,7 +30,6 @@ import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
 import java.util.Map;
 import org.joda.time.DateTime;
 
@@ -61,7 +61,7 @@ class TransactionInfo {
 
   TransactionInfo(DateTime now) {
     this.transactionTime = now;
-    ofy().load().key(bucketKey);  // Asynchronously load value into session cache.
+    ofy().load().key(bucketKey); // Asynchronously load value into session cache.
   }
 
   TransactionInfo setReadOnly() {
@@ -101,23 +101,10 @@ class TransactionInfo {
         .collect(toImmutableSet());
   }
 
-  // Mapping from class name to "weight" (which in this case is the order in which the class must
-  // be "put" in a transaction with respect to instances of other classes).  Lower weight classes
-  // are put first, by default all classes have a weight of zero.
-  static final ImmutableMap<String, Integer> CLASS_WEIGHTS =
-      ImmutableMap.of(
-          "HistoryEntry", -1,
-          "DomainBase", 1);
-
-  // The beginning of the range of weights reserved for delete.  This must be greater than any of
-  // the values in CLASS_WEIGHTS by enough overhead to accomodate any negative values in it.
-  @VisibleForTesting static final int DELETE_RANGE = Integer.MAX_VALUE / 2;
-
   /** Returns the weight of the entity type in the map entry. */
   @VisibleForTesting
   static int getWeight(ImmutableMap.Entry<Key<?>, Object> entry) {
-    int weight = CLASS_WEIGHTS.getOrDefault(entry.getKey().getKind(), 0);
-    return entry.getValue().equals(Delete.SENTINEL) ? DELETE_RANGE - weight : weight;
+    return getEntityPriority(entry.getKey().getKind(), entry.getValue().equals(Delete.SENTINEL));
   }
 
   private static int compareByWeight(
@@ -137,10 +124,9 @@ class TransactionInfo {
                         if (entry.getValue().equals(Delete.SENTINEL)) {
                           jpaTm().delete(VKey.from(entry.getKey()));
                         } else {
-                          for (SqlEntity entity :
-                              ((DatastoreEntity) entry.getValue()).toSqlEntities()) {
-                            jpaTm().put(entity);
-                          }
+                          ((DatastoreEntity) entry.getValue())
+                              .toSqlEntity()
+                              .ifPresent(jpaTm()::put);
                         }
                       });
             });

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -279,7 +279,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("ONE_TIME")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends PollMessage {
 
     // Response data. Objectify cannot persist a base class type, so we must have a separate field
@@ -432,7 +432,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("AUTORENEW")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -856,26 +856,6 @@ public class Registrar extends ImmutableObject
       }
     }
 
-    /**
-     * Sets client certificate hash, but not the certificate.
-     *
-     * <p><b>Warning:</b> {@link #setClientCertificate(String, DateTime)} sets the hash for you and
-     * is preferred. Calling this method will nullify the {@code clientCertificate} field.
-     */
-    public Builder setClientCertificateHash(String clientCertificateHash) {
-      if (clientCertificateHash != null) {
-        checkArgument(
-            Pattern.matches("[A-Za-z0-9+/]+", clientCertificateHash),
-            "--cert_hash not a valid base64 (no padding) value");
-        checkArgument(
-            base64().decode(clientCertificateHash).length == 256 / 8,
-            "--cert_hash base64 does not decode to 256 bits");
-      }
-      getInstance().clientCertificate = null;
-      getInstance().clientCertificateHash = clientCertificateHash;
-      return this;
-    }
-
     public Builder setContactsRequireSyncing(boolean contactsRequireSyncing) {
       getInstance().contactsRequireSyncing = contactsRequireSyncing;
       return this;

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -18,7 +18,6 @@ import static com.googlecode.objectify.Key.getKind;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -45,6 +44,7 @@ import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.persistence.Access;
@@ -314,8 +314,8 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
 
   // In SQL, save the child type
   @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of((SqlEntity) toChildHistoryEntity());
+  public Optional<SqlEntity> toSqlEntity() {
+    return Optional.of((SqlEntity) toChildHistoryEntity());
   }
 
   /** Creates a {@link VKey} instance from a {@link Key} instance. */

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java

@@ -18,13 +18,13 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import google.registry.util.DomainNameUtils;
+import java.util.Optional;
 import java.util.Set;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -111,8 +111,8 @@ public class Spec11ThreatMatch extends ImmutableObject implements Buildable, Sql
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not stored in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore
   }
 
   @Override

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatchDao.java

@@ -21,10 +21,10 @@ import javax.persistence.TemporalType;
 import org.joda.time.LocalDate;
 
 /**
- * Data access object for {@link google.registry.model.reporting.Spec11ThreatMatch}.
+ * Data access object for {@link Spec11ThreatMatch}.
  *
- * <p>A JpaTransactionManager is passed into each static method because they are called from a BEAM
- * pipeline and we don't know where it's coming from.
+ * <p>The transaction manager is passed as a parameter because this could be called either from a
+ * BEAM pipeline or standard non-BEAM code.
  */
 public class Spec11ThreatMatchDao {
 

core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java

@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.isEmpty;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -32,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EmbedMap;
 import com.googlecode.objectify.annotation.Entity;
@@ -82,8 +82,6 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 public class SignedMarkRevocationList extends ImmutableObject implements NonReplicatedEntity {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
   @VisibleForTesting static final int SHARD_SIZE = 10000;
 
   /** Common ancestor for queries. */
@@ -121,12 +119,11 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
       memoizeWithShortExpiration(
           () -> {
             SignedMarkRevocationList datastoreList = loadFromDatastore();
-            // Also load the list from Cloud SQL, compare the two lists, and log if different.
-            try {
-              loadAndCompareCloudSqlList(datastoreList);
-            } catch (Throwable t) {
-              logger.atSevere().withCause(t).log("Error comparing signed mark revocation lists.");
-            }
+            suppressExceptionUnlessInTest(
+                () -> {
+                  loadAndCompareCloudSqlList(datastoreList);
+                },
+                "Error comparing signed mark revocation lists.");
             return datastoreList;
           });
 
@@ -229,11 +226,12 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
           Maps.difference(datastoreList.revokes, cloudSqlList.revokes);
       if (!diff.areEqual()) {
         if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
+          String message =
               String.format(
                   "Unequal SM revocation lists detected, Cloud SQL list with revision id %d has %d"
                       + " different records than the current Datastore list.",
-                  cloudSqlList.revisionId, diff.entriesDiffering().size()));
+                  cloudSqlList.revisionId, diff.entriesDiffering().size());
+          throw new RuntimeException(message);
         } else {
           StringBuilder diffMessage = new StringBuilder("Unequal SM revocation lists detected:\n");
           diff.entriesDiffering()
@@ -243,11 +241,13 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
                           String.format(
                               "SMD %s has key %s in Datastore and key %s in Cloud SQL.\n",
                               label, valueDiff.leftValue(), valueDiff.rightValue())));
-          logger.atWarning().log(diffMessage.toString());
+          throw new RuntimeException(diffMessage.toString());
         }
       }
     } else {
-      logger.atWarning().log("Signed mark revocation list in Cloud SQL is empty.");
+      if (datastoreList.size() != 0) {
+        throw new RuntimeException("Signed mark revocation list in Cloud SQL is empty.");
+      }
     }
   }
 

core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java

@@ -15,6 +15,7 @@
 package google.registry.model.smd;
 
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.common.base.Supplier;
@@ -60,14 +61,14 @@ public class SignedMarkRevocationListDao {
    * the authoritative database.
    */
   static void trySave(SignedMarkRevocationList signedMarkRevocationList) {
-    try {
-      SignedMarkRevocationListDao.save(signedMarkRevocationList);
-      logger.atInfo().log(
-          "Inserted %,d signed mark revocations into Cloud SQL",
-          signedMarkRevocationList.revokes.size());
-    } catch (Throwable e) {
-      logger.atSevere().withCause(e).log("Error inserting signed mark revocations into Cloud SQL");
-    }
+    suppressExceptionUnlessInTest(
+        () -> {
+          SignedMarkRevocationListDao.save(signedMarkRevocationList);
+          logger.atInfo().log(
+              "Inserted %,d signed mark revocations into Cloud SQL.",
+              signedMarkRevocationList.revokes.size());
+        },
+        "Error inserting signed mark revocations into Cloud SQL.");
   }
 
   private static void save(SignedMarkRevocationList signedMarkRevocationList) {

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -20,7 +20,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.annotations.NotBackedUp;
@@ -28,9 +27,7 @@ import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
 import google.registry.model.tmch.TmchCrl.TmchCrlId;
 import google.registry.persistence.VKey;
-import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.NonReplicatedEntity;
-import google.registry.schema.replay.SqlEntity;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
@@ -106,16 +103,6 @@ public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEnt
     return updated;
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // dually-written
-  }
-
-  @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // dually-written
-  }
-
   static class TmchCrlId implements Serializable {
 
     @Column(name = "certificateRevocations")

core/src/main/java/google/registry/model/translators/DomainHistoryVKeyTranslatorFactory.java

@@ -1,48 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.model.translators;
-
-import com.google.appengine.api.datastore.Key;
-import google.registry.persistence.DomainHistoryVKey;
-import javax.annotation.Nullable;
-
-/** Translator factory for {@link DomainHistoryVKey}. */
-public class DomainHistoryVKeyTranslatorFactory
-    extends AbstractSimpleTranslatorFactory<DomainHistoryVKey, Key> {
-
-  public DomainHistoryVKeyTranslatorFactory() {
-    super(DomainHistoryVKey.class);
-  }
-
-  @Override
-  SimpleTranslator<DomainHistoryVKey, Key> createTranslator() {
-    return new SimpleTranslator<DomainHistoryVKey, Key>() {
-
-      @Nullable
-      @Override
-      public DomainHistoryVKey loadValue(@Nullable Key datastoreValue) {
-        return datastoreValue == null
-            ? null
-            : DomainHistoryVKey.create(com.googlecode.objectify.Key.create(datastoreValue));
-      }
-
-      @Nullable
-      @Override
-      public Key saveValue(@Nullable DomainHistoryVKey pojoValue) {
-        return pojoValue == null ? null : pojoValue.getOfyKey().getRaw();
-      }
-    };
-  }
-}

core/src/main/java/google/registry/model/translators/EppHistoryVKeyTranslatorFactory.java

@@ -0,0 +1,110 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.translators;
+
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static java.util.function.Function.identity;
+
+import com.google.appengine.api.datastore.Key;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
+import google.registry.persistence.DomainHistoryVKey;
+import google.registry.persistence.EppHistoryVKey;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import javax.annotation.Nullable;
+
+/** Translator factory for {@link EppHistoryVKey}. */
+public class EppHistoryVKeyTranslatorFactory
+    extends AbstractSimpleTranslatorFactory<EppHistoryVKey, Key> {
+
+  public EppHistoryVKeyTranslatorFactory() {
+    super(EppHistoryVKey.class);
+  }
+
+  // This map is used when we need to convert the raw Datastore key to its VKey instance. We have
+  // one dedicated VKey class, e.g. DomainHistoryVKey, for each such kind of entity, and we need
+  // a way to map the raw Datastore key to its VKey class. So, we use the kind path as the key of
+  // the map, and the kind path is created by concatenating all the kind strings in a raw Datastore
+  // key, e.g. the map key for ContactPollMessageVKey is "ContactResource/HistoryEntry/PollMessage".
+  @VisibleForTesting
+  static final ImmutableMap<String, Class<? extends EppHistoryVKey>> kindPathToVKeyClass =
+      ImmutableSet.of(DomainHistoryVKey.class, BillingEventVKey.class, BillingRecurrenceVKey.class)
+          .stream()
+          .collect(toImmutableMap(EppHistoryVKeyTranslatorFactory::getKindPath, identity()));
+
+  /**
+   * Gets the kind path string for the given {@link Class}.
+   *
+   * <p>This method calls the getKindPath method on an instance of the given {@link Class} to get
+   * the kind path string.
+   */
+  private static String getKindPath(Class<? extends EppHistoryVKey> clazz) {
+    try {
+      Constructor<?> constructor = clazz.getDeclaredConstructor();
+      constructor.setAccessible(true);
+      Object instance = constructor.newInstance();
+      Method getKindPathMethod = EppHistoryVKey.class.getDeclaredMethod("getKindPath");
+      getKindPathMethod.setAccessible(true);
+      return (String) getKindPathMethod.invoke(instance);
+    } catch (Throwable t) {
+      throw new IllegalStateException(t);
+    }
+  }
+
+  @Override
+  SimpleTranslator<EppHistoryVKey, Key> createTranslator() {
+    return new SimpleTranslator<EppHistoryVKey, Key>() {
+
+      @Nullable
+      @Override
+      public EppHistoryVKey loadValue(@Nullable Key datastoreValue) {
+        if (datastoreValue == null) {
+          return null;
+        } else {
+          com.googlecode.objectify.Key<?> ofyKey =
+              com.googlecode.objectify.Key.create(datastoreValue);
+          String kindPath = EppHistoryVKey.createKindPath(ofyKey);
+          if (kindPathToVKeyClass.containsKey(kindPath)) {
+            Class<? extends EppHistoryVKey> vKeyClass = kindPathToVKeyClass.get(kindPath);
+            try {
+              Method createVKeyMethod =
+                  vKeyClass.getDeclaredMethod("create", com.googlecode.objectify.Key.class);
+              return (EppHistoryVKey) createVKeyMethod.invoke(null, ofyKey);
+            } catch (NoSuchMethodException e) {
+              throw new IllegalStateException(
+                  "Missing static method create(com.googlecode.objectify.Key) on " + vKeyClass);
+            } catch (IllegalAccessException | InvocationTargetException e) {
+              throw new IllegalStateException("Error invoking createVKey on " + vKeyClass, e);
+            }
+          } else {
+            throw new IllegalStateException(
+                "Missing EppHistoryVKey implementation for kind path: " + kindPath);
+          }
+        }
+      }
+
+      @Nullable
+      @Override
+      public Key saveValue(@Nullable EppHistoryVKey pojoValue) {
+        return pojoValue == null ? null : pojoValue.createOfyKey().getRaw();
+      }
+    };
+  }
+}

core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java

@@ -46,7 +46,10 @@ public class UpdateAutoTimestampTranslatorFactory
       /** Save a timestamp, setting it to the current time. */
       @Override
       public Date saveValue(UpdateAutoTimestamp pojoValue) {
-        return tm().getTransactionTime().toDate();
-      }};
+        return UpdateAutoTimestamp.autoUpdateEnabled()
+            ? tm().getTransactionTime().toDate()
+            : pojoValue.getTimestamp().toDate();
+      }
+    };
   }
 }

core/src/main/java/google/registry/persistence/BillingVKey.java

@@ -0,0 +1,134 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+
+import com.googlecode.objectify.Key;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingEvent.OneTime;
+import google.registry.model.billing.BillingEvent.Recurring;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.HistoryEntry;
+import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.MappedSuperclass;
+
+/** Base class for {@link BillingEvent}'s {@link VKey}. */
+@MappedSuperclass
+public abstract class BillingVKey<K> extends EppHistoryVKey<K, DomainBase> {
+  Long billingId;
+
+  // Hibernate requires a default constructor.
+  BillingVKey() {}
+
+  BillingVKey(String repoId, long historyRevisionId, long billingId) {
+    super(repoId, historyRevisionId);
+    this.billingId = billingId;
+  }
+
+  Key<HistoryEntry> createHistoryEntryKey() {
+    return Key.create(Key.create(DomainBase.class, repoId), HistoryEntry.class, historyRevisionId);
+  }
+
+  @Override
+  public Object createSqlKey() {
+    return billingId;
+  }
+
+  /** VKey class for {@link BillingEvent.OneTime} that belongs to a {@link DomainBase} entity. */
+  @Embeddable
+  @AttributeOverrides({
+    @AttributeOverride(name = "repoId", column = @Column(name = "billing_event_domain_repo_id")),
+    @AttributeOverride(
+        name = "historyRevisionId",
+        column = @Column(name = "billing_event_history_id")),
+    @AttributeOverride(name = "billingId", column = @Column(name = "billing_event_id"))
+  })
+  public static class BillingEventVKey extends BillingVKey<OneTime> {
+
+    // Hibernate requires this default constructor
+    private BillingEventVKey() {}
+
+    private BillingEventVKey(String repoId, long historyRevisionId, long billingEventId) {
+      super(repoId, historyRevisionId, billingEventId);
+    }
+
+    @Override
+    public Key<OneTime> createOfyKey() {
+      return Key.create(createHistoryEntryKey(), BillingEvent.OneTime.class, billingId);
+    }
+
+    /** Creates a {@link BillingEventVKey} instance from the given {@link Key} instance. */
+    public static BillingEventVKey create(@Nullable Key<BillingEvent.OneTime> ofyKey) {
+      if (ofyKey == null) {
+        return null;
+      }
+      long billingEventId = ofyKey.getId();
+      long historyRevisionId = ofyKey.getParent().getId();
+      String repoId = ofyKey.getParent().getParent().getName();
+      return new BillingEventVKey(repoId, historyRevisionId, billingEventId);
+    }
+
+    /** Creates a {@link BillingEventVKey} instance from the given {@link VKey} instance. */
+    public static BillingEventVKey create(@Nullable VKey<BillingEvent.OneTime> vKey) {
+      return vKey == null ? null : create(vKey.getOfyKey());
+    }
+  }
+
+  /** VKey class for {@link BillingEvent.Recurring} that belongs to a {@link DomainBase} entity. */
+  @Embeddable
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "repoId",
+        column = @Column(name = "billing_recurrence_domain_repo_id")),
+    @AttributeOverride(
+        name = "historyRevisionId",
+        column = @Column(name = "billing_recurrence_history_id")),
+    @AttributeOverride(name = "billingId", column = @Column(name = "billing_recurrence_id"))
+  })
+  public static class BillingRecurrenceVKey extends BillingVKey<Recurring> {
+
+    // Hibernate requires this default constructor
+    private BillingRecurrenceVKey() {}
+
+    private BillingRecurrenceVKey(String repoId, long historyRevisionId, long billingEventId) {
+      super(repoId, historyRevisionId, billingEventId);
+    }
+
+    @Override
+    public Key<Recurring> createOfyKey() {
+      return Key.create(createHistoryEntryKey(), BillingEvent.Recurring.class, billingId);
+    }
+
+    /** Creates a {@link BillingRecurrenceVKey} instance from the given {@link Key} instance. */
+    public static BillingRecurrenceVKey create(@Nullable Key<BillingEvent.Recurring> ofyKey) {
+      if (ofyKey == null) {
+        return null;
+      }
+      long billingEventId = ofyKey.getId();
+      long historyRevisionId = ofyKey.getParent().getId();
+      String repoId = ofyKey.getParent().getParent().getName();
+      return new BillingRecurrenceVKey(repoId, historyRevisionId, billingEventId);
+    }
+
+    /** Creates a {@link BillingRecurrenceVKey} instance from the given {@link VKey} instance. */
+    public static BillingRecurrenceVKey create(@Nullable VKey<BillingEvent.Recurring> vKey) {
+      return vKey == null ? null : create(vKey.getOfyKey());
+    }
+  }
+}

core/src/main/java/google/registry/persistence/DomainHistoryVKey.java

@@ -18,45 +18,44 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import javax.persistence.Embeddable;
-import javax.persistence.PostLoad;
 
 /** {@link VKey} for {@link HistoryEntry} which parent is {@link DomainBase}. */
 @Embeddable
-public class DomainHistoryVKey extends VKey<HistoryEntry> {
-
-  private String domainRepoId;
-
-  private Long domainHistoryId;
+public class DomainHistoryVKey extends EppHistoryVKey<HistoryEntry, DomainBase> {
 
   // Hibernate requires a default constructor
   private DomainHistoryVKey() {}
 
-  private DomainHistoryVKey(String domainRepoId, long domainHistoryId) {
-    initWith(domainRepoId, domainHistoryId);
+  private DomainHistoryVKey(String repoId, long historyRevisionId) {
+    super(repoId, historyRevisionId);
   }
 
-  @PostLoad
-  void postLoad() {
-    initWith(domainRepoId, domainHistoryId);
+  @Override
+  public Object createSqlKey() {
+    return new DomainHistoryId(repoId, historyRevisionId);
   }
 
-  private void initWith(String domainRepoId, long domainHistoryId) {
-    this.kind = HistoryEntry.class;
-    this.ofyKey =
-        Key.create(Key.create(DomainBase.class, domainRepoId), HistoryEntry.class, domainHistoryId);
-    this.sqlKey = new DomainHistoryId(domainRepoId, domainHistoryId);
-    this.domainRepoId = domainRepoId;
-    this.domainHistoryId = domainHistoryId;
+  @Override
+  public Key<HistoryEntry> createOfyKey() {
+    return Key.create(Key.create(DomainBase.class, repoId), HistoryEntry.class, historyRevisionId);
   }
 
   /** Creates {@link DomainHistoryVKey} from the given {@link Key} instance. */
-  public static DomainHistoryVKey create(Key<HistoryEntry> ofyKey) {
+  public static DomainHistoryVKey create(Key<? extends HistoryEntry> ofyKey) {
     checkArgumentNotNull(ofyKey, "ofyKey must be specified");
-    String domainRepoId = ofyKey.getParent().getName();
-    long domainHistoryId = ofyKey.getId();
-    return new DomainHistoryVKey(domainRepoId, domainHistoryId);
+    String repoId = ofyKey.getParent().getName();
+    long historyRevisionId = ofyKey.getId();
+    return new DomainHistoryVKey(repoId, historyRevisionId);
+  }
+
+  public VKey<? extends HistoryEntry> createDomainHistoryVKey() {
+    return VKey.create(
+        DomainHistory.class,
+        createSqlKey(),
+        Key.create(Key.create(DomainBase.class, repoId), DomainHistory.class, historyRevisionId));
   }
 }

core/src/main/java/google/registry/persistence/EppHistoryVKey.java

@@ -0,0 +1,107 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import com.google.common.base.Joiner;
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.util.TypeUtils.TypeInstantiator;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
+
+/**
+ * Base class for {@link VKey} which ofyKey has a {@link HistoryEntry} key as its parent and a key
+ * for EPP resource as its grandparent.
+ *
+ * <p>For such a {@link VKey}, we need to provide two type parameters to indicate the type of {@link
+ * VKey} itself and the type of EPP resource respectively.
+ *
+ * @param <K> type of the {@link VKey}
+ * @param <E> type of the EPP resource that the key belongs to
+ */
+@MappedSuperclass
+@Access(AccessType.FIELD)
+public abstract class EppHistoryVKey<K, E extends EppResource> extends ImmutableObject
+    implements Serializable {
+
+  private static final long serialVersionUID = -3906580677709539818L;
+
+  String repoId;
+
+  Long historyRevisionId;
+
+  // Hibernate requires a default constructor.
+  EppHistoryVKey() {}
+
+  EppHistoryVKey(String repoId, long historyRevisionId) {
+    this.repoId = repoId;
+    this.historyRevisionId = historyRevisionId;
+  }
+
+  /**
+   * Returns the kind path for the ofyKey in this instance.
+   *
+   * <p>This method is only used reflectively by {@link EppHistoryVKeyTranslatorFactory} to get the
+   * kind path for a given {@link EppHistoryVKey} instance so it is marked as a private method.
+   *
+   * @see #createKindPath(Key)
+   */
+  @SuppressWarnings("unused")
+  private String getKindPath() {
+    String eppKind = Key.getKind(new TypeInstantiator<E>(getClass()) {}.getExactType());
+    String keyKind = Key.getKind(new TypeInstantiator<K>(getClass()) {}.getExactType());
+    if (keyKind.equals(Key.getKind(HistoryEntry.class))) {
+      return createKindPath(eppKind, keyKind);
+    } else {
+      return createKindPath(eppKind, Key.getKind(HistoryEntry.class), keyKind);
+    }
+  }
+
+  /**
+   * Creates the kind path for the given ofyKey}.
+   *
+   * <p>The kind path is a string including all kind names(delimited by slash) of a hierarchical
+   * {@link Key}, e.g., the kind path for BillingEvent.OneTime is "DomainBase/HistoryEntry/OneTime".
+   */
+  @Nullable
+  public static String createKindPath(@Nullable Key<?> ofyKey) {
+    if (ofyKey == null) {
+      return null;
+    } else if (ofyKey.getParent() == null) {
+      return ofyKey.getKind();
+    } else {
+      return createKindPath(createKindPath(ofyKey.getParent()), ofyKey.getKind());
+    }
+  }
+
+  private static String createKindPath(String... kinds) {
+    return Joiner.on("/").join(kinds);
+  }
+
+  /** Creates a {@link VKey} from this instance. */
+  public VKey<K> createVKey() {
+    Class<K> vKeyType = new TypeInstantiator<K>(getClass()) {}.getExactType();
+    return VKey.create(vKeyType, createSqlKey(), createOfyKey());
+  }
+
+  public abstract Object createSqlKey();
+
+  public abstract Key<K> createOfyKey();
+}

core/src/main/java/google/registry/persistence/PersistenceComponent.java

@@ -20,6 +20,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import javax.inject.Singleton;
 import javax.persistence.EntityManagerFactory;
@@ -32,6 +33,7 @@ import javax.persistence.EntityManagerFactory;
       CredentialModule.class,
       KmsModule.class,
       PersistenceModule.class,
+      SecretManagerModule.class,
       UtilsModule.class
     })
 public interface PersistenceComponent {

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -26,6 +26,7 @@ import com.google.api.client.auth.oauth2.Credential;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
@@ -33,6 +34,11 @@ import google.registry.keyring.kms.KmsKeyring;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotId;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
 import google.registry.tools.AuthModule.CloudSqlClientCredential;
 import google.registry.util.Clock;
 import java.lang.annotation.Documented;
@@ -47,6 +53,8 @@ import org.hibernate.cfg.Environment;
 /** Dagger module class for the persistence layer. */
 @Module
 public class PersistenceModule {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   // This name must be the same as the one defined in persistence.xml.
   public static final String PERSISTENCE_UNIT_NAME = "nomulus";
   public static final String HIKARI_CONNECTION_TIMEOUT = "hibernate.hikari.connectionTimeout";
@@ -122,11 +130,17 @@ public class PersistenceModule {
   static JpaTransactionManager provideAppEngineJpaTm(
       @Config("cloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -136,6 +150,7 @@ public class PersistenceModule {
   static JpaTransactionManager provideNomulusToolJpaTm(
       @Config("toolsCloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       @CloudSqlClientCredential Credential credential,
       Clock clock) {
@@ -143,6 +158,11 @@ public class PersistenceModule {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getToolsCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.TOOL),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -150,6 +170,7 @@ public class PersistenceModule {
   @Singleton
   @SocketFactoryJpaTm
   static JpaTransactionManager provideSocketFactoryJpaTm(
+      SqlCredentialStore credentialStore,
       @Config("beamCloudSqlUsername") String username,
       @Config("beamCloudSqlPassword") String password,
       @Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
@@ -159,6 +180,12 @@ public class PersistenceModule {
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, password);
     overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
+    // TODO(b/175700623): consider assigning different logins to pipelines
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -203,6 +230,30 @@ public class PersistenceModule {
     return emf;
   }
 
+  /** Verifies that the credential from the Secret Manager matches the one currently in use.
+   *
+   * <p>This is a helper for the transition to the Secret Manager, and will be removed once data
+   * and permissions are properly set up for all projects.
+   **/
+  private static void validateCredentialStore(
+      SqlCredentialStore credentialStore, SqlUser sqlUser, String login, String password) {
+    try {
+      SqlCredential credential = credentialStore.getCredential(sqlUser);
+      if (!credential.login().equals(login)) {
+        logger.atWarning().log(
+            "Wrong login for %s. Expecting %s, found %s.",
+            sqlUser.geUserName(), login, credential.login());
+        return;
+      }
+      if (!credential.password().equals(password)) {
+        logger.atWarning().log("Wrong password for %s.", sqlUser.geUserName());
+      }
+      logger.atWarning().log("Credentials in the kerying and the secret manager match.");
+    } catch (Throwable e) {
+      logger.atWarning().log(e.getMessage());
+    }
+  }
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
   @Qualifier
   @Documented

core/src/main/java/google/registry/persistence/WithLongVKey.java

@@ -31,4 +31,12 @@ public @interface WithLongVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/WithStringVKey.java

@@ -31,4 +31,12 @@ public @interface WithStringVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/converter/UpdateAutoTimestampConverter.java

@@ -31,7 +31,14 @@ public class UpdateAutoTimestampConverter
 
   @Override
   public Timestamp convertToDatabaseColumn(UpdateAutoTimestamp entity) {
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(jpaTm().getTransactionTime()).toInstant());
+    return Timestamp.from(
+        DateTimeUtils.toZonedDateTime(
+                UpdateAutoTimestamp.autoUpdateEnabled()
+                        || entity == null
+                        || entity.getTimestamp() == null
+                    ? jpaTm().getTransactionTime()
+                    : entity.getTimestamp())
+            .toInstant());
   }
 
   @Override

core/src/main/java/google/registry/persistence/converter/VKeyConverter.java

@@ -14,6 +14,7 @@
 
 package google.registry.persistence.converter;
 
+import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
@@ -29,7 +30,28 @@ public abstract class VKeyConverter<T, C> implements AttributeConverter<VKey<? e
   @Override
   @Nullable
   public VKey<? extends T> convertToEntityAttribute(@Nullable C dbData) {
-    return dbData == null ? null : VKey.createSql(getAttributeClass(), dbData);
+    if (dbData == null) {
+      return null;
+    }
+    Class<? extends T> clazz = getAttributeClass();
+    Key ofyKey = null;
+    if (!hasCompositeOfyKey()) {
+      // If this isn't a composite key, we can create the Ofy key from the SQL key.
+      ofyKey =
+          dbData instanceof String
+              ? Key.create(clazz, (String) dbData)
+              : Key.create(clazz, (Long) dbData);
+      return VKey.create(clazz, dbData, ofyKey);
+    } else {
+      // We don't know how to create the Ofy key and probably don't have everything necessary to do
+      // it anyway, so just create an asymmetric key - the containing object will have to convert it
+      // into a symmetric key.
+      return VKey.createSql(clazz, dbData);
+    }
+  }
+
+  protected boolean hasCompositeOfyKey() {
+    return false;
   }
 
   /** Returns the class of the attribute. */

core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java

@@ -14,9 +14,9 @@
 
 package google.registry.persistence.transaction;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import java.util.Optional;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
@@ -45,7 +45,7 @@ public class TransactionEntity implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not stored in Datastore per se
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore per se
   }
 }

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClient.java

@@ -22,6 +22,9 @@ import java.util.Optional;
 /** A Cloud Secret Manager client for Nomulus, bound to a specific GCP project. */
 public interface SecretManagerClient {
 
+  /** Returns the project name with which this client is associated. */
+  String getProject();
+
   /**
    * Creates a new secret in the Cloud Secret Manager with no data.
    *
@@ -32,6 +35,9 @@ public interface SecretManagerClient {
    */
   void createSecret(String secretId);
 
+  /** Checks if a secret with the given {@code secretId} already exists. */
+  boolean secretExists(String secretId);
+
   /** Returns all secret IDs in the Cloud Secret Manager. */
   Iterable<String> listSecrets();
 
@@ -67,6 +73,24 @@ public interface SecretManagerClient {
    */
   String getSecretData(String secretId, Optional<String> version);
 
+  /**
+   * Enables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void enableSecretVersion(String secretId, String version);
+
+  /**
+   * Disables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void disableSecretVersion(String secretId, String version);
+
   /**
    * Destroys a secret version.
    *

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClientImpl.java

@@ -29,12 +29,11 @@ import com.google.cloud.secretmanager.v1.SecretName;
 import com.google.cloud.secretmanager.v1.SecretPayload;
 import com.google.cloud.secretmanager.v1.SecretVersion;
 import com.google.cloud.secretmanager.v1.SecretVersionName;
-import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
 import com.google.protobuf.ByteString;
 import google.registry.util.Retrier;
 import java.util.Optional;
 import java.util.concurrent.Callable;
-import javax.inject.Inject;
 
 /** Implements {@link SecretManagerClient} on Google Cloud Platform. */
 public class SecretManagerClientImpl implements SecretManagerClient {
@@ -42,13 +41,17 @@ public class SecretManagerClientImpl implements SecretManagerClient {
   private final SecretManagerServiceClient csmClient;
   private final Retrier retrier;
 
-  @Inject
   SecretManagerClientImpl(String project, SecretManagerServiceClient csmClient, Retrier retrier) {
     this.project = project;
     this.csmClient = csmClient;
     this.retrier = retrier;
   }
 
+  @Override
+  public String getProject() {
+    return project;
+  }
+
   @Override
   public void createSecret(String secretId) {
     checkNotNull(secretId, "secretId");
@@ -57,12 +60,25 @@ public class SecretManagerClientImpl implements SecretManagerClient {
         () -> csmClient.createSecret(ProjectName.of(project), secretId, secretSettings));
   }
 
+  @Override
+  public boolean secretExists(String secretId) {
+    checkNotNull(secretId, "secretId");
+    try {
+      callSecretManager(() -> csmClient.getSecret(SecretName.of(project, secretId)));
+      return true;
+    } catch (NoSuchSecretResourceException e) {
+      return false;
+    }
+  }
+
   @Override
   public Iterable<String> listSecrets() {
     ListSecretsPagedResponse response =
         callSecretManager(() -> csmClient.listSecrets(ProjectName.of(project)));
-    return Iterables.transform(
-        response.iterateAll(), secret -> SecretName.parse(secret.getName()).getSecret());
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(secret -> SecretName.parse(secret.getName()).getSecret())
+            .iterator();
   }
 
   @Override
@@ -70,8 +86,10 @@ public class SecretManagerClientImpl implements SecretManagerClient {
     checkNotNull(secretId, "secretId");
     ListSecretVersionsPagedResponse response =
         callSecretManager(() -> csmClient.listSecretVersions(SecretName.of(project, secretId)));
-    return Iterables.transform(
-        response.iterateAll(), SecretManagerClientImpl::toSecretVersionState);
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(SecretManagerClientImpl::toSecretVersionState)
+            .iterator();
   }
 
   private static SecretVersionState toSecretVersionState(SecretVersion secretVersion) {
@@ -108,6 +126,22 @@ public class SecretManagerClientImpl implements SecretManagerClient {
                 .toStringUtf8());
   }
 
+  @Override
+  public void enableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.enableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
+  @Override
+  public void disableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.disableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
   @Override
   public void destroySecretVersion(String secretId, String version) {
     checkNotNull(secretId, "secretId");

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerModule.java

@@ -14,41 +14,47 @@
 
 package google.registry.privileges.secretmanager;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
-import dagger.Component;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
-import google.registry.util.UtilsModule;
 import java.io.IOException;
 import javax.inject.Singleton;
 
 /** Provides bindings for {@link SecretManagerClient}. */
 @Module
-public class SecretManagerModule {
-
-  private final String project;
-
-  public SecretManagerModule(String project) {
-    this.project = checkNotNull(project, "project");
-  }
+public abstract class SecretManagerModule {
 
   @Provides
   @Singleton
-  SecretManagerClient provideSecretManagerClient(Retrier retrier) {
+  static SecretManagerServiceSettings provideSecretManagerSetting(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle) {
     try {
-      return new SecretManagerClientImpl(project, SecretManagerServiceClient.create(), retrier);
+      return SecretManagerServiceSettings.newBuilder()
+          .setCredentialsProvider(() -> credentialsBundle.getGoogleCredentials())
+          .build();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  @Provides
   @Singleton
-  @Component(modules = {ConfigModule.class, SecretManagerModule.class, UtilsModule.class})
-  public interface SecretManagerComponent {
-    SecretManagerClient secretManagerClient();
+  static SecretManagerClient provideSecretManagerClient(
+      SecretManagerServiceSettings serviceSettings,
+      @Config("projectId") String project,
+      Retrier retrier) {
+    try {
+      SecretManagerServiceClient stub = SecretManagerServiceClient.create(serviceSettings);
+      Runtime.getRuntime().addShutdownHook(new Thread(stub::close));
+      return new SecretManagerClientImpl(project, stub, retrier);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
   }
 }

core/src/main/java/google/registry/privileges/secretmanager/SqlCredential.java

@@ -0,0 +1,55 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static avro.shaded.com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.value.AutoValue;
+import java.util.List;
+
+/**
+ * Contains the login name and password of a Cloud SQL user.
+ *
+ * <p>User must take care not to include the {@link #SEPARATOR} in property values.
+ */
+@AutoValue
+public abstract class SqlCredential {
+
+  public static final Character SEPARATOR = ' ';
+
+  public abstract String login();
+
+  public abstract String password();
+
+  @Override
+  public final String toString() {
+    // Use Object.toString(), which does not show object data.
+    return super.toString();
+  }
+
+  public final String toFormattedString() {
+    return String.format("%s%c%s", login(), SEPARATOR, password());
+  }
+
+  public static SqlCredential fromFormattedString(String sqlCredential) {
+    List<String> items = com.google.common.base.Splitter.on(SEPARATOR).splitToList(sqlCredential);
+    checkState(items.size() == 2, "Invalid SqlCredential string.");
+    return of(items.get(0), items.get(1));
+  }
+
+  public static SqlCredential of(String login, String password) {
+    return new AutoValue_SqlCredential(login, password);
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlCredentialStore.java

@@ -0,0 +1,118 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.cloud.secretmanager.v1.SecretVersionName;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Storage of SQL users' login credentials, backed by Cloud Secret Manager.
+ *
+ * <p>A user's credential is stored with one level of indirection using two secret IDs: Each version
+ * of the <em>credential data</em> is stored as follows: its secret ID is determined by {@link
+ * #getCredentialDataSecretId(SqlUser, String dbInstance)}, and the value of each version is a
+ * {@link SqlCredential}, serialized using {@link SqlCredential#toFormattedString}. The 'live'
+ * version of the credential is saved under the 'live pointer' secret explained below.
+ *
+ * <p>The pointer to the 'live' version of the credential data is stored as follows: its secret ID
+ * is determined by {@link #getLiveLabelSecretId(SqlUser, String dbInstance)}; and the value of each
+ * version is a {@link SecretVersionName} in String form, pointing to a version of the credential
+ * data. Only the 'latest' version of this secret should be used. It is guaranteed to be valid.
+ *
+ * <p>The indirection in credential storage makes it easy to handle failures in the credential
+ * change process.
+ */
+public class SqlCredentialStore {
+  private final SecretManagerClient csmClient;
+  private final String dbInstance;
+
+  @Inject
+  SqlCredentialStore(
+      SecretManagerClient csmClient, @Config("cloudSqlDbInstanceName") String dbInstance) {
+    this.csmClient = csmClient;
+    this.dbInstance = dbInstance;
+  }
+
+  public SqlCredential getCredential(SqlUser user) {
+    SecretVersionName credentialName = getLiveCredentialSecretVersion(user);
+    return SqlCredential.fromFormattedString(
+        csmClient.getSecretData(
+            credentialName.getSecret(), Optional.of(credentialName.getSecretVersion())));
+  }
+
+  public void createOrUpdateCredential(SqlUser user, String password) {
+    SecretVersionName dataName = saveCredentialData(user, password);
+    saveLiveLabel(user, dataName);
+  }
+
+  public void deleteCredential(SqlUser user) {
+    try {
+      csmClient.deleteSecret(getCredentialDataSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok
+    }
+    try {
+      csmClient.deleteSecret(getLiveLabelSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok.
+    }
+  }
+
+  private void createSecretIfAbsent(String secretId) {
+    try {
+      csmClient.createSecret(secretId);
+    } catch (SecretAlreadyExistsException ignore) {
+      // Not a problem.
+    }
+  }
+
+  private SecretVersionName saveCredentialData(SqlUser user, String password) {
+    String credentialDataSecretId = getCredentialDataSecretId(user, dbInstance);
+    createSecretIfAbsent(credentialDataSecretId);
+    String credentialVersion =
+        csmClient.addSecretVersion(
+            credentialDataSecretId,
+            SqlCredential.of(createDatabaseLoginName(user), password).toFormattedString());
+    return SecretVersionName.of(csmClient.getProject(), credentialDataSecretId, credentialVersion);
+  }
+
+  private void saveLiveLabel(SqlUser user, SecretVersionName dataVersionName) {
+    String liveLabelSecretId = getLiveLabelSecretId(user, dbInstance);
+    createSecretIfAbsent(liveLabelSecretId);
+    csmClient.addSecretVersion(liveLabelSecretId, dataVersionName.toString());
+  }
+
+  private SecretVersionName getLiveCredentialSecretVersion(SqlUser user) {
+    return SecretVersionName.parse(
+        csmClient.getSecretData(getLiveLabelSecretId(user, dbInstance), Optional.empty()));
+  }
+
+  private static String getLiveLabelSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-live-label-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  private static String getCredentialDataSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-data-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  // WIP: when b/170230882 is complete, login will be versioned.
+  private static String createDatabaseLoginName(SqlUser user) {
+    return user.geUserName();
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlUser.java

@@ -0,0 +1,67 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.common.base.Ascii;
+
+/**
+ * SQL user information for privilege management purposes.
+ *
+ * <p>A {@link RobotUser} represents a software system accessing the database using its own
+ * credential. Robots are well known and enumerated in {@link RobotId}.
+ */
+public abstract class SqlUser {
+
+  private final UserType type;
+  private final String userName;
+
+  protected SqlUser(UserType type, String userName) {
+    this.type = type;
+    this.userName = userName;
+  }
+
+  public UserType getType() {
+    return type;
+  }
+
+  public String geUserName() {
+    return userName;
+  }
+
+  /** Cloud SQL user types. Please see class javadoc of {@link SqlUser} for more information. */
+  enum UserType {
+    // Work in progress. Human user will be added.
+    ROBOT
+  }
+
+  /** Enumerates the {@link RobotUser RobotUsers} in the system. */
+  public enum RobotId {
+    NOMULUS,
+    /**
+     * Credential for RegistryTool. This is temporary, and will be removed when tool users are
+     * assigned their personal credentials.
+     */
+    TOOL;
+  }
+
+  /** Information of a RobotUser for privilege management purposes. */
+  // Work in progress. Eventually will be provided based on configuration.
+  public static class RobotUser extends SqlUser {
+
+    public RobotUser(RobotId robot) {
+      super(UserType.ROBOT, Ascii.toLowerCase(robot.name()));
+    }
+  }
+}

core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java

@@ -14,9 +14,11 @@
 
 package google.registry.reporting.spec11;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
@@ -59,6 +61,25 @@ public class Spec11RegistrarThreatMatchesParser {
     return getFromFile(getGcsFilename(date));
   }
 
+  /** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */
+  public ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
+      throws IOException {
+    if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) {
+      return ImmutableSet.of();
+    }
+    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
+    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename);
+        InputStreamReader isr = new InputStreamReader(in, UTF_8)) {
+      // Skip the header at line 0
+      return Splitter.on("\n")
+          .omitEmptyStrings()
+          .splitToStream(CharStreams.toString(isr))
+          .skip(1)
+          .map(this::parseRegistrarThreatMatch)
+          .collect(toImmutableSet());
+    }
+  }
+
   public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
     LocalDate yesterday = date.minusDays(1);
     GcsFilename gcsFilename = getGcsFilename(yesterday);
@@ -82,20 +103,6 @@ public class Spec11RegistrarThreatMatchesParser {
     return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
   }
 
-  private ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
-      throws IOException, JSONException {
-    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
-    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
-      ImmutableList<String> reportLines =
-          ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
-      // Iterate from 1 to size() to skip the header at line 0.
-      for (int i = 1; i < reportLines.size(); i++) {
-        builder.add(parseRegistrarThreatMatch(reportLines.get(i)));
-      }
-      return builder.build();
-    }
-  }
-
   private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
     JSONObject reportJSON = new JSONObject(line);
     String clientId = reportJSON.getString(Spec11Pipeline.REGISTRAR_CLIENT_ID_FIELD);

core/src/main/java/google/registry/schema/cursor/Cursor.java

@@ -16,7 +16,6 @@ package google.registry.schema.cursor;
 
 import static com.google.appengine.api.search.checkers.Preconditions.checkNotNull;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.common.Cursor.CursorType;
@@ -26,6 +25,7 @@ import google.registry.schema.replay.SqlEntity;
 import google.registry.util.DateTimeUtils;
 import java.io.Serializable;
 import java.time.ZonedDateTime;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.EnumType;
@@ -104,8 +104,8 @@ public class Cursor implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // Cursors are not converted since they are ephemeral
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Cursors are not converted since they are ephemeral
   }
 
   static class CursorId extends ImmutableObject implements Serializable {

core/src/main/java/google/registry/schema/domain/RegistryLock.java

@@ -19,7 +19,6 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.toZonedDateTime;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
@@ -234,8 +233,8 @@ public final class RegistryLock extends ImmutableObject implements Buildable, Sq
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not stored in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore
   }
 
   /** Builder for {@link google.registry.schema.domain.RegistryLock}. */

core/src/main/java/google/registry/schema/replay/DatastoreAndSqlEntity.java

@@ -14,18 +14,18 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /** An entity that has the same Java object representation in SQL and Datastore. */
 public interface DatastoreAndSqlEntity extends DatastoreEntity, SqlEntity {
 
   @Override
-  default ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(this);
+  default Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(this);
   }
 
   @Override
-  default ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(this);
+  default Optional<SqlEntity> toSqlEntity() {
+    return Optional.of(this);
   }
 }

core/src/main/java/google/registry/schema/replay/DatastoreEntity.java

@@ -14,7 +14,7 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /**
  * An object that can be stored in Datastore and serialized using Objectify's {@link
@@ -26,5 +26,5 @@ import com.google.common.collect.ImmutableList;
  */
 public interface DatastoreEntity {
 
-  ImmutableList<SqlEntity> toSqlEntities();
+  Optional<SqlEntity> toSqlEntity();
 }

core/src/main/java/google/registry/schema/replay/DatastoreOnlyEntity.java

@@ -14,11 +14,12 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
+/** An entity that is only stored in Datastore, that should not be replayed to SQL. */
 public interface DatastoreOnlyEntity extends DatastoreEntity {
   @Override
-  default ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of();
+  default Optional<SqlEntity> toSqlEntity() {
+    return Optional.empty();
   }
 }

core/src/main/java/google/registry/schema/replay/NonReplicatedEntity.java

@@ -14,7 +14,7 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /**
  * Represents an entity that should not participate in asynchronous replication.
@@ -24,12 +24,12 @@ import com.google.common.collect.ImmutableList;
 public interface NonReplicatedEntity extends DatastoreEntity, SqlEntity {
 
   @Override
-  default ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of();
+  default Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty();
   }
 
   @Override
-  default ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of();
+  default Optional<SqlEntity> toSqlEntity() {
+    return Optional.empty();
   }
 }

core/src/main/java/google/registry/schema/replay/SqlEntity.java

@@ -14,7 +14,7 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /**
  * An object that can be stored in Cloud SQL using {@link
@@ -25,5 +25,5 @@ import com.google.common.collect.ImmutableList;
  */
 public interface SqlEntity {
 
-  ImmutableList<DatastoreEntity> toDatastoreEntities();
+  Optional<DatastoreEntity> toDatastoreEntity();
 }

core/src/main/java/google/registry/schema/replay/SqlReplayCheckpoint.java

@@ -18,7 +18,7 @@ import static google.registry.model.common.CrossTldSingleton.SINGLETON_ID;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.joda.time.DateTime;
@@ -32,8 +32,8 @@ public class SqlReplayCheckpoint implements SqlEntity {
   private DateTime lastReplayTime;
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not necessary to persist in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not necessary to persist in Datastore
   }
 
   public static DateTime get() {

core/src/main/java/google/registry/schema/server/Lock.java

@@ -16,7 +16,6 @@ package google.registry.schema.server;
 
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
@@ -24,6 +23,7 @@ import google.registry.schema.server.Lock.LockId;
 import google.registry.util.DateTimeUtils;
 import java.io.Serializable;
 import java.time.ZonedDateTime;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -120,8 +120,8 @@ public class Lock implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // Locks are not converted since they are ephemeral
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Locks are not converted since they are ephemeral
   }
 
   static class LockId extends ImmutableObject implements Serializable {

core/src/main/java/google/registry/schema/tld/PremiumEntry.java

@@ -14,13 +14,13 @@
 
 package google.registry.schema.tld;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
 import google.registry.model.registry.label.PremiumList;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -47,7 +47,7 @@ public class PremiumEntry extends ImmutableObject implements Serializable, SqlEn
   private PremiumEntry() {}
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // PremiumList is dually-written
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // PremiumList is dually-written
   }
 }

core/src/main/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java

@@ -138,15 +138,6 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
       validateWith = PathParameter.InputFile.class)
   Path clientCertificateFilename;
 
-  @Nullable
-  @Parameter(
-    names = "--cert_hash",
-    description =
-        "Hash of client certificate (SHA256 base64 no padding). Do not use this unless "
-            + "you want to store ONLY the hash and not the full certificate"
-  )
-  String clientCertificateHash;
-
   @Nullable
   @Parameter(
       names = "--failover_cert_file",
@@ -375,14 +366,6 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
         }
         builder.setFailoverClientCertificate(asciiCert, now);
       }
-      if (!isNullOrEmpty(clientCertificateHash)) {
-        checkArgument(clientCertificateFilename == null,
-            "Can't specify both --cert_hash and --cert_file");
-        if ("null".equals(clientCertificateHash)) {
-          clientCertificateHash = null;
-        }
-        builder.setClientCertificateHash(clientCertificateHash);
-      }
       if (ianaId != null) {
         builder.setIanaIdentifier(ianaId.orElse(null));
       }

core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java

@@ -45,10 +45,10 @@ import google.registry.model.domain.DomainBase;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.OneTime entities with duplicate IDs.")
-public class DedupeOneTimeBillingEventIdsCommand extends DedupeEntityIdsCommand<OneTime> {
+public class DedupeOneTimeBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<OneTime> {
 
   @Override
-  void dedupe(OneTime entity) {
+  void process(OneTime entity) {
     Key<BillingEvent> key = Key.create(entity);
     Key<DomainBase> domainKey = getGrandParentAsDomain(key);
     DomainBase domain = ofy().load().key(domainKey).now();

core/src/main/java/google/registry/tools/DedupeRecurringBillingEventIdsCommand.java

@@ -54,11 +54,10 @@ import java.util.Set;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.Recurring entities with duplicate IDs.")
-public class DedupeRecurringBillingEventIdsCommand
-    extends DedupeEntityIdsCommand<BillingEvent.Recurring> {
+public class DedupeRecurringBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<Recurring> {
 
   @Override
-  void dedupe(Recurring recurring) {
+  void process(Recurring recurring) {
     // Loads the associated DomainBase and BillingEvent.OneTime entities that
     // may have reference to this BillingEvent.Recurring entity.
     Key<DomainBase> domainKey = getGrandParentAsDomain(Key.create(recurring));

core/src/main/java/google/registry/tools/GetSqlCredentialCommand.java

@@ -0,0 +1,73 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.io.FileOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to get a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(separators = " =", commandDescription = "Get the Cloud SQL Credential for a given user")
+public class GetSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"-o", "--output"},
+      description = "Name of output file for key data.",
+      validateWith = PathParameter.OutputFile.class)
+  private Path outputPath = null;
+
+  @Inject
+  GetSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+
+    SqlCredential credential;
+    try {
+      credential = store.getCredential(sqlUser);
+    } catch (SecretManagerException e) {
+      System.out.println(e.getMessage());
+      return;
+    }
+
+    if (outputPath == null) {
+      System.out.printf("[%s]\n", credential.toFormattedString());
+      return;
+    }
+    try (FileOutputStream out = new FileOutputStream(outputPath.toFile())) {
+      out.write(credential.toFormattedString().getBytes(StandardCharsets.UTF_8));
+    }
+  }
+}

core/src/main/java/google/registry/tools/ReadEntityFromKeyPathCommand.java

@@ -33,8 +33,13 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.List;
 
-/** Base Command to dedupe entities with duplicate IDs. */
-abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
+/**
+ * Base Command to read entities from Datastore by their key paths retrieved from BigQuery.
+ *
+ * <p>The key path is the value of column __key__.path of the entity's BigQuery table. Its value is
+ * converted from the entity's key.
+ */
+abstract class ReadEntityFromKeyPathCommand<T> extends MutatingCommand {
 
   @Parameter(
       names = "--key_paths_file",
@@ -48,7 +53,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
 
   private StringBuilder changeMessage = new StringBuilder();
 
-  abstract void dedupe(T entity);
+  abstract void process(T entity);
 
   @Override
   protected void init() throws Exception {
@@ -69,7 +74,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
       }
       Class<T> clazz = new TypeInstantiator<T>(getClass()) {}.getExactType();
       if (clazz.isInstance(entity)) {
-        dedupe((T) entity);
+        process((T) entity);
       } else {
         throw new IllegalArgumentException("Unsupported entity key: " + untypedKey);
       }

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
+import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 
@@ -32,6 +33,7 @@ public final class RegistryTool {
       new ImmutableMap.Builder<String, Class<? extends Command>>()
           .put("ack_poll_messages", AckPollMessagesCommand.class)
           .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
+          .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
@@ -78,6 +80,7 @@ public final class RegistryTool {
           .put("get_routing_map", GetRoutingMapCommand.class)
           .put("get_schema", GetSchemaCommand.class)
           .put("get_schema_tree", GetSchemaTreeCommand.class)
+          .put("get_sql_credential", GetSqlCredentialCommand.class)
           .put("get_tld", GetTldCommand.class)
           .put("ghostryde", GhostrydeCommand.class)
           .put("hash_certificate", HashCertificateCommand.class)
@@ -99,12 +102,15 @@ public final class RegistryTool {
           .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)
           .put("remove_ip_address", RemoveIpAddressCommand.class)
+          .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class)
           .put("renew_domain", RenewDomainCommand.class)
           .put("resave_entities", ResaveEntitiesCommand.class)
           .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
           .put("resave_epp_resource", ResaveEppResourceCommand.class)
+          .put("save_sql_credential", SaveSqlCredentialCommand.class)
           .put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class)
           .put("set_num_instances", SetNumInstancesCommand.class)
+          .put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class)
           .put("setup_ote", SetupOteCommand.class)
           .put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class)
           .put("unlock_domain", UnlockDomainCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -34,6 +34,7 @@ import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.NomulusToolJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
@@ -74,6 +75,7 @@ import javax.inject.Singleton;
       PersistenceModule.class,
       RdeModule.class,
       RequestFactoryModule.class,
+      SecretManagerModule.class,
       URLFetchServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
@@ -118,6 +120,8 @@ interface RegistryToolComponent {
 
   void inject(GetOperationStatusCommand command);
 
+  void inject(GetSqlCredentialCommand command);
+
   void inject(GhostrydeCommand command);
 
   void inject(ImportDatastoreCommand command);
@@ -138,10 +142,14 @@ interface RegistryToolComponent {
 
   void inject(RenewDomainCommand command);
 
+  void inject(SaveSqlCredentialCommand command);
+
   void inject(SendEscrowReportToIcannCommand command);
 
   void inject(SetNumInstancesCommand command);
 
+  void inject(SetSqlReplayCheckpointCommand command);
+
   void inject(SetupOteCommand command);
 
   void inject(UnlockDomainCommand command);

core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameters;
+import com.googlecode.objectify.Key;
+import google.registry.model.domain.DomainBase;
+import google.registry.persistence.VKey;
+import javax.annotation.Nullable;
+import org.joda.time.DateTime;
+
+/** Command to remove the Registry 1.0 key in {@link DomainBase} entity. */
+@Parameters(separators = " =", commandDescription = "Remove .")
+public class RemoveRegistryOneKeyCommand extends ReadEntityFromKeyPathCommand<DomainBase> {
+
+  @Override
+  void process(DomainBase entity) {
+    // Assert that the DomainBase entity must be deleted before 2017-08-01(most of the problematic
+    // entities were deleted before 2017, though there are still a few entities deleted in 2017-07).
+    // This is because we finished the Registry 2.0 migration in 2017 and should not generate any
+    // Registry 1.0 key after it.
+    if (!entity.getDeletionTime().isBefore(DateTime.parse("2017-08-01T00:00:00Z"))) {
+      throw new IllegalStateException(
+          String.format(
+              "Entity's deletion time %s is not before 2017-08-01T00:00:00Z",
+              entity.getDeletionTime()));
+    }
+    boolean hasChange = false;
+    DomainBase.Builder domainBuilder = entity.asBuilder();
+    // We only found the registry 1.0 key existed in fields autorenewBillingEvent,
+    // autorenewPollMessage and deletePollMessage so we just need to check these fields for each
+    // entity.
+    if (isRegistryOneKey(entity.getAutorenewBillingEvent())) {
+      domainBuilder.setAutorenewBillingEvent(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getAutorenewPollMessage())) {
+      domainBuilder.setAutorenewPollMessage(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getDeletePollMessage())) {
+      domainBuilder.setDeletePollMessage(null);
+      hasChange = true;
+    }
+    if (hasChange) {
+      stageEntityChange(entity, domainBuilder.build());
+    }
+  }
+
+  private static boolean isRegistryOneKey(@Nullable VKey<?> vKey) {
+    if (vKey == null || vKey.getOfyKey() == null || vKey.getOfyKey().getParent() == null) {
+      return false;
+    }
+    Key<?> parentKey = vKey.getOfyKey().getParent();
+    return parentKey.getKind().equals("EntityGroupRoot") && parentKey.getName().equals("per-tld");
+  }
+}

core/src/main/java/google/registry/tools/SaveSqlCredentialCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to create or update a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(
+    separators = " =",
+    commandDescription = "Create or update the Cloud SQL Credential for a given user")
+public class SaveSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"--input"},
+      description =
+          "Name of input file for the password. If absent, command will prompt for "
+              + "password in console.",
+      validateWith = PathParameter.InputFile.class)
+  private Path inputPath = null;
+
+  @Inject
+  SaveSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    String password = getPassword();
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+    store.createOrUpdateCredential(sqlUser, password);
+    System.out.printf("\nDone:[%s]\n", password);
+  }
+
+  private String getPassword() throws Exception {
+    if (inputPath != null) {
+      return Files.readAllLines(inputPath, StandardCharsets.UTF_8).get(0);
+    }
+    return System.console().readLine("Please enter the password: ").trim();
+  }
+}

core/src/main/java/google/registry/tools/SetSqlReplayCheckpointCommand.java

@@ -0,0 +1,48 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.Iterables;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import java.util.List;
+import org.joda.time.DateTime;
+
+/** Command to set {@link SqlReplayCheckpoint} to a particular, post-initial-population time. */
+@Parameters(separators = " =", commandDescription = "Set SqlReplayCheckpoint to a particular time")
+public class SetSqlReplayCheckpointCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  @Parameter(description = "Time to which SqlReplayCheckpoint will be set", required = true)
+  List<DateTime> mainParameters;
+
+  @Override
+  protected String prompt() {
+    checkArgument(mainParameters.size() == 1, "Must provide exactly one DateTime to set");
+    return String.format(
+        "Set SqlReplayCheckpoint to %s?", Iterables.getOnlyElement(mainParameters));
+  }
+
+  @Override
+  protected String execute() {
+    DateTime dateTime = Iterables.getOnlyElement(mainParameters);
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime));
+    return String.format("Set SqlReplayCheckpoint time to %s", dateTime);
+  }
+}

core/src/main/java/google/registry/tools/SetupOteCommand.java

@@ -65,13 +65,6 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo
       validateWith = PathParameter.InputFile.class)
   private Path certFile;
 
-  @Parameter(
-      names = {"-h", "--certhash"},
-      description =
-          "Hash of client certificate (SHA256 base64 no padding). Do not use this unless "
-              + "you want to store ONLY the hash and not the full certificate.")
-  private String certHash;
-
   @Parameter(
       names = {"--overwrite"},
       description = "Whether to replace existing entities if we encounter any, instead of failing.")
@@ -89,9 +82,7 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo
   /** Run any pre-execute command checks */
   @Override
   protected void init() throws Exception {
-    checkArgument(
-        certFile == null ^ certHash == null,
-        "Must specify exactly one of client certificate file or client certificate hash.");
+    checkArgument(certFile != null, "Must specify exactly one client certificate file.");
 
     password = passwordGenerator.createString(PASSWORD_LENGTH);
     oteAccountBuilder =
@@ -101,16 +92,10 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo
             .setIpAllowList(ipAllowList)
             .setReplaceExisting(overwrite);
 
-    if (certFile != null) {
       String asciiCert = MoreFiles.asCharSource(certFile, US_ASCII).read();
       // Don't wait for create_registrar to fail if it's a bad certificate file.
       loadCertificate(asciiCert);
       oteAccountBuilder.setCertificate(asciiCert, clock.nowUtc());
-    }
-
-    if (certHash != null) {
-      oteAccountBuilder.setCertificateHash(certHash);
-    }
   }
 
   @Override

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -0,0 +1,216 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.beam.spec11.ThreatMatch;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.model.reporting.Spec11ThreatMatchDao;
+import google.registry.reporting.spec11.RegistrarThreatMatches;
+import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.Clock;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+import javax.inject.Inject;
+import org.joda.time.LocalDate;
+
+/**
+ * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
+ *
+ * <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
+ * rough estimate of when we started using this format) and convert those RegistrarThreatMatches
+ * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
+ *
+ * <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
+ * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
+ * or if the domain objects were not active at the time of the scan.
+ */
+@Parameters(
+    commandDescription =
+        "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
+            + "Cloud SQL database.")
+public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
+
+  @Parameter(
+      names = {"-o", "--overwrite_existing_dates"},
+      description =
+          "Whether the command will overwrite data that already exists for dates that exist in the "
+              + "GCS bucket. Defaults to false.")
+  private boolean overrideExistingDates;
+
+  @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
+  // Inject the clock for testing purposes
+  @Inject Clock clock;
+
+  @Override
+  protected String prompt() {
+    return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
+  }
+
+  @Override
+  protected String execute() {
+    ImmutableList<LocalDate> dates = getDatesToBackfill();
+    ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
+        new ImmutableListMultimap.Builder<>();
+    for (LocalDate date : dates) {
+      try {
+        // It's OK if the file doesn't exist for a particular date; the result will be empty.
+        threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
+      } catch (IOException e) {
+        throw new RuntimeException(
+            String.format("Error parsing through file with date %s.", date), e);
+      }
+    }
+    ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
+        threatMatchesBuilder.build();
+    // Look up all possible DomainBases for these domain names, any of which can be in the past
+    ImmutableListMultimap<String, DomainBase> domainsByDomainName =
+        getDomainsByDomainName(threatMatches);
+
+    // For each date, convert all threat matches with the proper domain repo ID
+    int totalNumThreats = 0;
+    for (LocalDate date : threatMatches.keySet()) {
+      ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
+      for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
+        rtm.threatMatches().stream()
+            .map(
+                threatMatch ->
+                    threatMatchToCloudSqlObject(
+                        threatMatch, date, rtm.clientId(), domainsByDomainName))
+            .forEach(spec11ThreatsBuilder::add);
+      }
+      ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
+      jpaTm()
+          .transact(
+              () -> {
+                Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
+                jpaTm().putAll(spec11Threats);
+              });
+      totalNumThreats += spec11Threats.size();
+    }
+    return String.format(
+        "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
+  }
+
+  /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
+  private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
+      ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
+    return threatMatchesByDate.values().stream()
+        .map(RegistrarThreatMatches::threatMatches)
+        .flatMap(ImmutableList::stream)
+        .map(ThreatMatch::fullyQualifiedDomainName)
+        .distinct()
+        .collect(
+            flatteningToImmutableListMultimap(
+                Function.identity(),
+                (domainName) -> {
+                  List<DomainBase> domains =
+                      ofy()
+                          .load()
+                          .type(DomainBase.class)
+                          .filter("fullyQualifiedDomainName", domainName)
+                          .list();
+                  domains.sort(Comparator.comparing(DomainBase::getCreationTime).reversed());
+                  checkState(
+                      !domains.isEmpty(),
+                      "Domain name %s had no associated DomainBase objects.",
+                      domainName);
+                  return domains.stream();
+                }));
+  }
+
+  /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
+  private Spec11ThreatMatch threatMatchToCloudSqlObject(
+      ThreatMatch threatMatch,
+      LocalDate date,
+      String registrarId,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    DomainBase domain =
+        findDomainAsOfDateOrThrow(
+            threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
+    return new Spec11ThreatMatch.Builder()
+        .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
+        .setCheckDate(date)
+        .setRegistrarId(registrarId)
+        .setDomainName(threatMatch.fullyQualifiedDomainName())
+        .setDomainRepoId(domain.getRepoId())
+        .build();
+  }
+
+  /** Returns the DomainBase object as of the particular date, which is likely in the past. */
+  private DomainBase findDomainAsOfDateOrThrow(
+      String domainName,
+      LocalDate date,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
+    for (DomainBase domain : domains) {
+      // We only know the date (not datetime) of the threat scan, so we approximate
+      LocalDate creationDate = domain.getCreationTime().toLocalDate();
+      LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
+      if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
+        return domain;
+      }
+    }
+    throw new IllegalStateException(
+        String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
+  }
+
+  /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
+  private ImmutableList<LocalDate> getDatesToBackfill() {
+    ImmutableSet<LocalDate> datesToSkip =
+        overrideExistingDates ? ImmutableSet.of() : getExistingDates();
+    ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
+    LocalDate endDate = clock.nowUtc().toLocalDate();
+    for (LocalDate currentDate = START_DATE;
+        !currentDate.isAfter(endDate);
+        currentDate = currentDate.plusDays(1)) {
+      if (!datesToSkip.contains(currentDate)) {
+        result.add(currentDate);
+      }
+    }
+    return result.build();
+  }
+
+  private ImmutableSet<LocalDate> getExistingDates() {
+    return jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .getEntityManager()
+                    .createQuery(
+                        "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
+                    .getResultStream()
+                    .collect(toImmutableSet()));
+  }
+}

core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java

@@ -0,0 +1,458 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
+import static google.registry.backup.RestoreCommitLogsActionTest.GCS_BUCKET;
+import static google.registry.backup.RestoreCommitLogsActionTest.createCheckpoint;
+import static google.registry.backup.RestoreCommitLogsActionTest.saveDiffFile;
+import static google.registry.backup.RestoreCommitLogsActionTest.saveDiffFileNotToRestore;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistActiveContact;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.truth.Truth8;
+import com.googlecode.objectify.Key;
+import google.registry.config.RegistryConfig;
+import google.registry.model.common.Cursor;
+import google.registry.model.common.Cursor.CursorType;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.ofy.CommitLogBucket;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registry.label.ReservedList;
+import google.registry.model.server.Lock;
+import google.registry.model.tmch.ClaimsListShard;
+import google.registry.model.translators.VKeyTranslatorFactory;
+import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.TestObject;
+import google.registry.util.RequestStatusChecker;
+import java.io.IOException;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/** Tests for {@link ReplayCommitLogsToSqlAction}. */
+@ExtendWith(MockitoExtension.class)
+public class ReplayCommitLogsToSqlActionTest {
+
+  private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ"));
+
+  @RegisterExtension
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withDatastoreAndCloudSql()
+          .withClock(fakeClock)
+          .withOfyTestEntities(TestObject.class)
+          .withJpaUnitTestEntities(
+              RegistrarContact.class,
+              TestObject.class,
+              SqlReplayCheckpoint.class,
+              ContactResource.class,
+              DomainBase.class,
+              GracePeriod.class,
+              DelegationSignerData.class)
+          .build();
+
+  /** Local GCS service. */
+  private final GcsService gcsService = GcsServiceFactory.createGcsService();
+
+  private final ReplayCommitLogsToSqlAction action = new ReplayCommitLogsToSqlAction();
+  private final FakeResponse response = new FakeResponse();
+  @Mock private RequestStatusChecker requestStatusChecker;
+
+  @BeforeAll
+  static void beforeAll() {
+    VKeyTranslatorFactory.addTestEntityClass(TestObject.class);
+  }
+
+  @BeforeEach
+  void beforeEach() {
+    action.gcsService = gcsService;
+    action.response = response;
+    action.requestStatusChecker = requestStatusChecker;
+    action.diffLister = new GcsDiffFileLister();
+    action.diffLister.gcsService = gcsService;
+    action.diffLister.gcsBucket = GCS_BUCKET;
+    action.diffLister.executor = newDirectExecutorService();
+    RegistryConfig.overrideCloudSqlReplayCommitLogs(true);
+  }
+
+  @Test
+  void testReplay_multipleDiffFiles() throws Exception {
+    jpaTm()
+        .transact(
+            () -> {
+              jpaTm().insertWithoutBackup(TestObject.create("previous to keep"));
+              jpaTm().insertWithoutBackup(TestObject.create("previous to delete"));
+            });
+    DateTime now = fakeClock.nowUtc();
+    // Create 3 transactions, across two diff files.
+    // Before: {"previous to keep", "previous to delete"}
+    // 1a: Add {"a", "b"}, Delete {"previous to delete"}
+    // 1b: Add {"c", "d"}, Delete {"a"}
+    // 2:  Add {"e", "f"}, Delete {"c"}
+    // After:  {"previous to keep", "b", "d", "e", "f"}
+    Key<CommitLogManifest> manifest1aKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(3));
+    Key<CommitLogManifest> manifest1bKey =
+        CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
+    Key<CommitLogManifest> manifest2Key =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now.minusMinutes(3),
+            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))),
+        CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
+        CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
+        CommitLogManifest.create(
+            getBucketKey(2),
+            now.minusMinutes(2),
+            ImmutableSet.of(Key.create(TestObject.create("a")))),
+        CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
+        CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now.minusMinutes(1),
+            ImmutableSet.of(Key.create(TestObject.create("c")))),
+        CommitLogMutation.create(manifest2Key, TestObject.create("e")),
+        CommitLogMutation.create(manifest2Key, TestObject.create("f")));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    fakeClock.advanceOneMilli();
+    runAndAssertSuccess(now);
+    assertExpectedIds("previous to keep", "b", "d", "e", "f");
+  }
+
+  @Test
+  void testReplay_noManifests() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    saveDiffFile(gcsService, createCheckpoint(now.minusMillis(2)));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1)));
+    runAndAssertSuccess(now.minusMillis(1));
+    assertExpectedIds("previous to keep");
+  }
+
+  @Test
+  void testReplay_manifestWithNoDeletions() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
+    Key<CommitLogBucket> bucketKey = getBucketKey(1);
+    Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(bucketKey, now, null),
+        CommitLogMutation.create(manifestKey, TestObject.create("a")),
+        CommitLogMutation.create(manifestKey, TestObject.create("b")));
+    runAndAssertSuccess(now.minusMinutes(1));
+    assertExpectedIds("previous to keep", "a", "b");
+  }
+
+  @Test
+  void testReplay_manifestWithNoMutations() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm()
+        .transact(
+            () -> {
+              jpaTm().insertWithoutBackup(TestObject.create("previous to keep"));
+              jpaTm().insertWithoutBackup(TestObject.create("previous to delete"));
+            });
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now,
+            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    runAndAssertSuccess(now.minusMinutes(1));
+    assertExpectedIds("previous to keep");
+  }
+
+  @Test
+  void wtestReplay_mutateExistingEntity() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().put(TestObject.create("existing", "a")));
+    Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMillis(1)),
+        CommitLogManifest.create(getBucketKey(1), now, null),
+        CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
+    action.run();
+    TestObject fromDatabase =
+        jpaTm().transact(() -> jpaTm().load(VKey.createSql(TestObject.class, "existing")));
+    assertThat(fromDatabase.getField()).isEqualTo("b");
+  }
+
+  // This should be harmless
+  @Test
+  void testReplay_deleteMissingEntity() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().put(TestObject.create("previous to keep", "a")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMillis(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now,
+            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    action.run();
+    assertExpectedIds("previous to keep");
+  }
+
+  @Test
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  void testReplay_properlyWeighted() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    Key<CommitLogManifest> manifestKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+    // Create (but don't save to SQL) a domain + contact
+    createTld("tld");
+    DomainBase domain = newDomainBase("example.tld");
+    CommitLogMutation domainMutation =
+        tm().transact(() -> CommitLogMutation.create(manifestKey, domain));
+    ContactResource contact = tm().transact(() -> tm().load(domain.getRegistrant()));
+    CommitLogMutation contactMutation =
+        tm().transact(() -> CommitLogMutation.create(manifestKey, contact));
+
+    // Create and save to SQL a registrar contact that we will delete
+    RegistrarContact toDelete = AppEngineExtension.makeRegistrarContact1();
+    jpaTm().transact(() -> jpaTm().put(toDelete));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+
+    // spy the txn manager so we can see what order things were inserted/removed
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+    // Save in the commit logs the domain and contact (in that order) and the token deletion
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(toDelete))),
+        domainMutation,
+        contactMutation);
+
+    runAndAssertSuccess(now.minusMinutes(1));
+    // Verify two things:
+    // 1. that the contact insert occurred before the domain insert (necessary for FK ordering)
+    //    even though the domain came first in the file
+    // 2. that the allocation token delete occurred after the insertions
+    InOrder inOrder = Mockito.inOrder(spy);
+    inOrder.verify(spy).put(any(ContactResource.class));
+    inOrder.verify(spy).put(any(DomainBase.class));
+    inOrder.verify(spy).delete(toDelete.createVKey());
+    inOrder.verify(spy).put(any(SqlReplayCheckpoint.class));
+  }
+
+  @Test
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  void testReplay_properlyWeighted_doesNotApplyCrossTransactions() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    Key<CommitLogManifest> manifestKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+
+    // Create and save the standard contact
+    ContactResource contact = persistActiveContact("contact1234");
+    jpaTm().transact(() -> jpaTm().put(contact));
+
+    // Simulate a Datastore transaction with a new version of the contact
+    ContactResource contactWithEdit =
+        contact.asBuilder().setEmailAddress("replay@example.tld").build();
+    CommitLogMutation contactMutation =
+        tm().transact(() -> CommitLogMutation.create(manifestKey, contactWithEdit));
+
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+
+    // spy the txn manager so we can see what order things were inserted
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+    // Save two commits -- the deletion, then the new version of the contact
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1).plusMillis(1)),
+        CommitLogManifest.create(
+            getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(contact))),
+        CommitLogManifest.create(
+            getBucketKey(1), now.minusMinutes(1).plusMillis(1), ImmutableSet.of()),
+        contactMutation);
+    runAndAssertSuccess(now.minusMinutes(1).plusMillis(1));
+    // Verify that the delete occurred first (because it was in the first transaction) even though
+    // deletes have higher weight
+    ArgumentCaptor<Object> putCaptor = ArgumentCaptor.forClass(Object.class);
+    InOrder inOrder = Mockito.inOrder(spy);
+    inOrder.verify(spy).delete(contact.createVKey());
+    inOrder.verify(spy).put(putCaptor.capture());
+    assertThat(putCaptor.getValue().getClass()).isEqualTo(ContactResource.class);
+    assertThat(jpaTm().transact(() -> jpaTm().load(contact.createVKey()).getEmailAddress()))
+        .isEqualTo("replay@example.tld");
+  }
+
+  @Test
+  void testSuccess_nonReplicatedEntity_isNotReplayed() {
+    DateTime now = fakeClock.nowUtc();
+
+    // spy the txn manager so we can verify it's never called
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    Key<CommitLogManifest> manifestKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+
+    // Have a commit log with a couple objects that shouldn't be replayed
+    ReservedList reservedList =
+        new ReservedList.Builder().setReservedListMap(ImmutableMap.of()).setName("name").build();
+    Cursor cursor = Cursor.createGlobal(CursorType.RECURRING_BILLING, now.minusHours(1));
+    tm().transact(
+            () -> {
+              try {
+                saveDiffFile(
+                    gcsService,
+                    createCheckpoint(now.minusMinutes(1)),
+                    CommitLogManifest.create(
+                        getBucketKey(1), now.minusMinutes(1), ImmutableSet.of()),
+                    // Reserved list is dually-written non-replicated
+                    CommitLogMutation.create(manifestKey, reservedList),
+                    // Cursors aren't replayed to SQL at all
+                    CommitLogMutation.create(manifestKey, cursor));
+              } catch (IOException e) {
+                throw new RuntimeException(e);
+              }
+            });
+    runAndAssertSuccess(now.minusMinutes(1));
+    // jpaTm()::put should only have been called with the checkpoint
+    verify(spy, times(2)).put(any(SqlReplayCheckpoint.class));
+    verify(spy, times(2)).put(any());
+  }
+
+  @Test
+  void testSuccess_nonReplicatedEntity_isNotDeleted() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    // spy the txn manager so we can verify it's never called
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    // Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
+    Key<ClaimsListShard> claimsListKey = Key.create(ClaimsListShard.class, 1L);
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now.minusMinutes(1),
+            // one object only exists in Datastore, one is dually-written (so isn't replicated)
+            ImmutableSet.of(getCrossTldKey(), claimsListKey)));
+
+    runAndAssertSuccess(now.minusMinutes(1));
+    verify(spy, times(0)).delete(any(VKey.class));
+  }
+
+  @Test
+  void testFailure_notEnabled() {
+    RegistryConfig.overrideCloudSqlReplayCommitLogs(false);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
+    assertThat(response.getPayload())
+        .isEqualTo("ReplayCommitLogsToSqlAction was called but disabled in the config.");
+  }
+
+  @Test
+  void testFailure_cannotAcquireLock() {
+    Truth8.assertThat(
+            Lock.acquire(
+                ReplayCommitLogsToSqlAction.class.getSimpleName(),
+                null,
+                Duration.standardHours(1),
+                requestStatusChecker,
+                false))
+        .isPresent();
+    fakeClock.advanceOneMilli();
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
+    assertThat(response.getPayload())
+        .isEqualTo("Can't acquire SQL commit log replay lock, aborting.");
+  }
+
+  private void runAndAssertSuccess(DateTime expectedCheckpointTime) {
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(expectedCheckpointTime);
+  }
+
+  private void assertExpectedIds(String... expectedIds) {
+    ImmutableList<String> actualIds =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAll(TestObject.class).stream()
+                        .map(TestObject::getId)
+                        .collect(toImmutableList()));
+    assertThat(actualIds).containsExactlyElementsIn(expectedIds);
+  }
+}

core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java

@@ -61,7 +61,7 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 /** Unit tests for {@link RestoreCommitLogsAction}. */
 public class RestoreCommitLogsActionTest {
 
-  private static final String GCS_BUCKET = "gcs bucket";
+  static final String GCS_BUCKET = "gcs bucket";
 
   private final DateTime now = DateTime.now(UTC);
   private final RestoreCommitLogsAction action = new RestoreCommitLogsAction();
@@ -89,9 +89,10 @@ public class RestoreCommitLogsActionTest {
 
   @Test
   void testRestore_multipleDiffFiles() throws Exception {
-    ofy().saveWithoutBackup().entities(
-        TestObject.create("previous to keep"),
-        TestObject.create("previous to delete")).now();
+    ofy()
+        .saveWithoutBackup()
+        .entities(TestObject.create("previous to keep"), TestObject.create("previous to delete"))
+        .now();
     // Create 3 transactions, across two diff files.
     // Before: {"previous to keep", "previous to delete"}
     // 1a: Add {"a", "b"}, Delete {"previous to delete"}
@@ -104,29 +105,33 @@ public class RestoreCommitLogsActionTest {
         CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
     Key<CommitLogManifest> manifest2Key =
         CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
-    saveDiffFileNotToRestore(now.minusMinutes(2));
-    Iterable<ImmutableObject> file1CommitLogs = saveDiffFile(
-        createCheckpoint(now.minusMinutes(1)),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now.minusMinutes(3),
-            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))),
-        CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
-        CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
-        CommitLogManifest.create(
-            getBucketKey(2),
-            now.minusMinutes(2),
-            ImmutableSet.of(Key.create(TestObject.create("a")))),
-        CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
-        CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
-    Iterable<ImmutableObject> file2CommitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now.minusMinutes(1),
-            ImmutableSet.of(Key.create(TestObject.create("c")))),
-        CommitLogMutation.create(manifest2Key, TestObject.create("e")),
-        CommitLogMutation.create(manifest2Key, TestObject.create("f")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    Iterable<ImmutableObject> file1CommitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now.minusMinutes(1)),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now.minusMinutes(3),
+                ImmutableSet.of(Key.create(TestObject.create("previous to delete")))),
+            CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
+            CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
+            CommitLogManifest.create(
+                getBucketKey(2),
+                now.minusMinutes(2),
+                ImmutableSet.of(Key.create(TestObject.create("a")))),
+            CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
+            CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
+    Iterable<ImmutableObject> file2CommitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now.minusMinutes(1),
+                ImmutableSet.of(Key.create(TestObject.create("c")))),
+            CommitLogMutation.create(manifest2Key, TestObject.create("e")),
+            CommitLogMutation.create(manifest2Key, TestObject.create("f")));
     action.fromTime = now.minusMinutes(1).minusMillis(1);
     action.run();
     ofy().clearSessionCache();
@@ -139,10 +144,9 @@ public class RestoreCommitLogsActionTest {
 
   @Test
   void testRestore_noManifests() throws Exception {
-    ofy().saveWithoutBackup().entity(
-        TestObject.create("previous to keep")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(createCheckpoint(now));
+    ofy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs = saveDiffFile(gcsService, createCheckpoint(now));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -156,32 +160,37 @@ public class RestoreCommitLogsActionTest {
     ofy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
     Key<CommitLogBucket> bucketKey = getBucketKey(1);
     Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(bucketKey, now, null),
-        CommitLogMutation.create(manifestKey, TestObject.create("a")),
-        CommitLogMutation.create(manifestKey, TestObject.create("b")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(bucketKey, now, null),
+            CommitLogMutation.create(manifestKey, TestObject.create("a")),
+            CommitLogMutation.create(manifestKey, TestObject.create("b")));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep", "a", "b");
     assertInDatastore(commitLogs);
     assertInDatastore(CommitLogCheckpointRoot.create(now));
     assertCommitLogBuckets(ImmutableMap.of(1, now));
-}
+  }
 
   @Test
   void testRestore_manifestWithNoMutations() throws Exception {
-    ofy().saveWithoutBackup().entities(
-        TestObject.create("previous to keep"),
-        TestObject.create("previous to delete")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now,
-            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    ofy()
+        .saveWithoutBackup()
+        .entities(TestObject.create("previous to keep"), TestObject.create("previous to delete"))
+        .now();
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now,
+                ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -193,12 +202,13 @@ public class RestoreCommitLogsActionTest {
   // This is a pathological case that shouldn't be possible, but we should be robust to it.
   @Test
   void testRestore_manifestWithNoMutationsOrDeletions() throws Exception {
-    ofy().saveWithoutBackup().entities(
-        TestObject.create("previous to keep")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(getBucketKey(1), now, null));
+    ofy().saveWithoutBackup().entities(TestObject.create("previous to keep")).now();
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(getBucketKey(1), now, null));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -211,11 +221,13 @@ public class RestoreCommitLogsActionTest {
   void testRestore_mutateExistingEntity() throws Exception {
     ofy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now();
     Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(getBucketKey(1), now, null),
-        CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(getBucketKey(1), now, null),
+            CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
     action.run();
     ofy().clearSessionCache();
     assertThat(ofy().load().entity(TestObject.create("existing")).now().getField()).isEqualTo("b");
@@ -228,13 +240,15 @@ public class RestoreCommitLogsActionTest {
   @Test
   void testRestore_deleteMissingEntity() throws Exception {
     ofy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now,
-            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now,
+                ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -243,12 +257,13 @@ public class RestoreCommitLogsActionTest {
     assertInDatastore(CommitLogCheckpointRoot.create(now));
   }
 
-  private CommitLogCheckpoint createCheckpoint(DateTime now) {
+  static CommitLogCheckpoint createCheckpoint(DateTime now) {
     return CommitLogCheckpoint.create(now, toMap(getBucketIds(), x -> now));
   }
 
-  private Iterable<ImmutableObject> saveDiffFile(
-      CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException {
+  static Iterable<ImmutableObject> saveDiffFile(
+      GcsService gcsService, CommitLogCheckpoint checkpoint, ImmutableObject... entities)
+      throws IOException {
     DateTime now = checkpoint.getCheckpointTime();
     List<ImmutableObject> allEntities = Lists.asList(checkpoint, entities);
     ByteArrayOutputStream output = new ByteArrayOutputStream();
@@ -264,8 +279,9 @@ public class RestoreCommitLogsActionTest {
     return allEntities;
   }
 
-  private void saveDiffFileNotToRestore(DateTime now) throws Exception {
+  static void saveDiffFileNotToRestore(GcsService gcsService, DateTime now) throws Exception {
     saveDiffFile(
+        gcsService,
         createCheckpoint(now),
         CommitLogManifest.create(getBucketKey(1), now, null),
         CommitLogMutation.create(

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -20,7 +20,6 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.withSettings;
 
@@ -285,8 +284,7 @@ class Spec11PipelineTest {
             .build();
 
     verify(mockJpaTm).transact(any(Runnable.class));
-    verify(mockJpaTm).insert(expected);
-    verifyNoMoreInteractions(mockJpaTm);
+    verify(mockJpaTm).putAll(ImmutableList.of(expected));
   }
 
   /**

core/src/test/java/google/registry/flows/EppLifecycleDomainTest.java

@@ -501,7 +501,7 @@ class EppLifecycleDomainTest extends EppTestCase {
 
   @Test
   void testEapDomainDeletion_withinAddGracePeriod_eapFeeIsNotRefunded() throws Exception {
-    assertThatCommand("login_valid_fee_extension.xml").hasResponse("generic_success_response.xml");
+    assertThatCommand("login_valid_fee_extension.xml").hasSuccessfulLogin();
     createContacts(DateTime.parse("2000-06-01T00:00:00Z"));
 
     // Set the EAP schedule.
@@ -718,7 +718,7 @@ class EppLifecycleDomainTest extends EppTestCase {
             START_OF_TIME, PREDELEGATION,
             gaDate, GENERAL_AVAILABILITY));
 
-    assertThatCommand("login_valid_fee_extension.xml").hasResponse("generic_success_response.xml");
+    assertThatCommand("login_valid_fee_extension.xml").hasSuccessfulLogin();
 
     assertThatCommand("domain_check_fee_premium.xml")
         .atTime(gaDate.plusDays(1))
@@ -1196,7 +1196,7 @@ class EppLifecycleDomainTest extends EppTestCase {
 
     assertThatLogin("NewRegistrar", "foo-BAR2")
         .atTime(sunriseDate.minusDays(3))
-        .hasResponse("generic_success_response.xml");
+        .hasSuccessfulLogin();
 
     createContactsAndHosts();
 
@@ -1292,7 +1292,7 @@ class EppLifecycleDomainTest extends EppTestCase {
 
     assertThatLogin("NewRegistrar", "foo-BAR2")
         .atTime(sunriseDate.minusDays(3))
-        .hasResponse("generic_success_response.xml");
+        .hasSuccessfulLogin();
 
     createContactsAndHosts();
 

core/src/test/java/google/registry/flows/EppLoginTlsTest.java

@@ -44,13 +44,13 @@ class EppLoginTlsTest extends EppTestCase {
     persistResource(
         loadRegistrar("NewRegistrar")
             .asBuilder()
-            .setClientCertificateHash(CertificateSamples.SAMPLE_CERT_HASH)
+            .setClientCertificate(CertificateSamples.SAMPLE_CERT, DateTime.now(UTC))
             .build());
     // Set a cert for the second registrar, or else any cert will be allowed for login.
     persistResource(
         loadRegistrar("TheRegistrar")
             .asBuilder()
-            .setClientCertificateHash(CertificateSamples.SAMPLE_CERT2_HASH)
+            .setClientCertificate(CertificateSamples.SAMPLE_CERT2, DateTime.now(UTC))
             .build());
   }
 

core/src/test/java/google/registry/flows/EppTestCase.java

@@ -121,6 +121,10 @@ public class EppTestCase {
       return assertCommandAndResponse(
           inputFilename, inputSubstitutions, outputFilename, outputSubstitutions, now);
     }
+
+    public String hasSuccessfulLogin() throws Exception {
+      return assertLoginCommandAndResponse(inputFilename, inputSubstitutions, null, now);
+    }
   }
 
   protected CommandAsserter assertThatCommand(String inputFilename) {
@@ -137,13 +141,33 @@ public class EppTestCase {
   }
 
   protected void assertThatLoginSucceeds(String clientId, String password) throws Exception {
-    assertThatLogin(clientId, password).hasResponse("generic_success_response.xml");
+    assertThatLogin(clientId, password).hasSuccessfulLogin();
   }
 
   protected void assertThatLogoutSucceeds() throws Exception {
     assertThatCommand("logout.xml").hasResponse("logout_response.xml");
   }
 
+  private String assertLoginCommandAndResponse(
+      String inputFilename,
+      @Nullable Map<String, String> inputSubstitutions,
+      @Nullable Map<String, String> outputSubstitutions,
+      DateTime now)
+      throws Exception {
+    String outputFilename = "generic_success_response.xml";
+    clock.setTo(now);
+    String input = loadFile(EppTestCase.class, inputFilename, inputSubstitutions);
+    String expectedOutput = loadFile(EppTestCase.class, outputFilename, outputSubstitutions);
+    setUpSession();
+    FakeResponse response = executeXmlCommand(input);
+
+    // Check that the logged-in header was added to the response
+    assertThat(response.getHeaders()).isEqualTo(ImmutableMap.of("Logged-In", "true"));
+
+    return verifyAndReturnOutput(
+        response.getPayload(), expectedOutput, inputFilename, outputFilename);
+  }
+
   private String assertCommandAndResponse(
       String inputFilename,
       @Nullable Map<String, String> inputSubstitutions,
@@ -154,6 +178,18 @@ public class EppTestCase {
     clock.setTo(now);
     String input = loadFile(EppTestCase.class, inputFilename, inputSubstitutions);
     String expectedOutput = loadFile(EppTestCase.class, outputFilename, outputSubstitutions);
+    setUpSession();
+    FakeResponse response = executeXmlCommand(input);
+
+    // Checks that the Logged-In header is not in the response. If testing the login command, use
+    // assertLoginCommandAndResponse instead of this method.
+    assertThat(response.getHeaders()).doesNotContainEntry("Logged-In", "true");
+
+    return verifyAndReturnOutput(
+        response.getPayload(), expectedOutput, inputFilename, outputFilename);
+  }
+
+  private void setUpSession() {
     if (sessionMetadata == null) {
       sessionMetadata =
           new HttpSessionMetadata(new FakeHttpSession()) {
@@ -165,7 +201,13 @@ public class EppTestCase {
             }
           };
     }
-    String actualOutput = executeXmlCommand(input);
+  }
+
+  private String verifyAndReturnOutput(
+      String actualOutput, String expectedOutput, String inputFilename, String outputFilename)
+      throws Exception {
+    // Run the resulting xml through the unmarshaller to verify that it was valid.
+    EppXmlTransformer.validateOutput(actualOutput);
     assertXmlEqualsWithMessage(
         expectedOutput,
         actualOutput,
@@ -176,7 +218,7 @@ public class EppTestCase {
     return actualOutput;
   }
 
-  private String executeXmlCommand(String inputXml) throws Exception {
+  private FakeResponse executeXmlCommand(String inputXml) throws Exception {
     EppRequestHandler handler = new EppRequestHandler();
     FakeResponse response = new FakeResponse();
     handler.response = response;
@@ -195,10 +237,7 @@ public class EppTestCase {
         inputXml.getBytes(UTF_8));
     assertThat(response.getStatus()).isEqualTo(SC_OK);
     assertThat(response.getContentType()).isEqualTo(APPLICATION_EPP_XML_UTF8);
-    String result = response.getPayload();
-    // Run the resulting xml through the unmarshaller to verify that it was valid.
-    EppXmlTransformer.validateOutput(result);
-    return result;
+    return response;
   }
 
   EppMetric getRecordedEppMetric() {

core/src/test/java/google/registry/flows/domain/DomainCheckFlowTest.java

@@ -71,7 +71,6 @@ import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.model.registry.label.ReservedList;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -169,7 +168,7 @@ class DomainCheckFlowTest extends ResourceCheckFlowTestCase<DomainCheckFlow, Dom
         new AllocationToken.Builder()
             .setToken("abc123")
             .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
             .build());
     doCheckTest(
         create(false, "example1.tld", "In use"),

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -162,7 +162,6 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.monitoring.whitebox.EppMetric;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.math.BigDecimal;
@@ -505,7 +504,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
         new AllocationToken.Builder()
             .setToken("abc123")
             .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
             .build());
     clock.advanceOneMilli();
     EppException thrown =
@@ -528,7 +527,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     HistoryEntry historyEntry =
         ofy().load().type(HistoryEntry.class).ancestor(reloadResourceByForeignKey()).first().now();
     assertThat(ofy().load().entity(token).now().getRedemptionHistoryEntry())
-        .hasValue(DomainHistoryVKey.create(Key.create(historyEntry)));
+        .hasValue(HistoryEntry.createVKey(Key.create(historyEntry)));
   }
 
   @Test
@@ -1275,7 +1274,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     assertThat(reloadedToken.isRedeemed()).isTrue();
     assertThat(reloadedToken.getRedemptionHistoryEntry())
         .hasValue(
-            DomainHistoryVKey.create(
+            HistoryEntry.createVKey(
                 Key.create(getHistoryEntries(reloadResourceByForeignKey()).get(0))));
   }
 

core/src/test/java/google/registry/flows/domain/DomainRenewFlowTest.java

@@ -26,6 +26,7 @@ import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
 import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.DatabaseHelper.persistResources;
 import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
@@ -69,12 +70,17 @@ import google.registry.model.registry.Registry;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.testing.ReplayExtension;
 import java.util.Map;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link DomainRenewFlow}. */
 class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBase> {
@@ -96,50 +102,66 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
 
   private final DateTime expirationTime = DateTime.parse("2000-04-03T22:00:00.0Z");
 
+  @Order(value = Order.DEFAULT - 3)
+  @RegisterExtension
+  final SetClockExtension setClockExtension = new SetClockExtension();
+
+  @Order(value = Order.DEFAULT - 2)
+  @RegisterExtension
+  final ReplayExtension replayExtension = new ReplayExtension(clock);
+
   @BeforeEach
   void initDomainTest() {
     createTld("tld");
-    clock.setTo(expirationTime.minusMillis(2));
     setEppInput("domain_renew.xml", ImmutableMap.of("DOMAIN", "example.tld", "YEARS", "5"));
   }
 
   private void persistDomain(StatusValue... statusValues) throws Exception {
     DomainBase domain = newDomainBase(getUniqueIdFromCommand());
-    HistoryEntry historyEntryDomainCreate =
-        persistResource(
-            new HistoryEntry.Builder()
-                .setParent(domain)
-                .setType(HistoryEntry.Type.DOMAIN_CREATE)
-                .build());
-    BillingEvent.Recurring autorenewEvent =
-        persistResource(
-            new BillingEvent.Recurring.Builder()
-                .setReason(Reason.RENEW)
-                .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
-                .setTargetId(getUniqueIdFromCommand())
-                .setClientId("TheRegistrar")
-                .setEventTime(expirationTime)
-                .setRecurrenceEndTime(END_OF_TIME)
-                .setParent(historyEntryDomainCreate)
-                .build());
-    PollMessage.Autorenew autorenewPollMessage =
-        persistResource(
-            new PollMessage.Autorenew.Builder()
-                .setTargetId(getUniqueIdFromCommand())
-                .setClientId("TheRegistrar")
-                .setEventTime(expirationTime)
-                .setAutorenewEndTime(END_OF_TIME)
-                .setMsg("Domain was auto-renewed.")
-                .setParent(historyEntryDomainCreate)
-                .build());
-    persistResource(
-        domain
-            .asBuilder()
-            .setRegistrationExpirationTime(expirationTime)
-            .setStatusValues(ImmutableSet.copyOf(statusValues))
-            .setAutorenewBillingEvent(autorenewEvent.createVKey())
-            .setAutorenewPollMessage(autorenewPollMessage.createVKey())
-            .build());
+    tm().transact(
+            () -> {
+              try {
+                HistoryEntry historyEntryDomainCreate =
+                    new HistoryEntry.Builder()
+                        .setParent(domain)
+                        .setType(HistoryEntry.Type.DOMAIN_CREATE)
+                        .setModificationTime(clock.nowUtc())
+                        .build();
+                BillingEvent.Recurring autorenewEvent =
+                    new BillingEvent.Recurring.Builder()
+                        .setReason(Reason.RENEW)
+                        .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
+                        .setTargetId(getUniqueIdFromCommand())
+                        .setClientId("TheRegistrar")
+                        .setEventTime(expirationTime)
+                        .setRecurrenceEndTime(END_OF_TIME)
+                        .setParent(historyEntryDomainCreate)
+                        .build();
+                PollMessage.Autorenew autorenewPollMessage =
+                    new PollMessage.Autorenew.Builder()
+                        .setTargetId(getUniqueIdFromCommand())
+                        .setClientId("TheRegistrar")
+                        .setEventTime(expirationTime)
+                        .setAutorenewEndTime(END_OF_TIME)
+                        .setMsg("Domain was auto-renewed.")
+                        .setParent(historyEntryDomainCreate)
+                        .build();
+                DomainBase newDomain =
+                    domain
+                        .asBuilder()
+                        .setRegistrationExpirationTime(expirationTime)
+                        .setStatusValues(ImmutableSet.copyOf(statusValues))
+                        .setAutorenewBillingEvent(autorenewEvent.createVKey())
+                        .setAutorenewPollMessage(autorenewPollMessage.createVKey())
+                        .build();
+                persistResources(
+                    ImmutableSet.of(
+                        historyEntryDomainCreate, autorenewEvent,
+                        autorenewPollMessage, newDomain));
+              } catch (Exception e) {
+                throw new RuntimeException("Error persisting domain", e);
+              }
+            });
     clock.advanceOneMilli();
   }
 
@@ -255,6 +277,7 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
 
   @Test
   void testSuccess() throws Exception {
+    clock.advanceOneMilli();
     persistDomain();
     doSuccessfulTest(
         "domain_renew_response.xml",
@@ -789,4 +812,21 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
                 TransactionReportField.netRenewsFieldFromYears(5),
                 1));
   }
+
+  /**
+   * Local extension so we can set the clock correctly prior to using it to define basic entities in
+   * AppEngineExtension.
+   *
+   * <p>This has to happen first, we'll get timestamp inversions if we try to set the clock after
+   * these objects are created.
+   */
+  class SetClockExtension implements BeforeEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext context) {
+      // we have to go far enough back before the expiration time so that the clock advances we do
+      // after each persist don't move us into a grace period.  The current value is likely beyond
+      // what is necessary, but this doesn't hurt.
+      clock.setTo(expirationTime.minusMillis(20));
+    }
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -90,11 +90,14 @@ import google.registry.model.host.HostResource;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.testing.ReplayExtension;
 import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link DomainUpdateFlow}. */
 class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, DomainBase> {
@@ -112,6 +115,10 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   private ContactResource mak21Contact;
   private ContactResource unusedContact;
 
+  @Order(value = Order.DEFAULT - 2)
+  @RegisterExtension
+  final ReplayExtension replayExtension = new ReplayExtension(clock);
+
   @BeforeEach
   void initDomainTest() {
     createTld("tld");
@@ -146,6 +153,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     persistResource(
         new HistoryEntry.Builder()
             .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setModificationTime(clock.nowUtc())
             .setParent(domain)
             .build());
     clock.advanceOneMilli();
@@ -168,6 +176,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     persistResource(
         new HistoryEntry.Builder()
             .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setModificationTime(clock.nowUtc())
             .setParent(domain)
             .build());
     clock.advanceOneMilli();

core/src/test/java/google/registry/flows/domain/token/AllocationTokenFlowUtilsTest.java

@@ -49,7 +49,6 @@ import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.testing.AppEngineExtension;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
@@ -198,7 +197,7 @@ class AllocationTokenFlowUtilsTest {
         new AllocationToken.Builder()
             .setToken("tokeN")
             .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
             .build());
     assertThat(
             flowUtils

core/src/test/java/google/registry/flows/session/LoginFlowTestCase.java

@@ -14,6 +14,7 @@
 
 package google.registry.flows.session;
 
+import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.DatabaseHelper.deleteResource;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.persistResource;
@@ -32,6 +33,7 @@ import google.registry.flows.session.LoginFlow.PasswordChangesNotSupportedExcept
 import google.registry.flows.session.LoginFlow.RegistrarAccountNotActiveException;
 import google.registry.flows.session.LoginFlow.TooManyFailedLoginsException;
 import google.registry.flows.session.LoginFlow.UnsupportedLanguageException;
+import google.registry.model.eppoutput.EppOutput;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.State;
 import org.junit.jupiter.api.BeforeEach;
@@ -74,6 +76,14 @@ public abstract class LoginFlowTestCase extends FlowTestCase<LoginFlow> {
     doSuccessfulTest("login_valid.xml");
   }
 
+  @Test
+  void testSuccess_setsIsLoginResponse() throws Exception {
+    setEppInput("login_valid.xml");
+    assertTransactionalFlow(false);
+    EppOutput output = runFlow();
+    assertThat(output.getResponse().isLoginResponse()).isTrue();
+  }
+
   @Test
   void testSuccess_suspendedRegistrar() throws Exception {
     persistResource(getRegistrarBuilder().setState(State.SUSPENDED).build());

core/src/test/java/google/registry/flows/session/LoginFlowViaTlsTest.java

@@ -15,6 +15,7 @@
 package google.registry.flows.session;
 
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.net.InetAddresses;
@@ -26,6 +27,7 @@ import google.registry.model.registrar.Registrar;
 import google.registry.testing.CertificateSamples;
 import google.registry.util.CidrAddressBlock;
 import java.util.Optional;
+import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link LoginFlow} when accessed via a TLS transport. */
@@ -41,7 +43,7 @@ public class LoginFlowViaTlsTest extends LoginFlowTestCase {
   @Override
   protected Registrar.Builder getRegistrarBuilder() {
     return super.getRegistrarBuilder()
-        .setClientCertificateHash(GOOD_CERT)
+        .setClientCertificate(CertificateSamples.SAMPLE_CERT, DateTime.now(UTC))
         .setIpAddressAllowList(
             ImmutableList.of(CidrAddressBlock.create(InetAddresses.forString(GOOD_IP.get()), 32)));
   }

core/src/test/java/google/registry/model/EntityTestCase.java

@@ -28,6 +28,7 @@ import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.annotation.Serialize;
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.ofy.Ofy;
+import google.registry.persistence.EppHistoryVKey;
 import google.registry.persistence.VKey;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
@@ -173,7 +174,8 @@ public abstract class EntityTestCase {
           }
           // Descend into persisted ImmutableObject classes, but not anything else.
           if (ImmutableObject.class.isAssignableFrom(fieldClass)
-              && !VKey.class.isAssignableFrom(fieldClass)) {
+              && !VKey.class.isAssignableFrom(fieldClass)
+              && !EppHistoryVKey.class.isAssignableFrom(fieldClass)) {
             getAllPotentiallyIndexedFieldPaths(fieldClass).stream()
                 .map(subfield -> field.getName() + "." + subfield)
                 .distinct()

core/src/test/java/google/registry/model/OteAccountBuilderTest.java

@@ -156,16 +156,6 @@ public final class OteAccountBuilderTest {
         .isTrue();
   }
 
-  @Test
-  void testCreateOteEntities_setCertificateHash() {
-    OteAccountBuilder.forClientId("myclientid")
-        .setCertificateHash(SAMPLE_CERT_HASH)
-        .buildAndPersist();
-
-    assertThat(Registrar.loadByClientId("myclientid-3").get().getClientCertificateHash())
-        .isEqualTo(SAMPLE_CERT_HASH);
-  }
-
   @Test
   void testCreateOteEntities_setCertificate() {
     OteAccountBuilder.forClientId("myclientid")

core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java

@@ -15,64 +15,106 @@
 package google.registry.model;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.joda.time.DateTimeZone.UTC;
 
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.persistence.VKey;
 import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.FakeClock;
+import google.registry.testing.TestOfyAndSql;
 import org.joda.time.DateTime;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link UpdateAutoTimestamp}. */
+@DualDatabaseTest
 public class UpdateAutoTimestampTest {
 
+  FakeClock clock = new FakeClock();
+
   @RegisterExtension
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder()
           .withDatastoreAndCloudSql()
+          .withJpaUnitTestEntities(UpdateAutoTimestampTestObject.class)
           .withOfyTestEntities(UpdateAutoTimestampTestObject.class)
+          .withClock(clock)
           .build();
 
   /** Timestamped class. */
   @Entity(name = "UatTestEntity")
+  @javax.persistence.Entity
   @EntityForTesting
   public static class UpdateAutoTimestampTestObject extends CrossTldSingleton {
+    @Ignore @javax.persistence.Id long id = SINGLETON_ID;
     UpdateAutoTimestamp updateTime = UpdateAutoTimestamp.create(null);
   }
 
   private UpdateAutoTimestampTestObject reload() {
-    return ofy().load().entity(new UpdateAutoTimestampTestObject()).now();
+    return tm().transact(
+            () ->
+                tm().load(
+                        VKey.create(
+                            UpdateAutoTimestampTestObject.class,
+                            1L,
+                            Key.create(new UpdateAutoTimestampTestObject()))));
   }
 
-  @Test
+  @TestOfyAndSql
   void testSaveSetsTime() {
     DateTime transactionTime =
         tm().transact(
                 () -> {
                   UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                   assertThat(object.updateTime.timestamp).isNull();
-                  ofy().save().entity(object);
+                  tm().insert(object);
                   return tm().getTransactionTime();
                 });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
   }
 
-  @Test
+  @TestOfyAndSql
+  void testDisabledUpdates() throws Exception {
+    DateTime initialTime =
+        tm().transact(
+                () -> {
+                  tm().insert(new UpdateAutoTimestampTestObject());
+                  return tm().getTransactionTime();
+                });
+
+    UpdateAutoTimestampTestObject object = reload();
+    clock.advanceOneMilli();
+
+    try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+        new UpdateAutoTimestamp.DisableAutoUpdateResource()) {
+      DateTime secondTransactionTime =
+          tm().transact(
+                  () -> {
+                    tm().put(object);
+                    return tm().getTransactionTime();
+                  });
+      assertThat(secondTransactionTime).isGreaterThan(initialTime);
+    }
+    assertThat(reload().updateTime.timestamp).isEqualTo(initialTime);
+  }
+
+  @TestOfyAndSql
   void testResavingOverwritesOriginalTime() {
     DateTime transactionTime =
         tm().transact(
                 () -> {
                   UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                   object.updateTime = UpdateAutoTimestamp.create(DateTime.now(UTC).minusDays(1));
-                  ofy().save().entity(object);
+                  tm().insert(object);
                   return tm().getTransactionTime();
                 });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
   }
 }

core/src/test/java/google/registry/model/domain/DomainBaseSqlTest.java

@@ -84,10 +84,10 @@ public class DomainBaseSqlTest {
     saveRegistrar("registrar1");
     saveRegistrar("registrar2");
     saveRegistrar("registrar3");
-    contactKey = VKey.createSql(ContactResource.class, "contact_id1");
-    contact2Key = VKey.createSql(ContactResource.class, "contact_id2");
+    contactKey = createKey(ContactResource.class, "contact_id1");
+    contact2Key = createKey(ContactResource.class, "contact_id2");
 
-    host1VKey = VKey.createSql(HostResource.class, "host1");
+    host1VKey = createKey(HostResource.class, "host1");
 
     domain =
         new DomainBase.Builder()
@@ -633,14 +633,13 @@ public class DomainBaseSqlTest {
                           "4-COM",
                           END_OF_TIME,
                           "registrar1",
-                          createLegacyVKey(
-                              BillingEvent.OneTime.class, oneTimeBillingEvent.getId())),
+                          oneTimeBillingEvent.createVKey()),
                       GracePeriod.createForRecurring(
                           GracePeriodStatus.AUTO_RENEW,
                           "4-COM",
                           END_OF_TIME,
                           "registrar1",
-                          createLegacyVKey(BillingEvent.Recurring.class, billEvent.getId())));
+                          billEvent.createVKey()));
 
               jpaTm().insert(contact);
               jpaTm().insert(contact2);
@@ -696,6 +695,10 @@ public class DomainBaseSqlTest {
     assertThat(domain.getGracePeriods()).isEqualTo(gracePeriods);
   }
 
+  private <T> VKey<T> createKey(Class<T> clazz, String name) {
+    return VKey.create(clazz, name, Key.create(clazz, name));
+  }
+
   private <T> VKey<T> createLegacyVKey(Class<T> clazz, long id) {
     return VKey.create(
         clazz, id, Key.create(Key.create(EntityGroupRoot.class, "per-tld"), clazz, id));

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -875,16 +875,13 @@ public class DomainBaseTest extends EntityTestCase {
     ImmutableSet<BillEventInfo> historyIds =
         domain.getGracePeriods().stream()
             .map(
-                gp ->
-                    new BillEventInfo(
-                        gp.getRecurringBillingEvent(), gp.billingEventRecurringHistoryId,
-                        gp.getOneTimeBillingEvent(), gp.billingEventOneTimeHistoryId))
+                gp -> new BillEventInfo(gp.getRecurringBillingEvent(), gp.getOneTimeBillingEvent()))
             .collect(toImmutableSet());
     assertThat(historyIds)
         .isEqualTo(
             ImmutableSet.of(
-                new BillEventInfo(null, null, oneTimeBillKey, historyEntryKey.getId()),
-                new BillEventInfo(recurringBillKey, historyEntryKey.getId(), null, null)));
+                new BillEventInfo(null, oneTimeBillKey),
+                new BillEventInfo(recurringBillKey, null)));
   }
 
   static class BillEventInfo extends ImmutableObject {
@@ -895,13 +892,9 @@ public class DomainBaseTest extends EntityTestCase {
 
     BillEventInfo(
         VKey<BillingEvent.Recurring> billingEventRecurring,
-        Long billingEventRecurringHistoryId,
-        VKey<BillingEvent.OneTime> billingEventOneTime,
-        Long billingEventOneTimeHistoryId) {
+        VKey<BillingEvent.OneTime> billingEventOneTime) {
       this.billingEventRecurring = billingEventRecurring;
-      this.billingEventRecurringHistoryId = billingEventRecurringHistoryId;
       this.billingEventOneTime = billingEventOneTime;
-      this.billingEventOneTimeHistoryId = billingEventOneTimeHistoryId;
     }
   }
 }

core/src/test/java/google/registry/model/domain/GracePeriodTest.java

@@ -63,11 +63,11 @@ public class GracePeriodTest {
     recurringKey =
         VKey.create(
             Recurring.class,
-            12345,
+            12345L,
             Key.create(
                 Key.create(Key.create(DomainBase.class, "1-TEST"), HistoryEntry.class, 343L),
                 Recurring.class,
-                12345));
+                12345L));
   }
 
   @Test
@@ -80,8 +80,6 @@ public class GracePeriodTest {
     assertThat(gracePeriod.getClientId()).isEqualTo("TheRegistrar");
     assertThat(gracePeriod.getExpirationTime()).isEqualTo(now.plusDays(1));
     assertThat(gracePeriod.hasBillingEvent()).isTrue();
-    assertThat(gracePeriod.billingEventOneTimeHistoryId).isEqualTo(12345L);
-    assertThat(gracePeriod.billingEventRecurringHistoryId).isNull();
   }
 
   @Test
@@ -96,8 +94,6 @@ public class GracePeriodTest {
     assertThat(gracePeriod.getClientId()).isEqualTo("TheRegistrar");
     assertThat(gracePeriod.getExpirationTime()).isEqualTo(now.plusDays(1));
     assertThat(gracePeriod.hasBillingEvent()).isTrue();
-    assertThat(gracePeriod.billingEventOneTimeHistoryId).isNull();
-    assertThat(gracePeriod.billingEventRecurringHistoryId).isEqualTo(343L);
   }
 
   @Test