Validate SQL credentials in Secret Manager (#907)

* Validate SQL credentials in Secret Manager

Load SQL credentials from the SecretManager and compare them with the
ones currently in use in Nomulus server, beam pipeline, and the registry
tool. Normal operations are not affected by failures related to the
SecretManager, be it IOException, insufficient permission , or wrong or
missing credential.

The appengine and compute engine default service accounts must be
granted the permission to access the secret data. In the short term, we
will grant the secretmanager.secretAccessor role to these accounts. In
the long term, with the proposed privilege service, access will be granted
on per-secret basis.

config/presubmits.py

@@ -65,7 +65,7 @@ class PresubmitCheck:
     for pattern in self.skipped_patterns:
       if pattern in file:
         return False
-    with open(file, "r") as f:
+    with open(file, "r", encoding='utf8') as f:
       file_content = f.read()
       matches = re.match(self.regex, file_content, re.DOTALL)
       if self.regex_type == FORBIDDEN:
@@ -241,7 +241,7 @@ def verify_flyway_index():
 
   # Remove the sequence numbers and compare against the index file contents.
   files = [filename[1] for filename in sorted(files)]
-  with open('db/src/main/resources/sql/flyway.txt') as index:
+  with open('db/src/main/resources/sql/flyway.txt', encoding='utf8') as index:
     indexed_files = index.read().splitlines()
   if files != indexed_files:
     unindexed = set(files) - set(indexed_files)

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -15,10 +15,10 @@
 package google.registry.backup;
 
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.backup.BackupUtils.createDeserializingIterator;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Streams;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ofy.CommitLogCheckpoint;
 import google.registry.model.ofy.CommitLogManifest;
@@ -31,7 +31,6 @@ import java.io.InputStream;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.util.Iterator;
-import java.util.stream.Stream;
 
 /**
  * Helpers for reading CommitLog records from a file.
@@ -43,6 +42,56 @@ public final class CommitLogImports {
 
   private CommitLogImports() {}
 
+  /**
+   * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link
+   * ImmutableList} of {@link ImmutableList}s of {@link VersionedEntity} records where the inner
+   * lists each consist of one transaction. Upon completion the {@code inputStream} is closed.
+   *
+   * <p>The returned list may be empty, since CommitLogs are written at fixed intervals regardless
+   * if actual changes exist. Each sublist, however, will not be empty.
+   *
+   * <p>A CommitLog file starts with a {@link CommitLogCheckpoint}, followed by (repeated)
+   * subsequences of [{@link CommitLogManifest}, [{@link CommitLogMutation}] ...]. Each subsequence
+   * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
+   * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
+   */
+  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+      InputStream inputStream) {
+    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
+        InputStream input = new BufferedInputStream(inputStream)) {
+      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
+      checkState(commitLogs.hasNext());
+      checkState(commitLogs.next() instanceof CommitLogCheckpoint);
+
+      ImmutableList.Builder<ImmutableList<VersionedEntity>> resultBuilder =
+          new ImmutableList.Builder<>();
+      ImmutableList.Builder<VersionedEntity> currentTransactionBuilder =
+          new ImmutableList.Builder<>();
+
+      while (commitLogs.hasNext()) {
+        ImmutableObject currentObject = commitLogs.next();
+        if (currentObject instanceof CommitLogManifest) {
+          // CommitLogManifest means we are starting a new transaction
+          addIfNonempty(resultBuilder, currentTransactionBuilder);
+          currentTransactionBuilder = new ImmutableList.Builder<>();
+          VersionedEntity.fromManifest((CommitLogManifest) currentObject)
+              .forEach(currentTransactionBuilder::add);
+        } else if (currentObject instanceof CommitLogMutation) {
+          currentTransactionBuilder.add(
+              VersionedEntity.fromMutation((CommitLogMutation) currentObject));
+        } else {
+          throw new IllegalStateException(
+              String.format("Unknown entity type %s in commit logs", currentObject.getClass()));
+        }
+      }
+      // Add the last transaction in (if it's not empty)
+      addIfNonempty(resultBuilder, currentTransactionBuilder);
+      return resultBuilder.build();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   /**
    * Returns entities in an {@code inputStream} (from a single CommitLog file) as an {@link
    * ImmutableList} of {@link VersionedEntity} records. Upon completion the {@code inputStream} is
@@ -57,23 +106,9 @@ public final class CommitLogImports {
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
   public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
-    try (AppEngineEnvironment appEngineEnvironment = new AppEngineEnvironment();
-        InputStream input = new BufferedInputStream(inputStream)) {
-      Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input);
-      checkState(commitLogs.hasNext());
-      checkState(commitLogs.next() instanceof CommitLogCheckpoint);
-
-      return Streams.stream(commitLogs)
-          .map(
-              e ->
-                  e instanceof CommitLogManifest
-                      ? VersionedEntity.fromManifest((CommitLogManifest) e)
-                      : Stream.of(VersionedEntity.fromMutation((CommitLogMutation) e)))
-          .flatMap(s -> s)
-          .collect(ImmutableList.toImmutableList());
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
+    return loadEntitiesByTransaction(inputStream).stream()
+        .flatMap(ImmutableList::stream)
+        .collect(toImmutableList());
   }
 
   /** Covenience method that adapts {@link #loadEntities(InputStream)} to a {@link File}. */
@@ -92,4 +127,13 @@ public final class CommitLogImports {
   public static ImmutableList<VersionedEntity> loadEntities(ReadableByteChannel channel) {
     return loadEntities(Channels.newInputStream(channel));
   }
+
+  private static void addIfNonempty(
+      ImmutableList.Builder<ImmutableList<VersionedEntity>> resultBuilder,
+      ImmutableList.Builder<VersionedEntity> currentTransactionBuilder) {
+    ImmutableList<VersionedEntity> currentTransaction = currentTransactionBuilder.build();
+    if (!currentTransaction.isEmpty()) {
+      resultBuilder.add(currentTransaction);
+    }
+  }
 }

core/src/main/java/google/registry/backup/GcsDiffFileLister.java

@@ -122,7 +122,7 @@ class GcsDiffFileLister {
 
     // Reconstruct the sequence of files by traversing backwards from "lastUpperBoundTime" (i.e. the
     // last file that we found) and finding its previous file until we either run out of files or
-    // get to one that preceeds "fromTime".
+    // get to one that precedes "fromTime".
     //
     // GCS file listing is eventually consistent, so it's possible that we are missing a file. The
     // metadata of a file is sufficient to identify the preceding file, so if we start from the

core/src/main/java/google/registry/backup/ReplayCommitLogsToSqlAction.java

@@ -0,0 +1,188 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static google.registry.backup.ExportCommitLogDiffAction.DIFF_FILE_PREFIX;
+import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static org.joda.time.Duration.standardHours;
+
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.Key;
+import com.google.appengine.tools.cloudstorage.GcsFileMetadata;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryConfig;
+import google.registry.model.server.Lock;
+import google.registry.model.translators.VKeyTranslatorFactory;
+import google.registry.persistence.VKey;
+import google.registry.request.Action;
+import google.registry.request.Response;
+import google.registry.request.auth.Auth;
+import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
+import google.registry.schema.replay.NonReplicatedEntity;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import google.registry.util.RequestStatusChecker;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.util.Optional;
+import javax.inject.Inject;
+import javax.servlet.http.HttpServletResponse;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/** Action that replays commit logs to Cloud SQL to keep it up to date. */
+@Action(
+    service = Action.Service.BACKEND,
+    path = ReplayCommitLogsToSqlAction.PATH,
+    method = Action.Method.POST,
+    automaticallyPrintOk = true,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public class ReplayCommitLogsToSqlAction implements Runnable {
+
+  static final String PATH = "/_dr/task/replayCommitLogsToSql";
+
+  private static final int BLOCK_SIZE =
+      1024 * 1024; // Buffer 1mb at a time, for no particular reason.
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final Duration LEASE_LENGTH = standardHours(1);
+
+  @Inject GcsService gcsService;
+  @Inject Response response;
+  @Inject RequestStatusChecker requestStatusChecker;
+  @Inject GcsDiffFileLister diffLister;
+
+  @Inject
+  ReplayCommitLogsToSqlAction() {}
+
+  @Override
+  public void run() {
+    if (!RegistryConfig.getCloudSqlReplayCommitLogs()) {
+      String message = "ReplayCommitLogsToSqlAction was called but disabled in the config.";
+      logger.atWarning().log(message);
+      // App Engine will retry on any non-2xx status code, which we don't want in this case.
+      response.setStatus(SC_NO_CONTENT);
+      response.setPayload(message);
+      return;
+    }
+    Optional<Lock> lock =
+        Lock.acquire(
+            this.getClass().getSimpleName(), null, LEASE_LENGTH, requestStatusChecker, false);
+    if (lock.isEmpty()) {
+      String message = "Can't acquire SQL commit log replay lock, aborting.";
+      logger.atSevere().log(message);
+      // App Engine will retry on any non-2xx status code, which we don't want in this case.
+      // Let the next run after the next export happen naturally.
+      response.setStatus(SC_NO_CONTENT);
+      response.setPayload(message);
+      return;
+    }
+    try {
+      replayFiles();
+      response.setStatus(HttpServletResponse.SC_OK);
+      logger.atInfo().log("ReplayCommitLogsToSqlAction completed successfully.");
+    } finally {
+      lock.ifPresent(Lock::release);
+    }
+  }
+
+  private void replayFiles() {
+    // Start at the first millisecond we haven't seen yet
+    DateTime fromTime = jpaTm().transact(() -> SqlReplayCheckpoint.get().plusMillis(1));
+    // If there's an inconsistent file set, this will throw IllegalStateException and the job
+    // will try later -- this is likely because an export hasn't finished yet.
+    ImmutableList<GcsFileMetadata> commitLogFiles =
+        diffLister.listDiffFiles(fromTime, /* current time */ null);
+    for (GcsFileMetadata metadata : commitLogFiles) {
+      // One transaction per GCS file
+      jpaTm().transact(() -> processFile(metadata));
+    }
+    logger.atInfo().log("Replayed %d commit log files to SQL successfully.", commitLogFiles.size());
+  }
+
+  private void processFile(GcsFileMetadata metadata) {
+    try (InputStream input =
+        Channels.newInputStream(
+            gcsService.openPrefetchingReadChannel(metadata.getFilename(), 0, BLOCK_SIZE))) {
+      // Load and process the Datastore transactions one at a time
+      ImmutableList<ImmutableList<VersionedEntity>> allTransactions =
+          CommitLogImports.loadEntitiesByTransaction(input);
+      allTransactions.forEach(this::replayTransaction);
+      // if we succeeded, set the last-seen time
+      DateTime checkpoint =
+          DateTime.parse(
+              metadata.getFilename().getObjectName().substring(DIFF_FILE_PREFIX.length()));
+      SqlReplayCheckpoint.set(checkpoint);
+      logger.atInfo().log("Replayed %d transactions from commit log file.", allTransactions.size());
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void replayTransaction(ImmutableList<VersionedEntity> transaction) {
+    transaction.stream()
+        .sorted(ReplayCommitLogsToSqlAction::compareByWeight)
+        .forEach(
+            versionedEntity ->
+                versionedEntity
+                    .getEntity()
+                    .ifPresentOrElse(
+                        this::handleEntityPut, () -> handleEntityDelete(versionedEntity)));
+  }
+
+  private void handleEntityPut(Entity entity) {
+    Object ofyPojo = ofy().toPojo(entity);
+    if (ofyPojo instanceof DatastoreEntity) {
+      DatastoreEntity datastoreEntity = (DatastoreEntity) ofyPojo;
+      datastoreEntity.toSqlEntity().ifPresent(jpaTm()::put);
+    } else {
+      // this should never happen, but we shouldn't fail on it
+      logger.atSevere().log(
+          "%s does not implement DatastoreEntity, which is necessary for SQL replay.",
+          ofyPojo.getClass());
+    }
+  }
+
+  private void handleEntityDelete(VersionedEntity entityToDelete) {
+    Key key = entityToDelete.key();
+    VKey<?> entityVKey;
+    try {
+      entityVKey = VKeyTranslatorFactory.createVKey(key);
+    } catch (RuntimeException e) {
+      // This means that the key wasn't convertible to VKey through the standard methods or via
+      // a createVKey method. This means that the object isn't persisted in SQL so we ignore it.
+      logger.atInfo().log(
+          "Skipping SQL delete for kind %s since it is not convertible.", key.getKind());
+      return;
+    }
+    Class<?> entityClass = entityVKey.getKind();
+    // Delete the key iff the class represents a JPA entity that is replicated
+    if (!NonReplicatedEntity.class.isAssignableFrom(entityClass)
+        && !DatastoreOnlyEntity.class.isAssignableFrom(entityClass)
+        && entityClass.getAnnotation(javax.persistence.Entity.class) != null) {
+      jpaTm().delete(entityVKey);
+    }
+  }
+
+  private static int compareByWeight(VersionedEntity a, VersionedEntity b) {
+    return getEntityPriority(a.key().getKind(), a.getEntity().isEmpty())
+        - getEntityPriority(b.key().getKind(), b.getEntity().isEmpty());
+  }
+}

core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java

@@ -50,11 +50,21 @@ public class ResaveAllEppResourcesAction implements Runnable {
   @Inject Response response;
   @Inject ResaveAllEppResourcesAction() {}
 
+  /**
+   * The number of shards to run the map-only mapreduce on.
+   *
+   * <p>This is less than the default of 100 because we only run this action monthly and can afford
+   * it being slower, but we don't want to write out lots of large commit logs in a short period of
+   * time because they make the Cloud SQL migration tougher.
+   */
+  private static final int NUM_SHARDS = 10;
+
   @Override
   public void run() {
     mrRunner
         .setJobName("Re-save all EPP resources")
         .setModuleName("backend")
+        .setDefaultMapShards(NUM_SHARDS)
         .runMapOnly(
             new ResaveAllEppResourcesActionMapper(),
             ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))

core/src/main/java/google/registry/beam/initsql/BeamJpaModule.java

@@ -31,6 +31,7 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.JdbcJpaTm;
 import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -168,6 +169,7 @@ public class BeamJpaModule {
         BeamJpaModule.class,
         KmsModule.class,
         PersistenceModule.class,
+        SecretManagerModule.class,
         UtilsModule.class
       })
   public interface JpaTransactionManagerComponent {

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Throwables.throwIfUnchecked;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.JpaRetries.isFailedTxnRetriable;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.setJpaTm;
@@ -68,6 +69,7 @@ import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -264,9 +266,9 @@ public final class Transforms {
   }
 
   /**
-   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
-   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
-   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   * Returns a {@link PTransform} that writes a {@link PCollection} of {@link VersionedEntity}s to a
+   * SQL database. and outputs an empty {@code PCollection<Void>}. This allows other operations to
+   * {@link org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
    *
    * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
    * job, we will not add features unless proven necessary.
@@ -282,16 +284,53 @@ public final class Transforms {
       int maxWriters,
       int batchSize,
       SerializableSupplier<JpaTransactionManager> jpaSupplier) {
-    return new PTransform<PCollection<VersionedEntity>, PCollection<Void>>() {
+    return writeToSql(
+        transformId,
+        maxWriters,
+        batchSize,
+        jpaSupplier,
+        (e) -> ofy().toPojo(e.getEntity().get()),
+        TypeDescriptor.of(VersionedEntity.class));
+  }
+
+  /**
+   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
+   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
+   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   *
+   * <p>The converter and type descriptor are generics so that we can convert any type of entity to
+   * an object to be placed in SQL.
+   *
+   * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
+   * job, we will not add features unless proven necessary.
+   *
+   * @param transformId a unique ID for an instance of the returned transform
+   * @param maxWriters the max number of concurrent writes to SQL, which also determines the max
+   *     number of connection pools created
+   * @param batchSize the number of entities to write in each operation
+   * @param jpaSupplier supplier of a {@link JpaTransactionManager}
+   * @param jpaConverter the function that converts the input object to a JPA entity
+   * @param objectDescriptor the type descriptor of the input object
+   */
+  public static <T> PTransform<PCollection<T>, PCollection<Void>> writeToSql(
+      String transformId,
+      int maxWriters,
+      int batchSize,
+      SerializableSupplier<JpaTransactionManager> jpaSupplier,
+      SerializableFunction<T, Object> jpaConverter,
+      TypeDescriptor<T> objectDescriptor) {
+    return new PTransform<PCollection<T>, PCollection<Void>>() {
       @Override
-      public PCollection<Void> expand(PCollection<VersionedEntity> input) {
+      public PCollection<Void> expand(PCollection<T> input) {
         return input
             .apply(
                 "Shard data for " + transformId,
-                MapElements.into(kvs(integers(), TypeDescriptor.of(VersionedEntity.class)))
+                MapElements.into(kvs(integers(), objectDescriptor))
                     .via(ve -> KV.of(ThreadLocalRandom.current().nextInt(maxWriters), ve)))
             .apply("Batch output by shard " + transformId, GroupIntoBatches.ofSize(batchSize))
-            .apply("Write in batch for " + transformId, ParDo.of(new SqlBatchWriter(jpaSupplier)));
+            .apply(
+                "Write in batch for " + transformId,
+                ParDo.of(new SqlBatchWriter<T>(jpaSupplier, jpaConverter)));
       }
     };
   }
@@ -385,18 +424,22 @@ public final class Transforms {
    * to hold the {@code JpaTransactionManager} instance, we must ensure that JpaTransactionManager
    * is not changed or torn down while being used by some instance.
    */
-  private static class SqlBatchWriter extends DoFn<KV<Integer, Iterable<VersionedEntity>>, Void> {
+  private static class SqlBatchWriter<T> extends DoFn<KV<Integer, Iterable<T>>, Void> {
 
     private static int instanceCount = 0;
     private static JpaTransactionManager originalJpa;
 
     private final SerializableSupplier<JpaTransactionManager> jpaSupplier;
+    private final SerializableFunction<T, Object> jpaConverter;
 
     private transient Ofy ofy;
     private transient SystemSleeper sleeper;
 
-    SqlBatchWriter(SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+    SqlBatchWriter(
+        SerializableSupplier<JpaTransactionManager> jpaSupplier,
+        SerializableFunction<T, Object> jpaConverter) {
       this.jpaSupplier = jpaSupplier;
+      this.jpaConverter = jpaConverter;
     }
 
     @Setup
@@ -429,13 +472,11 @@ public final class Transforms {
     }
 
     @ProcessElement
-    public void processElement(@Element KV<Integer, Iterable<VersionedEntity>> kv) {
+    public void processElement(@Element KV<Integer, Iterable<T>> kv) {
       try (AppEngineEnvironment env = new AppEngineEnvironment()) {
         ImmutableList<Object> ofyEntities =
             Streams.stream(kv.getValue())
-                .map(VersionedEntity::getEntity)
-                .map(Optional::get)
-                .map(ofy::toPojo)
+                .map(this.jpaConverter::apply)
                 .collect(ImmutableList.toImmutableList());
         retry(() -> jpaTm().transact(() -> jpaTm().putAll(ofyEntities)));
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -20,7 +20,7 @@ import static google.registry.beam.BeamUtils.getQueryFromFile;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
-import google.registry.backup.AppEngineEnvironment;
+import google.registry.beam.initsql.Transforms;
 import google.registry.beam.initsql.Transforms.SerializableSupplier;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
 import google.registry.config.CredentialModule.LocalCredential;
@@ -43,7 +43,6 @@ import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
-import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -191,34 +190,27 @@ public class Spec11Pipeline implements Serializable {
       PCollection<Subdomain> domains,
       EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
       ValueProvider<String> dateProvider) {
-
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsSql =
         domains.apply("Run through SafeBrowsing API", ParDo.of(evaluateSafeBrowsingFn));
-    /* Store ThreatMatch objects in SQL. */
+    TypeDescriptor<KV<Subdomain, ThreatMatch>> descriptor =
+        new TypeDescriptor<KV<Subdomain, ThreatMatch>>() {};
     subdomainsSql.apply(
-        ParDo.of(
-            new DoFn<KV<Subdomain, ThreatMatch>, Void>() {
-              @ProcessElement
-              public void processElement(ProcessContext context) {
-                // create the Spec11ThreatMatch from Subdomain and ThreatMatch
-                try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-                  Subdomain subdomain = context.element().getKey();
-                  Spec11ThreatMatch threatMatch =
-                      new Spec11ThreatMatch.Builder()
-                          .setThreatTypes(
-                              ImmutableSet.of(
-                                  ThreatType.valueOf(context.element().getValue().threatType())))
-                          .setCheckDate(
-                              LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
-                          .setDomainName(subdomain.domainName())
-                          .setDomainRepoId(subdomain.domainRepoId())
-                          .setRegistrarId(subdomain.registrarId())
-                          .build();
-                  JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
-                  jpaTransactionManager.transact(() -> jpaTransactionManager.insert(threatMatch));
-                }
-              }
-            }));
+        Transforms.writeToSql(
+            "Spec11ThreatMatch",
+            4,
+            4,
+            jpaSupplierFactory,
+            (kv) -> {
+              Subdomain subdomain = kv.getKey();
+              return new Spec11ThreatMatch.Builder()
+                  .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
+                  .setCheckDate(LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
+                  .setDomainName(subdomain.domainName())
+                  .setDomainRepoId(subdomain.domainRepoId())
+                  .setRegistrarId(subdomain.registrarId())
+                  .build();
+            },
+            descriptor));
 
     /* Store ThreatMatch objects in JSON. */
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -415,6 +415,14 @@ public final class RegistryConfig {
       return config.cloudSql.instanceConnectionName;
     }
 
+    @Provides
+    @Config("cloudSqlDbInstanceName")
+    public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
+      // Format of instanceConnectionName: project-id:region:instance-name
+      int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
+      return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);
+    }
+
     @Provides
     @Config("cloudDnsRootUrl")
     public static Optional<String> getCloudDnsRootUrl(RegistryConfigSettings config) {
@@ -1592,6 +1600,22 @@ public final class RegistryConfig {
     CONFIG_SETTINGS.get().cloudSql.replicateTransactions = replicateTransactions;
   }
 
+  /**
+   * Returns whether or not to replay commit logs to the SQL database after export to GCS.
+   *
+   * <p>If true, we will trigger the {@link google.registry.backup.ReplayCommitLogsToSqlAction}
+   * after the {@link google.registry.backup.ExportCommitLogDiffAction} to load the commit logs and
+   * replay them to SQL.
+   */
+  public static boolean getCloudSqlReplayCommitLogs() {
+    return CONFIG_SETTINGS.get().cloudSql.replayCommitLogs;
+  }
+
+  @VisibleForTesting
+  public static void overrideCloudSqlReplayCommitLogs(boolean replayCommitLogs) {
+    CONFIG_SETTINGS.get().cloudSql.replayCommitLogs = replayCommitLogs;
+  }
+
   /** Returns the roid suffix to be used for the roids of all contacts and hosts. */
   public static String getContactAndHostRoidSuffix() {
     return CONFIG_SETTINGS.get().registryPolicy.contactAndHostRoidSuffix;

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -126,6 +126,7 @@ public class RegistryConfigSettings {
     public String username;
     public String instanceConnectionName;
     public boolean replicateTransactions;
+    public boolean replayCommitLogs;
   }
 
   /** Configuration for Apache Beam (Cloud Dataflow). */

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -233,6 +233,8 @@ cloudSql:
   # Set this to true to replicate cloud SQL transactions to datastore in the
   # background.
   replicateTransactions: false
+  # Set this to true to enable replay of commit logs to SQL
+  replayCommitLogs: false
 
 cloudDns:
   # Set both properties to null in Production.

core/src/main/java/google/registry/env/common/default/WEB-INF/queue.xml

@@ -208,6 +208,12 @@
     <max-concurrent-requests>5</max-concurrent-requests>
   </queue>
 
+  <!-- Queue for replaying commit logs to SQL during the transition from Datastore -> SQL. -->
+  <queue>
+    <name>replay-commit-logs-to-sql</name>
+    <rate>1/s</rate>
+  </queue>
+
   <!-- The load[0-9] queues are used for load-testing, and can be safely deleted
        in any environment that doesn't require load-testing. -->
   <queue>

core/src/main/java/google/registry/flows/EppRequestHandler.java

@@ -74,6 +74,14 @@ public class EppRequestHandler {
           && eppOutput.getResponse().getResult().getCode() == SUCCESS_AND_CLOSE) {
         response.setHeader("Epp-Session", "close");
       }
+      // If a login request returns a success, a logged-in header is added to the response to inform
+      // the proxy that it is no longer necessary to send the full client certificate to the backend
+      // for this connection.
+      if (eppOutput.isResponse()
+          && eppOutput.getResponse().isLoginResponse()
+          && eppOutput.isSuccess()) {
+        response.setHeader("Logged-In", "true");
+      }
     } catch (Exception e) {
       logger.atWarning().withCause(e).log("handleEppCommand general exception");
       response.setStatus(SC_BAD_REQUEST);

core/src/main/java/google/registry/flows/session/LoginFlow.java

@@ -141,7 +141,7 @@ public class LoginFlow implements Flow {
     sessionMetadata.resetFailedLoginAttempts();
     sessionMetadata.setClientId(login.getClientId());
     sessionMetadata.setServiceExtensionUris(serviceExtensionUrisBuilder.build());
-    return responseBuilder.build();
+    return responseBuilder.setIsLoginResponse().build();
   }
 
   /** Registrar with this client ID could not be found. */

core/src/main/java/google/registry/model/DatabaseMigrationUtils.java

@@ -0,0 +1,38 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryEnvironment;
+
+/** Utility methods related to migrating dual-read/dual-write entities. */
+public class DatabaseMigrationUtils {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Throws exceptions only in unit tests, otherwise only logs exceptions. */
+  public static void suppressExceptionUnlessInTest(Runnable work, String message) {
+    try {
+      work.run();
+    } catch (Exception e) {
+      if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)) {
+        throw e;
+      }
+      logger.atWarning().withCause(e).log(message);
+    }
+  }
+
+  private DatabaseMigrationUtils() {}
+}

core/src/main/java/google/registry/model/ModelUtils.java

@@ -87,7 +87,7 @@ public class ModelUtils {
               });
 
   /** Lists all instance fields on an object, including non-public and inherited fields. */
-  static Map<String, Field> getAllFields(Class<?> clazz) {
+  public static Map<String, Field> getAllFields(Class<?> clazz) {
     return ALL_FIELDS_CACHE.getUnchecked(clazz);
   }
 

core/src/main/java/google/registry/model/OteAccountBuilder.java

@@ -211,11 +211,6 @@ public final class OteAccountBuilder {
     return transformRegistrars(builder -> builder.setPassword(password));
   }
 
-  /** Sets the client certificate hash to all the OT&E Registrars. */
-  public OteAccountBuilder setCertificateHash(String certHash) {
-    return transformRegistrars(builder -> builder.setClientCertificateHash(certHash));
-  }
-
   /** Sets the client certificate to all the OT&E Registrars. */
   public OteAccountBuilder setCertificate(String asciiCert, DateTime now) {
     return transformRegistrars(builder -> builder.setClientCertificate(asciiCert, now));

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -28,6 +28,11 @@ import org.joda.time.DateTime;
  */
 public class UpdateAutoTimestamp extends ImmutableObject {
 
+  // When set to true, database converters/translators should do tha auto update.  When set to
+  // false, auto update should be suspended (this exists to allow us to preserve the original value
+  // during a replay).
+  static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
+
   DateTime timestamp;
 
   /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
@@ -40,4 +45,30 @@ public class UpdateAutoTimestamp extends ImmutableObject {
     instance.timestamp = timestamp;
     return instance;
   }
+
+  // TODO(b/175610935): Remove the auto-update disabling code below after migration.
+
+  /** Class to allow us to safely disable auto-update in a try-with-resources block. */
+  public static class DisableAutoUpdateResource implements AutoCloseable {
+    DisableAutoUpdateResource() {
+      autoUpdateEnabled.set(false);
+    }
+
+    @Override
+    public void close() {
+      autoUpdateEnabled.set(true);
+    }
+  }
+
+  /**
+   * Resturns a resource that disables auto-updates on all {@link UpdateAutoTimestamp}s in the
+   * current thread, suitable for use with in a try-with-resources block.
+   */
+  public static DisableAutoUpdateResource disableAutoUpdate() {
+    return new DisableAutoUpdateResource();
+  }
+
+  public static boolean autoUpdateEnabled() {
+    return autoUpdateEnabled.get();
+  }
 }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -24,7 +24,6 @@ import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
@@ -50,8 +49,7 @@ import google.registry.model.transfer.TransferData.TransferServerApproveEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -292,7 +290,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "allocationToken")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
@@ -466,7 +464,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {
 
     /**
@@ -561,7 +559,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "billingTime")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billing time of the charge that is being cancelled. */
@@ -682,8 +680,8 @@ public abstract class BillingEvent extends ImmutableObject
   /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
-  @WithLongVKey
-  public static class Modification extends BillingEvent implements DatastoreEntity {
+  @WithLongVKey(compositeKey = true)
+  public static class Modification extends BillingEvent implements DatastoreOnlyEntity {
 
     /** The change in cost that should be applied to the original billing event. */
     Money cost;
@@ -745,11 +743,6 @@ public abstract class BillingEvent extends ImmutableObject
           .build();
     }
 
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of(); // not persisted in SQL
-    }
-
     /** A builder for {@link Modification} since it is immutable. */
     public static class Builder extends BillingEvent.Builder<Modification, Builder> {
 

core/src/main/java/google/registry/model/common/Cursor.java

@@ -21,7 +21,6 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.base.Splitter;
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -29,8 +28,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.registry.Registry;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import java.util.List;
 import org.joda.time.DateTime;
 
@@ -40,7 +38,7 @@ import org.joda.time.DateTime;
  * scoped on {@link EntityGroupRoot}.
  */
 @Entity
-public class Cursor extends ImmutableObject implements DatastoreEntity {
+public class Cursor extends ImmutableObject implements DatastoreOnlyEntity {
 
   /** The types of cursors, used as the string id field for each cursor in Datastore. */
   public enum CursorType {
@@ -137,11 +135,6 @@ public class Cursor extends ImmutableObject implements DatastoreEntity {
     return CursorType.valueOf(String.join("_", id.subList(1, id.size())));
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // Cursors are not converted since they are ephemeral
-  }
-
   /**
    * Checks that the type of the scoped object (or null) matches the required type for the specified
    * cursor (or null, if the cursor is a global cursor).

core/src/main/java/google/registry/model/common/EntityGroupRoot.java

@@ -14,13 +14,11 @@
 
 package google.registry.model.common;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.BackupGroupRoot;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 
 /**
  * The root key for the entity group which is known as the cross-tld entity group for historical
@@ -37,7 +35,7 @@ import google.registry.schema.replay.SqlEntity;
  * entity group for the single namespace where global data applicable for all TLDs lived.
  */
 @Entity
-public class EntityGroupRoot extends BackupGroupRoot implements DatastoreEntity {
+public class EntityGroupRoot extends BackupGroupRoot implements DatastoreOnlyEntity {
 
   @SuppressWarnings("unused")
   @Id
@@ -47,9 +45,4 @@ public class EntityGroupRoot extends BackupGroupRoot implements DatastoreEntity
   public static Key<EntityGroupRoot> getCrossTldKey() {
     return Key.create(EntityGroupRoot.class, "cross-tld");
   }
-
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not persisted in SQL
-  }
 }

core/src/main/java/google/registry/model/contact/ContactHistory.java

@@ -14,7 +14,6 @@
 
 package google.registry.model.contact;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -95,9 +94,9 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
   }
 
   /** Creates a {@link VKey} instance for this entity. */
+  @SuppressWarnings("unchecked")
   public VKey<ContactHistory> createVKey() {
-    return VKey.create(
-        ContactHistory.class, new ContactHistoryId(getContactRepoId(), getId()), Key.create(this));
+    return (VKey<ContactHistory>) createVKey(Key.create(this));
   }
 
   @PostLoad
@@ -111,12 +110,12 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
 
   // In Datastore, save as a HistoryEntry object regardless of this object's type
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(asHistoryEntry());
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(asHistoryEntry());
   }
 
   /** Class to represent the composite primary key of {@link ContactHistory} entity. */
-  static class ContactHistoryId extends ImmutableObject implements Serializable {
+  public static class ContactHistoryId extends ImmutableObject implements Serializable {
 
     private String contactRepoId;
 
@@ -125,7 +124,7 @@ public class ContactHistory extends HistoryEntry implements SqlEntity {
     /** Hibernate requires this default constructor. */
     private ContactHistoryId() {}
 
-    ContactHistoryId(String contactRepoId, long id) {
+    public ContactHistoryId(String contactRepoId, long id) {
       this.contactRepoId = contactRepoId;
       this.id = id;
     }

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -17,7 +17,6 @@ package google.registry.model.domain;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
@@ -101,6 +100,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   @Column(name = "host_repo_id")
   Set<VKey<HostResource>> nsHosts;
 
+  @Ignore
   @OneToMany(
       cascade = {CascadeType.ALL},
       fetch = FetchType.EAGER,
@@ -117,8 +117,9 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
         insertable = false,
         updatable = false)
   })
-  Set<DomainDsDataHistory> dsDataHistories;
+  Set<DomainDsDataHistory> dsDataHistories = ImmutableSet.of();
 
+  @Ignore
   @OneToMany(
       cascade = {CascadeType.ALL},
       fetch = FetchType.EAGER,
@@ -135,18 +136,14 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
         insertable = false,
         updatable = false)
   })
-  Set<GracePeriodHistory> gracePeriodHistories;
+  Set<GracePeriodHistory> gracePeriodHistories = ImmutableSet.of();
 
   @Override
   @Nullable
   @Access(AccessType.PROPERTY)
   @AttributeOverrides({
-      @AttributeOverride(
-          name = "unit",
-          column = @Column(name = "historyPeriodUnit")),
-      @AttributeOverride(
-          name = "value",
-          column = @Column(name = "historyPeriodValue"))
+    @AttributeOverride(name = "unit", column = @Column(name = "historyPeriodUnit")),
+    @AttributeOverride(name = "value", column = @Column(name = "historyPeriodValue"))
   })
   public Period getPeriod() {
     return super.getPeriod();
@@ -229,9 +226,9 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
   }
 
   /** Creates a {@link VKey} instance for this entity. */
+  @SuppressWarnings("unchecked")
   public VKey<DomainHistory> createVKey() {
-    return VKey.create(
-        DomainHistory.class, new DomainHistoryId(getDomainRepoId(), getId()), Key.create(this));
+    return (VKey<DomainHistory>) createVKey(Key.create(this));
   }
 
   @PostLoad
@@ -252,8 +249,8 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
 
   // In Datastore, save as a HistoryEntry object regardless of this object's type
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(asHistoryEntry());
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(asHistoryEntry());
   }
 
   /** Class to represent the composite primary key of {@link DomainHistory} entity. */

core/src/main/java/google/registry/model/domain/secdns/DomainDsDataHistory.java

@@ -14,11 +14,11 @@
 
 package google.registry.model.domain.secdns;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.ofy.ObjectifyService;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import java.util.Optional;
 import javax.persistence.Access;
 import javax.persistence.AccessType;
 import javax.persistence.Column;
@@ -86,7 +86,7 @@ public class DomainDsDataHistory extends DomainDsDataBase implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not persisted in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore
   }
 }

core/src/main/java/google/registry/model/eppoutput/EppResponse.java

@@ -65,6 +65,7 @@ import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlElementRef;
 import javax.xml.bind.annotation.XmlElementRefs;
 import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.XmlType;
 
 /**
@@ -87,6 +88,9 @@ public class EppResponse extends ImmutableObject implements ResponseOrGreeting {
   /** The command result. The RFC allows multiple failure results, but we always return one. */
   Result result;
 
+  /** Indicates if this response is for a login request. */
+  @XmlTransient boolean isLoginResponse = false;
+
   /**
    * Information about messages queued for retrieval. This may appear in response to any EPP message
    * (if messages are queued), but in practice this will only be set in response to a poll request.
@@ -178,6 +182,10 @@ public class EppResponse extends ImmutableObject implements ResponseOrGreeting {
     return result;
   }
 
+  public boolean isLoginResponse() {
+    return isLoginResponse;
+  }
+
   /** Marker interface for types that can go in the {@link #resData} field. */
   public interface ResponseData {}
 
@@ -222,5 +230,10 @@ public class EppResponse extends ImmutableObject implements ResponseOrGreeting {
       getInstance().extensions = forceEmptyToNull(extensions);
       return this;
     }
+
+    public Builder setIsLoginResponse() {
+      getInstance().isLoginResponse = true;
+      return this;
+    }
   }
 }

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -14,7 +14,6 @@
 
 package google.registry.model.host;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EntitySubclass;
 import google.registry.model.ImmutableObject;
@@ -96,9 +95,9 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
   }
 
   /** Creates a {@link VKey} instance for this entity. */
+  @SuppressWarnings("unchecked")
   public VKey<HostHistory> createVKey() {
-    return VKey.create(
-        HostHistory.class, new HostHistoryId(getHostRepoId(), getId()), Key.create(this));
+    return (VKey<HostHistory>) createVKey(Key.create(this));
   }
 
   @PostLoad
@@ -112,12 +111,12 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
 
   // In Datastore, save as a HistoryEntry object regardless of this object's type
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(asHistoryEntry());
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(asHistoryEntry());
   }
 
   /** Class to represent the composite primary key of {@link HostHistory} entity. */
-  static class HostHistoryId extends ImmutableObject implements Serializable {
+  public static class HostHistoryId extends ImmutableObject implements Serializable {
 
     private String hostRepoId;
 
@@ -126,7 +125,7 @@ public class HostHistory extends HistoryEntry implements SqlEntity {
     /** Hibernate requires this default constructor. */
     private HostHistoryId() {}
 
-    HostHistoryId(String hostRepoId, long id) {
+    public HostHistoryId(String hostRepoId, long id) {
       this.hostRepoId = hostRepoId;
       this.id = id;
     }

core/src/main/java/google/registry/model/index/EppResourceIndex.java

@@ -17,7 +17,6 @@ package google.registry.model.index;
 import static google.registry.util.TypeUtils.instantiate;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -26,25 +25,21 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.BackupGroupRoot;
 import google.registry.model.EppResource;
 import google.registry.model.annotations.ReportedOn;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 
 /** An index that allows for quick enumeration of all EppResource entities (e.g. via map reduce). */
 @ReportedOn
 @Entity
-public class EppResourceIndex extends BackupGroupRoot implements DatastoreEntity {
+public class EppResourceIndex extends BackupGroupRoot implements DatastoreOnlyEntity {
 
-  @Id
-  String id;
+  @Id String id;
 
-  @Parent
-  Key<EppResourceIndexBucket> bucket;
+  @Parent Key<EppResourceIndexBucket> bucket;
 
   /** Although this field holds a {@link Key} it is named "reference" for historical reasons. */
   Key<? extends EppResource> reference;
 
-  @Index
-  String kind;
+  @Index String kind;
 
   public String getId() {
     return id;
@@ -69,7 +64,7 @@ public class EppResourceIndex extends BackupGroupRoot implements DatastoreEntity
     EppResourceIndex instance = instantiate(EppResourceIndex.class);
     instance.reference = resourceKey;
     instance.kind = resourceKey.getKind();
-    instance.id = resourceKey.getString();  // creates a web-safe key string
+    instance.id = resourceKey.getString(); // creates a web-safe key string
     instance.bucket = bucket;
     return instance;
   }
@@ -77,9 +72,4 @@ public class EppResourceIndex extends BackupGroupRoot implements DatastoreEntity
   public static <T extends EppResource> EppResourceIndex create(Key<T> resourceKey) {
     return create(EppResourceIndexBucket.getBucketKey(resourceKey), resourceKey);
   }
-
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not relevant in SQL
-  }
 }

core/src/main/java/google/registry/model/index/EppResourceIndexBucket.java

@@ -24,23 +24,17 @@ import com.googlecode.objectify.annotation.Id;
 import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.VirtualEntity;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 
 /** A virtual entity to represent buckets to which EppResourceIndex objects are randomly added. */
 @Entity
 @VirtualEntity
-public class EppResourceIndexBucket extends ImmutableObject implements DatastoreEntity {
+public class EppResourceIndexBucket extends ImmutableObject implements DatastoreOnlyEntity {
 
   @SuppressWarnings("unused")
   @Id
   private long bucketId;
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not relevant in SQL
-  }
-
   /**
    * Deterministic function that returns a bucket id based on the resource's roid.
    * NB: At the moment, nothing depends on this being deterministic, so we have the ability to

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -42,8 +42,7 @@ import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.host.HostResource;
 import google.registry.persistence.VKey;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.NonFinalForTesting;
 import java.util.Map;
 import java.util.Optional;
@@ -63,34 +62,19 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
   @ReportedOn
   @Entity
   public static class ForeignKeyContactIndex extends ForeignKeyIndex<ContactResource>
-      implements DatastoreEntity {
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of(); // not relevant in SQL
-    }
-  }
+      implements DatastoreOnlyEntity {}
 
   /** The {@link ForeignKeyIndex} type for {@link DomainBase} entities. */
   @ReportedOn
   @Entity
   public static class ForeignKeyDomainIndex extends ForeignKeyIndex<DomainBase>
-      implements DatastoreEntity {
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of(); // not relevant in SQL
-    }
-  }
+      implements DatastoreOnlyEntity {}
 
   /** The {@link ForeignKeyIndex} type for {@link HostResource} entities. */
   @ReportedOn
   @Entity
   public static class ForeignKeyHostIndex extends ForeignKeyIndex<HostResource>
-      implements DatastoreEntity {
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of(); // not relevant in SQL
-    }
-  }
+      implements DatastoreOnlyEntity {}
 
   static final ImmutableMap<Class<? extends EppResource>, Class<? extends ForeignKeyIndex<?>>>
       RESOURCE_CLASS_TO_FKI_CLASS =

core/src/main/java/google/registry/model/ofy/CommitLogBucket.java

@@ -22,7 +22,6 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.collect.ContiguousSet;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Range;
@@ -34,8 +33,7 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.NonFinalForTesting;
 import java.util.Random;
 import java.util.function.Supplier;
@@ -53,7 +51,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogBucket extends ImmutableObject implements Buildable, DatastoreEntity {
+public class CommitLogBucket extends ImmutableObject implements Buildable, DatastoreOnlyEntity {
 
   /**
    * Ranges from 1 to {@link RegistryConfig#getCommitLogBucketCount()}, inclusive; starts at 1 since
@@ -72,11 +70,6 @@ public class CommitLogBucket extends ImmutableObject implements Buildable, Datas
     return lastWrittenTime;
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not persisted in SQL
-  }
-
   /**
    * Returns the key for the specified bucket ID.
    *

core/src/main/java/google/registry/model/ofy/CommitLogCheckpoint.java

@@ -27,8 +27,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -46,7 +45,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogCheckpoint extends ImmutableObject implements DatastoreEntity {
+public class CommitLogCheckpoint extends ImmutableObject implements DatastoreOnlyEntity {
 
   /** Shared singleton parent entity for commit log checkpoints. */
   @Parent
@@ -73,11 +72,6 @@ public class CommitLogCheckpoint extends ImmutableObject implements DatastoreEnt
     return builder.build();
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not persisted in SQL
-  }
-
   /**
    * Creates a CommitLogCheckpoint for the given wall time and bucket checkpoint times, specified as
    * a map from bucket ID to bucket commit timestamp.

core/src/main/java/google/registry/model/ofy/CommitLogCheckpointRoot.java

@@ -17,21 +17,19 @@ package google.registry.model.ofy;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import org.joda.time.DateTime;
 
 /** Singleton parent entity for all commit log checkpoints. */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreEntity {
+public class CommitLogCheckpointRoot extends ImmutableObject implements DatastoreOnlyEntity {
 
   public static final long SINGLETON_ID = 1;  // There is always exactly one of these.
 
@@ -50,11 +48,6 @@ public class CommitLogCheckpointRoot extends ImmutableObject implements Datastor
     return lastWrittenTime;
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not persisted in SQL
-  }
-
   public static CommitLogCheckpointRoot loadRoot() {
     CommitLogCheckpointRoot root = ofy().load().key(getKey()).now();
     return root == null ? new CommitLogCheckpointRoot() : root;

core/src/main/java/google/registry/model/ofy/CommitLogManifest.java

@@ -17,7 +17,6 @@ package google.registry.model.ofy;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static org.joda.time.DateTimeZone.UTC;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -26,8 +25,7 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import java.util.LinkedHashSet;
 import java.util.Set;
 import org.joda.time.DateTime;
@@ -41,7 +39,7 @@ import org.joda.time.DateTime;
  */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogManifest extends ImmutableObject implements DatastoreEntity {
+public class CommitLogManifest extends ImmutableObject implements DatastoreOnlyEntity {
 
   /** Commit log manifests are parented on a random bucket. */
   @Parent
@@ -70,11 +68,6 @@ public class CommitLogManifest extends ImmutableObject implements DatastoreEntit
     return nullToEmptyImmutableCopy(deletions);
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not persisted in SQL
-  }
-
   public static CommitLogManifest create(
       Key<CommitLogBucket> parent, DateTime commitTime, Set<Key<?>> deletions) {
     CommitLogManifest instance = new CommitLogManifest();

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -21,7 +21,6 @@ import static google.registry.model.ofy.ObjectifyService.ofy;
 
 import com.google.appengine.api.datastore.KeyFactory;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -29,13 +28,12 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 
 /** Representation of a saved entity in a {@link CommitLogManifest} (not deletes). */
 @Entity
 @NotBackedUp(reason = Reason.COMMIT_LOGS)
-public class CommitLogMutation extends ImmutableObject implements DatastoreEntity {
+public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyEntity {
 
   /** The manifest this belongs to. */
   @Parent
@@ -61,11 +59,6 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreEntit
     return createFromPbBytes(entityProtoBytes);
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not persisted in SQL
-  }
-
   /**
    * Returns a new mutation entity created from an @Entity ImmutableObject instance.
    *

core/src/main/java/google/registry/model/ofy/EntityWritePriorities.java

@@ -0,0 +1,60 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.ofy;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * Contains the mapping from class names to SQL-replay-write priorities.
+ *
+ * <p>When replaying Datastore commit logs to SQL (asynchronous replication), in order to avoid
+ * issues with foreign keys, we should replay entity writes so that foreign key references are
+ * always written after the entity that they reference. This class represents that DAG, where lower
+ * values represent an earlier write (and later delete). Higher-valued classes can have foreign keys
+ * on lower-valued classes, but not vice versa.
+ */
+public class EntityWritePriorities {
+
+  /**
+   * Mapping from class name to "priority".
+   *
+   * <p>Here, "priority" means the order in which the class should be inserted / updated in a
+   * transaction with respect to instances of other classes. By default, all classes have a priority
+   * number of zero.
+   *
+   * <p>For each transaction, classes should be written in priority order from the lowest number to
+   * the highest, in order to maintain foreign-key write consistency. For the same reason, deletes
+   * should happen after all writes.
+   */
+  static final ImmutableMap<String, Integer> CLASS_PRIORITIES =
+      ImmutableMap.of(
+          "HistoryEntry", -10,
+          "AllocationToken", -9,
+          "ContactResource", 5,
+          "DomainBase", 10);
+
+  // The beginning of the range of priority numbers reserved for delete.  This must be greater than
+  // any of the values in CLASS_PRIORITIES by enough overhead to accommodate  any negative values in
+  // it. Note: by design, deletions will happen in the opposite order of insertions, which is
+  // necessary to make sure foreign keys aren't violated during deletion.
+  @VisibleForTesting static final int DELETE_RANGE = Integer.MAX_VALUE / 2;
+
+  /** Returns the priority of the entity type in the map entry. */
+  public static int getEntityPriority(String kind, boolean isDelete) {
+    int priority = CLASS_PRIORITIES.getOrDefault(kind, 0);
+    return isDelete ? DELETE_RANGE - priority : priority;
+  }
+}

core/src/main/java/google/registry/model/ofy/TransactionInfo.java

@@ -20,6 +20,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Maps.filterValues;
 import static com.google.common.collect.Maps.toMap;
 import static google.registry.model.ofy.CommitLogBucket.getArbitraryBucketId;
+import static google.registry.model.ofy.EntityWritePriorities.getEntityPriority;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
@@ -29,7 +30,6 @@ import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
 import java.util.Map;
 import org.joda.time.DateTime;
 
@@ -61,7 +61,7 @@ class TransactionInfo {
 
   TransactionInfo(DateTime now) {
     this.transactionTime = now;
-    ofy().load().key(bucketKey);  // Asynchronously load value into session cache.
+    ofy().load().key(bucketKey); // Asynchronously load value into session cache.
   }
 
   TransactionInfo setReadOnly() {
@@ -101,23 +101,10 @@ class TransactionInfo {
         .collect(toImmutableSet());
   }
 
-  // Mapping from class name to "weight" (which in this case is the order in which the class must
-  // be "put" in a transaction with respect to instances of other classes).  Lower weight classes
-  // are put first, by default all classes have a weight of zero.
-  static final ImmutableMap<String, Integer> CLASS_WEIGHTS =
-      ImmutableMap.of(
-          "HistoryEntry", -1,
-          "DomainBase", 1);
-
-  // The beginning of the range of weights reserved for delete.  This must be greater than any of
-  // the values in CLASS_WEIGHTS by enough overhead to accomodate any negative values in it.
-  @VisibleForTesting static final int DELETE_RANGE = Integer.MAX_VALUE / 2;
-
   /** Returns the weight of the entity type in the map entry. */
   @VisibleForTesting
   static int getWeight(ImmutableMap.Entry<Key<?>, Object> entry) {
-    int weight = CLASS_WEIGHTS.getOrDefault(entry.getKey().getKind(), 0);
-    return entry.getValue().equals(Delete.SENTINEL) ? DELETE_RANGE - weight : weight;
+    return getEntityPriority(entry.getKey().getKind(), entry.getValue().equals(Delete.SENTINEL));
   }
 
   private static int compareByWeight(
@@ -137,10 +124,9 @@ class TransactionInfo {
                         if (entry.getValue().equals(Delete.SENTINEL)) {
                           jpaTm().delete(VKey.from(entry.getKey()));
                         } else {
-                          for (SqlEntity entity :
-                              ((DatastoreEntity) entry.getValue()).toSqlEntities()) {
-                            jpaTm().put(entity);
-                          }
+                          ((DatastoreEntity) entry.getValue())
+                              .toSqlEntity()
+                              .ifPresent(jpaTm()::put);
                         }
                       });
             });

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -279,7 +279,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("ONE_TIME")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends PollMessage {
 
     // Response data. Objectify cannot persist a base class type, so we must have a separate field
@@ -432,7 +432,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("AUTORENEW")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */

core/src/main/java/google/registry/model/registrar/Registrar.java

@@ -121,33 +121,34 @@ public class Registrar extends ImmutableObject
     REAL(Objects::nonNull),
 
     /**
-     * A registrar account used by a real third-party registrar undergoing operational testing
-     * and evaluation. Should only be created in sandbox, and should have null IANA/billing IDs.
+     * A registrar account used by a real third-party registrar undergoing operational testing and
+     * evaluation. Should only be created in sandbox, and should have null IANA/billing IDs.
      */
     OTE(Objects::isNull),
 
     /**
-     * A registrar used for predelegation testing.  Should have a null billing ID.  The IANA ID
-     * should be either 9995 or 9996, which are reserved for predelegation testing.
+     * A registrar used for predelegation testing. Should have a null billing ID. The IANA ID should
+     * be either 9995 or 9996, which are reserved for predelegation testing.
      */
     PDT(n -> ImmutableSet.of(9995L, 9996L).contains(n)),
 
     /**
-     * A registrar used for external monitoring by ICANN.  Should have IANA ID 9997 and a null
+     * A registrar used for external monitoring by ICANN. Should have IANA ID 9997 and a null
      * billing ID.
      */
     EXTERNAL_MONITORING(isEqual(9997L)),
 
     /**
-     * A registrar used for when the registry acts as a registrar.  Must have either IANA ID
-     * 9998 (for billable transactions) or 9999 (for non-billable transactions). */
+     * A registrar used for when the registry acts as a registrar. Must have either IANA ID 9998
+     * (for billable transactions) or 9999 (for non-billable transactions).
+     */
     // TODO(b/13786188): determine what billing ID for this should be, if any.
     INTERNAL(n -> ImmutableSet.of(9998L, 9999L).contains(n)),
 
-    /** A registrar used for internal monitoring.  Should have null IANA/billing IDs. */
+    /** A registrar used for internal monitoring. Should have null IANA/billing IDs. */
     MONITORING(Objects::isNull),
 
-    /** A registrar used for internal testing.  Should have null IANA/billing IDs. */
+    /** A registrar used for internal testing. Should have null IANA/billing IDs. */
     TEST(Objects::isNull);
 
     /**
@@ -225,10 +226,7 @@ public class Registrar extends ImmutableObject
    */
   private static final Supplier<ImmutableMap<String, Registrar>> CACHE_BY_CLIENT_ID =
       memoizeWithShortExpiration(
-          () ->
-              tm()
-                  .doTransactionless(
-                      () -> Maps.uniqueIndex(loadAll(), Registrar::getClientId)));
+          () -> tm().doTransactionless(() -> Maps.uniqueIndex(loadAll(), Registrar::getClientId)));
 
   @Parent @Transient Key<EntityGroupRoot> parent = getCrossTldKey();
 
@@ -383,12 +381,10 @@ public class Registrar extends ImmutableObject
   @Index @Nullable Long ianaIdentifier;
 
   /** Identifier of registrar used in external billing system (e.g. Oracle). */
-  @Nullable
-  Long billingIdentifier;
+  @Nullable Long billingIdentifier;
 
   /** Purchase Order number used for invoices in external billing system, if applicable. */
-  @Nullable
-  String poNumber;
+  @Nullable String poNumber;
 
   /**
    * Map of currency-to-billing account for the registrar.
@@ -498,9 +494,7 @@ public class Registrar extends ImmutableObject
     if (billingAccountMap == null) {
       return ImmutableMap.of();
     }
-    return billingAccountMap
-        .entrySet()
-        .stream()
+    return billingAccountMap.entrySet().stream()
         .collect(toImmutableSortedMap(natural(), Map.Entry::getKey, v -> v.getValue().accountId));
   }
 
@@ -722,14 +716,18 @@ public class Registrar extends ImmutableObject
 
   /** Creates a {@link VKey} for this instance. */
   public VKey<Registrar> createVKey() {
-    return VKey.create(Registrar.class, clientIdentifier, Key.create(this));
+    return createVKey(Key.create(this));
   }
 
   /** Creates a {@link VKey} for the given {@code registrarId}. */
   public static VKey<Registrar> createVKey(String registrarId) {
     checkArgumentNotNull(registrarId, "registrarId must be specified");
-    return VKey.create(
-        Registrar.class, registrarId, Key.create(getCrossTldKey(), Registrar.class, registrarId));
+    return createVKey(Key.create(getCrossTldKey(), Registrar.class, registrarId));
+  }
+
+  /** Creates a {@link VKey} instance from a {@link Key} instance. */
+  public static VKey<Registrar> createVKey(Key<Registrar> key) {
+    return VKey.create(Registrar.class, key.getName(), key);
   }
 
   /** A builder for constructing {@link Registrar}, since it is immutable. */
@@ -744,21 +742,22 @@ public class Registrar extends ImmutableObject
       // Client id must be [3,16] chars long. See "clIDType" in the base EPP schema of RFC 5730.
       // (Need to validate this here as there's no matching EPP XSD for validation.)
       checkArgument(
-          Range.closed(3,  16).contains(clientId.length()),
+          Range.closed(3, 16).contains(clientId.length()),
           "Client identifier must be 3-16 characters long.");
       getInstance().clientIdentifier = clientId;
       return this;
     }
 
     public Builder setIanaIdentifier(@Nullable Long ianaIdentifier) {
-      checkArgument(ianaIdentifier == null || ianaIdentifier > 0,
-          "IANA ID must be a positive number");
+      checkArgument(
+          ianaIdentifier == null || ianaIdentifier > 0, "IANA ID must be a positive number");
       getInstance().ianaIdentifier = ianaIdentifier;
       return this;
     }
 
     public Builder setBillingIdentifier(@Nullable Long billingIdentifier) {
-      checkArgument(billingIdentifier == null || billingIdentifier > 0,
+      checkArgument(
+          billingIdentifier == null || billingIdentifier > 0,
           "Billing ID must be a positive number");
       getInstance().billingIdentifier = billingIdentifier;
       return this;
@@ -774,9 +773,7 @@ public class Registrar extends ImmutableObject
         getInstance().billingAccountMap = null;
       } else {
         getInstance().billingAccountMap =
-            billingAccountMap
-                .entrySet()
-                .stream()
+            billingAccountMap.entrySet().stream()
                 .collect(toImmutableMap(Map.Entry::getKey, BillingAccountEntry::new));
       }
       return this;
@@ -859,26 +856,6 @@ public class Registrar extends ImmutableObject
       }
     }
 
-    /**
-     * Sets client certificate hash, but not the certificate.
-     *
-     * <p><b>Warning:</b> {@link #setClientCertificate(String, DateTime)} sets the hash for you and
-     * is preferred. Calling this method will nullify the {@code clientCertificate} field.
-     */
-    public Builder setClientCertificateHash(String clientCertificateHash) {
-      if (clientCertificateHash != null) {
-        checkArgument(
-            Pattern.matches("[A-Za-z0-9+/]+", clientCertificateHash),
-            "--cert_hash not a valid base64 (no padding) value");
-        checkArgument(
-            base64().decode(clientCertificateHash).length == 256 / 8,
-            "--cert_hash base64 does not decode to 256 bits");
-      }
-      getInstance().clientCertificate = null;
-      getInstance().clientCertificateHash = clientCertificateHash;
-      return this;
-    }
-
     public Builder setContactsRequireSyncing(boolean contactsRequireSyncing) {
       getInstance().contactsRequireSyncing = contactsRequireSyncing;
       return this;
@@ -900,16 +877,12 @@ public class Registrar extends ImmutableObject
     }
 
     public Builder setPhoneNumber(String phoneNumber) {
-      getInstance().phoneNumber = (phoneNumber == null)
-          ? null
-          : checkValidPhoneNumber(phoneNumber);
+      getInstance().phoneNumber = (phoneNumber == null) ? null : checkValidPhoneNumber(phoneNumber);
       return this;
     }
 
     public Builder setFaxNumber(String faxNumber) {
-      getInstance().faxNumber = (faxNumber == null)
-          ? null
-          : checkValidPhoneNumber(faxNumber);
+      getInstance().faxNumber = (faxNumber == null) ? null : checkValidPhoneNumber(faxNumber);
       return this;
     }
 
@@ -944,7 +917,8 @@ public class Registrar extends ImmutableObject
     }
 
     public Builder setDriveFolderId(@Nullable String driveFolderId) {
-      checkArgument(driveFolderId == null || !driveFolderId.contains("/"),
+      checkArgument(
+          driveFolderId == null || !driveFolderId.contains("/"),
           "Drive folder ID must not be a full URL");
       getInstance().driveFolderId = driveFolderId;
       return this;
@@ -962,9 +936,10 @@ public class Registrar extends ImmutableObject
 
     /** @throws IllegalArgumentException if provided passcode is not 5-digit numeric */
     public Builder setPhonePasscode(String phonePasscode) {
-      checkArgument(phonePasscode == null
-          || PHONE_PASSCODE_PATTERN.matcher(phonePasscode).matches(),
-          "Not a valid telephone passcode (must be 5 digits long): %s", phonePasscode);
+      checkArgument(
+          phonePasscode == null || PHONE_PASSCODE_PATTERN.matcher(phonePasscode).matches(),
+          "Not a valid telephone passcode (must be 5 digits long): %s",
+          phonePasscode);
       getInstance().phonePasscode = phonePasscode;
       return this;
     }
@@ -982,9 +957,11 @@ public class Registrar extends ImmutableObject
       checkArgument(
           getInstance().localizedAddress != null || getInstance().internationalizedAddress != null,
           "Must specify at least one of localized or internationalized address");
-      checkArgument(getInstance().type.isValidIanaId(getInstance().ianaIdentifier),
-          String.format("Supplied IANA ID is not valid for %s registrar type: %s",
-            getInstance().type, getInstance().ianaIdentifier));
+      checkArgument(
+          getInstance().type.isValidIanaId(getInstance().ianaIdentifier),
+          String.format(
+              "Supplied IANA ID is not valid for %s registrar type: %s",
+              getInstance().type, getInstance().ianaIdentifier));
       return cloneEmptyToNull(super.build());
     }
   }

core/src/main/java/google/registry/model/registrar/RegistrarContact.java

@@ -119,9 +119,7 @@ public class RegistrarContact extends ImmutableObject
   String name;
 
   /** The email address of the contact. */
-  @Id
-  @javax.persistence.Id
-  String emailAddress;
+  @Id @javax.persistence.Id String emailAddress;
 
   @Ignore @javax.persistence.Id String registrarId;
 
@@ -147,8 +145,7 @@ public class RegistrarContact extends ImmutableObject
    *
    * @see com.google.appengine.api.users.User#getUserId()
    */
-  @Index
-  String gaeUserId;
+  @Index String gaeUserId;
 
   /**
    * Whether this contact is publicly visible in WHOIS registrar query results as an Admin contact.
@@ -202,8 +199,7 @@ public class RegistrarContact extends ImmutableObject
    */
   public static void updateContacts(
       final Registrar registrar, final Set<RegistrarContact> contacts) {
-    tm()
-        .transact(
+    tm().transact(
             () -> {
               ofy()
                   .delete()
@@ -364,8 +360,15 @@ public class RegistrarContact extends ImmutableObject
   }
 
   public VKey<RegistrarContact> createVKey() {
-    return VKey.create(
-        RegistrarContact.class, new RegistrarPocId(emailAddress, registrarId), Key.create(this));
+    return createVKey(Key.create(this));
+  }
+
+  /** Creates a {@link VKey} instance from a {@link Key} instance. */
+  public static VKey<RegistrarContact> createVKey(Key<RegistrarContact> key) {
+    Key<Registrar> parent = key.getParent();
+    String registrarId = parent.getName();
+    String emailAddress = key.getName();
+    return VKey.create(RegistrarContact.class, new RegistrarPocId(emailAddress, registrarId), key);
   }
 
   /** Class to represent the composite primary key for {@link RegistrarContact} entity. */

core/src/main/java/google/registry/model/registry/label/PremiumList.java

@@ -32,7 +32,6 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.hash.BloomFilter;
 import com.google.common.util.concurrent.UncheckedExecutionException;
@@ -45,9 +44,8 @@ import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.registry.Registry;
-import google.registry.schema.replay.DatastoreEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.schema.replay.NonReplicatedEntity;
-import google.registry.schema.replay.SqlEntity;
 import google.registry.schema.tld.PremiumListDao;
 import google.registry.util.NonFinalForTesting;
 import java.io.ByteArrayOutputStream;
@@ -114,7 +112,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   /** Virtual parent entity for premium list entry entities associated with a single revision. */
   @ReportedOn
   @Entity
-  public static class PremiumListRevision extends ImmutableObject implements DatastoreEntity {
+  public static class PremiumListRevision extends ImmutableObject implements DatastoreOnlyEntity {
 
     @Parent Key<PremiumList> parent;
 
@@ -171,11 +169,6 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
       }
       return revision;
     }
-
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of(); // not persisted in SQL
-    }
   }
 
   /**
@@ -332,7 +325,7 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
   @ReportedOn
   @Entity
   public static class PremiumListEntry extends DomainLabelEntry<Money, PremiumListEntry>
-      implements Buildable, DatastoreEntity {
+      implements Buildable, DatastoreOnlyEntity {
 
     @Parent
     Key<PremiumListRevision> parent;
@@ -349,11 +342,6 @@ public final class PremiumList extends BaseDomainLabelList<Money, PremiumList.Pr
       return new Builder(clone(this));
     }
 
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of();
-    }
-
     /** A builder for constructing {@link PremiumListEntry} objects, since they are immutable. */
     public static class Builder extends DomainLabelEntry.Builder<PremiumListEntry, Builder> {
 

core/src/main/java/google/registry/model/reporting/HistoryEntry.java

@@ -18,7 +18,6 @@ import static com.googlecode.objectify.Key.getKind;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -32,15 +31,20 @@ import google.registry.model.EppResource;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactHistory.ContactHistoryId;
 import google.registry.model.contact.ContactResource;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.domain.Period;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostHistory.HostHistoryId;
 import google.registry.model.host.HostResource;
+import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import java.util.Optional;
 import java.util.Set;
 import javax.annotation.Nullable;
 import javax.persistence.Access;
@@ -310,8 +314,35 @@ public class HistoryEntry extends ImmutableObject implements Buildable, Datastor
 
   // In SQL, save the child type
   @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of((SqlEntity) toChildHistoryEntity());
+  public Optional<SqlEntity> toSqlEntity() {
+    return Optional.of((SqlEntity) toChildHistoryEntity());
+  }
+
+  /** Creates a {@link VKey} instance from a {@link Key} instance. */
+  public static VKey<? extends HistoryEntry> createVKey(Key<HistoryEntry> key) {
+    String repoId = key.getParent().getName();
+    long id = key.getId();
+    Key<EppResource> parent = key.getParent();
+    String parentKind = parent.getKind();
+    if (parentKind.equals(getKind(DomainBase.class))) {
+      return VKey.create(
+          DomainHistory.class,
+          new DomainHistoryId(repoId, id),
+          Key.create(parent, DomainHistory.class, id));
+    } else if (parentKind.equals(getKind(HostResource.class))) {
+      return VKey.create(
+          HostHistory.class,
+          new HostHistoryId(repoId, id),
+          Key.create(parent, HostHistory.class, id));
+    } else if (parentKind.equals(getKind(ContactResource.class))) {
+      return VKey.create(
+          ContactHistory.class,
+          new ContactHistoryId(repoId, id),
+          Key.create(parent, ContactHistory.class, id));
+    } else {
+      throw new IllegalStateException(
+          String.format("Unknown kind of HistoryEntry parent %s", parentKind));
+    }
   }
 
   /** A builder for {@link HistoryEntry} since it is immutable */

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatch.java

@@ -18,13 +18,13 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.Buildable;
 import google.registry.model.ImmutableObject;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import google.registry.util.DomainNameUtils;
+import java.util.Optional;
 import java.util.Set;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -111,8 +111,8 @@ public class Spec11ThreatMatch extends ImmutableObject implements Buildable, Sql
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not stored in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore
   }
 
   @Override

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatchDao.java

@@ -21,10 +21,10 @@ import javax.persistence.TemporalType;
 import org.joda.time.LocalDate;
 
 /**
- * Data access object for {@link google.registry.model.reporting.Spec11ThreatMatch}.
+ * Data access object for {@link Spec11ThreatMatch}.
  *
- * <p>A JpaTransactionManager is passed into each static method because they are called from a BEAM
- * pipeline and we don't know where it's coming from.
+ * <p>The transaction manager is passed as a parameter because this could be called either from a
+ * BEAM pipeline or standard non-BEAM code.
  */
 public class Spec11ThreatMatchDao {
 

core/src/main/java/google/registry/model/server/KmsSecret.java

@@ -16,7 +16,6 @@ package google.registry.model.server;
 
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
@@ -24,13 +23,12 @@ import com.googlecode.objectify.annotation.Parent;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.ReportedOn;
 import google.registry.model.common.EntityGroupRoot;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 
 /** Pointer to the latest {@link KmsSecretRevision}. */
 @Entity
 @ReportedOn
-public class KmsSecret extends ImmutableObject implements DatastoreEntity {
+public class KmsSecret extends ImmutableObject implements DatastoreOnlyEntity {
 
   /** The unique name of this {@link KmsSecret}. */
   @Id String name;
@@ -48,11 +46,6 @@ public class KmsSecret extends ImmutableObject implements DatastoreEntity {
     return latestRevision;
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // not persisted in SQL
-  }
-
   public static KmsSecret create(String name, KmsSecretRevision latestRevision) {
     KmsSecret instance = new KmsSecret();
     instance.name = name;

core/src/main/java/google/registry/model/server/Lock.java

@@ -22,15 +22,13 @@ import static google.registry.util.DateTimeUtils.isAtOrAfter;
 import com.google.auto.value.AutoValue;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.annotation.Entity;
 import com.googlecode.objectify.annotation.Id;
 import google.registry.model.ImmutableObject;
 import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
 import google.registry.util.RequestStatusChecker;
 import google.registry.util.RequestStatusCheckerImpl;
 import java.io.Serializable;
@@ -50,7 +48,7 @@ import org.joda.time.Duration;
  */
 @Entity
 @NotBackedUp(reason = Reason.TRANSIENT)
-public class Lock extends ImmutableObject implements DatastoreEntity, Serializable {
+public class Lock extends ImmutableObject implements DatastoreOnlyEntity, Serializable {
 
   private static final long serialVersionUID = 756397280691684645L;
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -259,9 +257,4 @@ public class Lock extends ImmutableObject implements DatastoreEntity, Serializab
               }
             });
   }
-
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // Locks are not converted since they are ephemeral
-  }
 }

core/src/main/java/google/registry/model/server/ServerSecret.java

@@ -23,7 +23,6 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Longs;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
@@ -34,8 +33,7 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
 import google.registry.persistence.VKey;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.NonReplicatedEntity;
 import java.nio.ByteBuffer;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
@@ -50,7 +48,7 @@ import javax.persistence.Transient;
 @Unindex
 @NotBackedUp(reason = Reason.AUTO_GENERATED)
 // TODO(b/27427316): Replace this with an entry in KMSKeyring
-public class ServerSecret extends CrossTldSingleton implements DatastoreEntity, SqlEntity {
+public class ServerSecret extends CrossTldSingleton implements NonReplicatedEntity {
 
   /**
    * Cache of the singleton ServerSecret instance that creates it if not present.
@@ -139,16 +137,6 @@ public class ServerSecret extends CrossTldSingleton implements DatastoreEntity,
         .array();
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // dually-written
-  }
-
-  @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // dually-written
-  }
-
   @VisibleForTesting
   static void resetCache() {
     CACHE.invalidateAll();

core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java

@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.isEmpty;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -32,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EmbedMap;
 import com.googlecode.objectify.annotation.Entity;
@@ -82,8 +82,6 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 public class SignedMarkRevocationList extends ImmutableObject implements NonReplicatedEntity {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
   @VisibleForTesting static final int SHARD_SIZE = 10000;
 
   /** Common ancestor for queries. */
@@ -121,12 +119,11 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
       memoizeWithShortExpiration(
           () -> {
             SignedMarkRevocationList datastoreList = loadFromDatastore();
-            // Also load the list from Cloud SQL, compare the two lists, and log if different.
-            try {
-              loadAndCompareCloudSqlList(datastoreList);
-            } catch (Throwable t) {
-              logger.atSevere().withCause(t).log("Error comparing signed mark revocation lists.");
-            }
+            suppressExceptionUnlessInTest(
+                () -> {
+                  loadAndCompareCloudSqlList(datastoreList);
+                },
+                "Error comparing signed mark revocation lists.");
             return datastoreList;
           });
 
@@ -229,11 +226,12 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
           Maps.difference(datastoreList.revokes, cloudSqlList.revokes);
       if (!diff.areEqual()) {
         if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
+          String message =
               String.format(
                   "Unequal SM revocation lists detected, Cloud SQL list with revision id %d has %d"
                       + " different records than the current Datastore list.",
-                  cloudSqlList.revisionId, diff.entriesDiffering().size()));
+                  cloudSqlList.revisionId, diff.entriesDiffering().size());
+          throw new RuntimeException(message);
         } else {
           StringBuilder diffMessage = new StringBuilder("Unequal SM revocation lists detected:\n");
           diff.entriesDiffering()
@@ -243,11 +241,13 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
                           String.format(
                               "SMD %s has key %s in Datastore and key %s in Cloud SQL.\n",
                               label, valueDiff.leftValue(), valueDiff.rightValue())));
-          logger.atWarning().log(diffMessage.toString());
+          throw new RuntimeException(diffMessage.toString());
         }
       }
     } else {
-      logger.atWarning().log("Signed mark revocation list in Cloud SQL is empty.");
+      if (datastoreList.size() != 0) {
+        throw new RuntimeException("Signed mark revocation list in Cloud SQL is empty.");
+      }
     }
   }
 

core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java

@@ -15,6 +15,7 @@
 package google.registry.model.smd;
 
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.common.base.Supplier;
@@ -60,14 +61,14 @@ public class SignedMarkRevocationListDao {
    * the authoritative database.
    */
   static void trySave(SignedMarkRevocationList signedMarkRevocationList) {
-    try {
-      SignedMarkRevocationListDao.save(signedMarkRevocationList);
-      logger.atInfo().log(
-          "Inserted %,d signed mark revocations into Cloud SQL",
-          signedMarkRevocationList.revokes.size());
-    } catch (Throwable e) {
-      logger.atSevere().withCause(e).log("Error inserting signed mark revocations into Cloud SQL");
-    }
+    suppressExceptionUnlessInTest(
+        () -> {
+          SignedMarkRevocationListDao.save(signedMarkRevocationList);
+          logger.atInfo().log(
+              "Inserted %,d signed mark revocations into Cloud SQL.",
+              signedMarkRevocationList.revokes.size());
+        },
+        "Error inserting signed mark revocations into Cloud SQL.");
   }
 
   private static void save(SignedMarkRevocationList signedMarkRevocationList) {

core/src/main/java/google/registry/model/tmch/ClaimsListShard.java

@@ -26,7 +26,6 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.MapDifference.ValueDifference;
@@ -46,9 +45,8 @@ import google.registry.model.annotations.NotBackedUp;
 import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.annotations.VirtualEntity;
 import google.registry.model.common.CrossTldSingleton;
-import google.registry.schema.replay.DatastoreAndSqlEntity;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.DatastoreOnlyEntity;
+import google.registry.schema.replay.NonReplicatedEntity;
 import google.registry.util.CollectionUtils;
 import google.registry.util.Concurrent;
 import google.registry.util.Retrier;
@@ -97,7 +95,7 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 @javax.persistence.Entity(name = "ClaimsList")
 @Table
-public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlEntity {
+public class ClaimsListShard extends ImmutableObject implements NonReplicatedEntity {
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
@@ -146,63 +144,62 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlE
   private static final Retrier LOADER_RETRIER = new Retrier(new SystemSleeper(), 2);
 
   private static ClaimsListShard loadClaimsListShard() {
-        // Find the most recent revision.
-        Key<ClaimsListRevision> revisionKey = getCurrentRevision();
+    // Find the most recent revision.
+    Key<ClaimsListRevision> revisionKey = getCurrentRevision();
 
-        Map<String, String> combinedLabelsToKeys = new HashMap<>();
-        DateTime creationTime = START_OF_TIME;
-        if (revisionKey != null) {
-          // Grab all of the keys for the shards that belong to the current revision.
-          final List<Key<ClaimsListShard>> shardKeys =
-              ofy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list();
+    Map<String, String> combinedLabelsToKeys = new HashMap<>();
+    DateTime creationTime = START_OF_TIME;
+    if (revisionKey != null) {
+      // Grab all of the keys for the shards that belong to the current revision.
+      final List<Key<ClaimsListShard>> shardKeys =
+          ofy().load().type(ClaimsListShard.class).ancestor(revisionKey).keys().list();
 
-          List<ClaimsListShard> shards;
-          try {
-            // Load all of the shards concurrently, each in a separate transaction.
-            shards =
-                Concurrent.transform(
-                    shardKeys,
-                    key ->
-                        tm().transactNewReadOnly(
-                                () -> {
-                                  ClaimsListShard claimsListShard = ofy().load().key(key).now();
-                                  checkState(
-                                      claimsListShard != null,
-                                      "Key not found when loading claims list shards.");
-                                  return claimsListShard;
-                                }));
-          } catch (UncheckedExecutionException e) {
-            // We retry on IllegalStateException. However, there's a checkState inside the
-            // Concurrent.transform, so if it's thrown it'll be wrapped in an
-            // UncheckedExecutionException. We want to unwrap it so it's caught by the retrier.
-            if (e.getCause() != null) {
-              throwIfUnchecked(e.getCause());
-            }
-            throw e;
-          }
-
-          // Combine the shards together and return the concatenated ClaimsList.
-          if (!shards.isEmpty()) {
-            creationTime = shards.get(0).creationTime;
-            for (ClaimsListShard shard : shards) {
-              combinedLabelsToKeys.putAll(shard.labelsToKeys);
-              checkState(
-                  creationTime.equals(shard.creationTime),
-                  "Inconsistent claims list shard creation times.");
-            }
-          }
+      List<ClaimsListShard> shards;
+      try {
+        // Load all of the shards concurrently, each in a separate transaction.
+        shards =
+            Concurrent.transform(
+                shardKeys,
+                key ->
+                    tm().transactNewReadOnly(
+                            () -> {
+                              ClaimsListShard claimsListShard = ofy().load().key(key).now();
+                              checkState(
+                                  claimsListShard != null,
+                                  "Key not found when loading claims list shards.");
+                              return claimsListShard;
+                            }));
+      } catch (UncheckedExecutionException e) {
+        // We retry on IllegalStateException. However, there's a checkState inside the
+        // Concurrent.transform, so if it's thrown it'll be wrapped in an
+        // UncheckedExecutionException. We want to unwrap it so it's caught by the retrier.
+        if (e.getCause() != null) {
+          throwIfUnchecked(e.getCause());
         }
+        throw e;
+      }
 
-        ClaimsListShard datastoreList =
-            create(creationTime, ImmutableMap.copyOf(combinedLabelsToKeys));
-        // Also load the list from Cloud SQL, compare the two lists, and log if different.
-        try {
-          loadAndCompareCloudSqlList(datastoreList);
-        } catch (Throwable t) {
-          logger.atSevere().withCause(t).log("Error comparing claims lists.");
+      // Combine the shards together and return the concatenated ClaimsList.
+      if (!shards.isEmpty()) {
+        creationTime = shards.get(0).creationTime;
+        for (ClaimsListShard shard : shards) {
+          combinedLabelsToKeys.putAll(shard.labelsToKeys);
+          checkState(
+              creationTime.equals(shard.creationTime),
+              "Inconsistent claims list shard creation times.");
         }
-        return datastoreList;
-      };
+      }
+    }
+
+    ClaimsListShard datastoreList = create(creationTime, ImmutableMap.copyOf(combinedLabelsToKeys));
+    // Also load the list from Cloud SQL, compare the two lists, and log if different.
+    try {
+      loadAndCompareCloudSqlList(datastoreList);
+    } catch (Throwable t) {
+      logger.atSevere().withCause(t).log("Error comparing claims lists.");
+    }
+    return datastoreList;
+  };
 
   private static void loadAndCompareCloudSqlList(ClaimsListShard datastoreList) {
     Optional<ClaimsListShard> maybeCloudSqlList = ClaimsListDao.getLatestRevision();
@@ -304,8 +301,7 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlE
     Concurrent.transform(
         CollectionUtils.partitionMap(labelsToKeys, shardSize),
         (final ImmutableMap<String, String> labelsToKeysShard) ->
-            tm()
-                .transactNew(
+            tm().transactNew(
                     () -> {
                       ClaimsListShard shard = create(creationTime, labelsToKeysShard);
                       shard.isShard = true;
@@ -315,8 +311,7 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlE
                     }));
 
     // Persist the new revision, thus causing the newly created shards to go live.
-    tm()
-        .transactNew(
+    tm().transactNew(
             () -> {
               verify(
                   (getCurrentRevision() == null && oldRevision == null)
@@ -358,12 +353,10 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlE
   /** Virtual parent entity for claims list shards of a specific revision. */
   @Entity
   @VirtualEntity
-  public static class ClaimsListRevision extends ImmutableObject implements DatastoreEntity {
-    @Parent
-    Key<ClaimsListSingleton> parent;
+  public static class ClaimsListRevision extends ImmutableObject implements DatastoreOnlyEntity {
+    @Parent Key<ClaimsListSingleton> parent;
 
-    @Id
-    long versionId;
+    @Id long versionId;
 
     @VisibleForTesting
     public static Key<ClaimsListRevision> createKey(ClaimsListSingleton singleton) {
@@ -377,11 +370,6 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlE
     public static Key<ClaimsListRevision> createKey() {
       return createKey(new ClaimsListSingleton());
     }
-
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of(); // ClaimsLists are dually written
-    }
   }
 
   /**
@@ -390,7 +378,7 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlE
    */
   @Entity
   @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
-  public static class ClaimsListSingleton extends CrossTldSingleton implements DatastoreEntity {
+  public static class ClaimsListSingleton extends CrossTldSingleton implements DatastoreOnlyEntity {
     Key<ClaimsListRevision> activeRevision;
 
     static ClaimsListSingleton create(Key<ClaimsListRevision> revision) {
@@ -403,11 +391,6 @@ public class ClaimsListShard extends ImmutableObject implements DatastoreAndSqlE
     public void setActiveRevision(Key<ClaimsListRevision> revision) {
       activeRevision = revision;
     }
-
-    @Override
-    public ImmutableList<SqlEntity> toSqlEntities() {
-      return ImmutableList.of(); // ClaimsLists are dually written
-    }
   }
 
   /**

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -20,7 +20,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
-import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.annotations.NotBackedUp;
@@ -28,8 +27,7 @@ import google.registry.model.annotations.NotBackedUp.Reason;
 import google.registry.model.common.CrossTldSingleton;
 import google.registry.model.tmch.TmchCrl.TmchCrlId;
 import google.registry.persistence.VKey;
-import google.registry.schema.replay.DatastoreEntity;
-import google.registry.schema.replay.SqlEntity;
+import google.registry.schema.replay.NonReplicatedEntity;
 import java.io.Serializable;
 import java.util.Optional;
 import javax.annotation.concurrent.Immutable;
@@ -44,7 +42,7 @@ import org.joda.time.DateTime;
 @Immutable
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 @IdClass(TmchCrlId.class)
-public final class TmchCrl extends CrossTldSingleton implements DatastoreEntity, SqlEntity {
+public final class TmchCrl extends CrossTldSingleton implements NonReplicatedEntity {
 
   @Id String crl;
 
@@ -105,16 +103,6 @@ public final class TmchCrl extends CrossTldSingleton implements DatastoreEntity,
     return updated;
   }
 
-  @Override
-  public ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(); // dually-written
-  }
-
-  @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // dually-written
-  }
-
   static class TmchCrlId implements Serializable {
 
     @Column(name = "certificateRevocations")

core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java

@@ -46,7 +46,10 @@ public class UpdateAutoTimestampTranslatorFactory
       /** Save a timestamp, setting it to the current time. */
       @Override
       public Date saveValue(UpdateAutoTimestamp pojoValue) {
-        return tm().getTransactionTime().toDate();
-      }};
+        return UpdateAutoTimestamp.autoUpdateEnabled()
+            ? tm().getTransactionTime().toDate()
+            : pojoValue.getTimestamp().toDate();
+      }
+    };
   }
 }

core/src/main/java/google/registry/persistence/PersistenceComponent.java

@@ -20,6 +20,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import javax.inject.Singleton;
 import javax.persistence.EntityManagerFactory;
@@ -32,6 +33,7 @@ import javax.persistence.EntityManagerFactory;
       CredentialModule.class,
       KmsModule.class,
       PersistenceModule.class,
+      SecretManagerModule.class,
       UtilsModule.class
     })
 public interface PersistenceComponent {

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -26,6 +26,7 @@ import com.google.api.client.auth.oauth2.Credential;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
@@ -33,6 +34,11 @@ import google.registry.keyring.kms.KmsKeyring;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotId;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
 import google.registry.tools.AuthModule.CloudSqlClientCredential;
 import google.registry.util.Clock;
 import java.lang.annotation.Documented;
@@ -47,6 +53,8 @@ import org.hibernate.cfg.Environment;
 /** Dagger module class for the persistence layer. */
 @Module
 public class PersistenceModule {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   // This name must be the same as the one defined in persistence.xml.
   public static final String PERSISTENCE_UNIT_NAME = "nomulus";
   public static final String HIKARI_CONNECTION_TIMEOUT = "hibernate.hikari.connectionTimeout";
@@ -122,11 +130,17 @@ public class PersistenceModule {
   static JpaTransactionManager provideAppEngineJpaTm(
       @Config("cloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -136,6 +150,7 @@ public class PersistenceModule {
   static JpaTransactionManager provideNomulusToolJpaTm(
       @Config("toolsCloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       @CloudSqlClientCredential Credential credential,
       Clock clock) {
@@ -143,6 +158,11 @@ public class PersistenceModule {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getToolsCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.TOOL),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -150,6 +170,7 @@ public class PersistenceModule {
   @Singleton
   @SocketFactoryJpaTm
   static JpaTransactionManager provideSocketFactoryJpaTm(
+      SqlCredentialStore credentialStore,
       @Config("beamCloudSqlUsername") String username,
       @Config("beamCloudSqlPassword") String password,
       @Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
@@ -159,6 +180,12 @@ public class PersistenceModule {
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, password);
     overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
+    // TODO(b/175700623): consider assigning different logins to pipelines
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -203,6 +230,30 @@ public class PersistenceModule {
     return emf;
   }
 
+  /** Verifies that the credential from the Secret Manager matches the one currently in use.
+   *
+   * <p>This is a helper for the transition to the Secret Manager, and will be removed once data
+   * and permissions are properly set up for all projects.
+   **/
+  private static void validateCredentialStore(
+      SqlCredentialStore credentialStore, SqlUser sqlUser, String login, String password) {
+    try {
+      SqlCredential credential = credentialStore.getCredential(sqlUser);
+      if (!credential.login().equals(login)) {
+        logger.atWarning().log(
+            "Wrong login for %s. Expecting %s, found %s.",
+            sqlUser.geUserName(), login, credential.login());
+        return;
+      }
+      if (!credential.password().equals(password)) {
+        logger.atWarning().log("Wrong password for %s.", sqlUser.geUserName());
+      }
+      logger.atWarning().log("Credentials in the kerying and the secret manager match.");
+    } catch (Throwable e) {
+      logger.atWarning().log(e.getMessage());
+    }
+  }
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
   @Qualifier
   @Documented

core/src/main/java/google/registry/persistence/WithLongVKey.java

@@ -31,4 +31,12 @@ public @interface WithLongVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/WithStringVKey.java

@@ -31,4 +31,12 @@ public @interface WithStringVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/converter/UpdateAutoTimestampConverter.java

@@ -31,7 +31,14 @@ public class UpdateAutoTimestampConverter
 
   @Override
   public Timestamp convertToDatabaseColumn(UpdateAutoTimestamp entity) {
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(jpaTm().getTransactionTime()).toInstant());
+    return Timestamp.from(
+        DateTimeUtils.toZonedDateTime(
+                UpdateAutoTimestamp.autoUpdateEnabled()
+                        || entity == null
+                        || entity.getTimestamp() == null
+                    ? jpaTm().getTransactionTime()
+                    : entity.getTimestamp())
+            .toInstant());
   }
 
   @Override

core/src/main/java/google/registry/persistence/converter/VKeyConverter.java

@@ -14,6 +14,7 @@
 
 package google.registry.persistence.converter;
 
+import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
@@ -29,7 +30,28 @@ public abstract class VKeyConverter<T, C> implements AttributeConverter<VKey<? e
   @Override
   @Nullable
   public VKey<? extends T> convertToEntityAttribute(@Nullable C dbData) {
-    return dbData == null ? null : VKey.createSql(getAttributeClass(), dbData);
+    if (dbData == null) {
+      return null;
+    }
+    Class<? extends T> clazz = getAttributeClass();
+    Key ofyKey = null;
+    if (!hasCompositeOfyKey()) {
+      // If this isn't a composite key, we can create the Ofy key from the SQL key.
+      ofyKey =
+          dbData instanceof String
+              ? Key.create(clazz, (String) dbData)
+              : Key.create(clazz, (Long) dbData);
+      return VKey.create(clazz, dbData, ofyKey);
+    } else {
+      // We don't know how to create the Ofy key and probably don't have everything necessary to do
+      // it anyway, so just create an asymmetric key - the containing object will have to convert it
+      // into a symmetric key.
+      return VKey.createSql(clazz, dbData);
+    }
+  }
+
+  protected boolean hasCompositeOfyKey() {
+    return false;
   }
 
   /** Returns the class of the attribute. */

core/src/main/java/google/registry/persistence/transaction/TransactionEntity.java

@@ -14,9 +14,9 @@
 
 package google.registry.persistence.transaction;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
+import java.util.Optional;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
@@ -45,7 +45,7 @@ public class TransactionEntity implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not stored in Datastore per se
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore per se
   }
 }

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClient.java

@@ -22,6 +22,9 @@ import java.util.Optional;
 /** A Cloud Secret Manager client for Nomulus, bound to a specific GCP project. */
 public interface SecretManagerClient {
 
+  /** Returns the project name with which this client is associated. */
+  String getProject();
+
   /**
    * Creates a new secret in the Cloud Secret Manager with no data.
    *
@@ -32,6 +35,9 @@ public interface SecretManagerClient {
    */
   void createSecret(String secretId);
 
+  /** Checks if a secret with the given {@code secretId} already exists. */
+  boolean secretExists(String secretId);
+
   /** Returns all secret IDs in the Cloud Secret Manager. */
   Iterable<String> listSecrets();
 
@@ -67,6 +73,24 @@ public interface SecretManagerClient {
    */
   String getSecretData(String secretId, Optional<String> version);
 
+  /**
+   * Enables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void enableSecretVersion(String secretId, String version);
+
+  /**
+   * Disables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void disableSecretVersion(String secretId, String version);
+
   /**
    * Destroys a secret version.
    *

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClientImpl.java

@@ -29,12 +29,11 @@ import com.google.cloud.secretmanager.v1.SecretName;
 import com.google.cloud.secretmanager.v1.SecretPayload;
 import com.google.cloud.secretmanager.v1.SecretVersion;
 import com.google.cloud.secretmanager.v1.SecretVersionName;
-import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
 import com.google.protobuf.ByteString;
 import google.registry.util.Retrier;
 import java.util.Optional;
 import java.util.concurrent.Callable;
-import javax.inject.Inject;
 
 /** Implements {@link SecretManagerClient} on Google Cloud Platform. */
 public class SecretManagerClientImpl implements SecretManagerClient {
@@ -42,13 +41,17 @@ public class SecretManagerClientImpl implements SecretManagerClient {
   private final SecretManagerServiceClient csmClient;
   private final Retrier retrier;
 
-  @Inject
   SecretManagerClientImpl(String project, SecretManagerServiceClient csmClient, Retrier retrier) {
     this.project = project;
     this.csmClient = csmClient;
     this.retrier = retrier;
   }
 
+  @Override
+  public String getProject() {
+    return project;
+  }
+
   @Override
   public void createSecret(String secretId) {
     checkNotNull(secretId, "secretId");
@@ -57,12 +60,25 @@ public class SecretManagerClientImpl implements SecretManagerClient {
         () -> csmClient.createSecret(ProjectName.of(project), secretId, secretSettings));
   }
 
+  @Override
+  public boolean secretExists(String secretId) {
+    checkNotNull(secretId, "secretId");
+    try {
+      callSecretManager(() -> csmClient.getSecret(SecretName.of(project, secretId)));
+      return true;
+    } catch (NoSuchSecretResourceException e) {
+      return false;
+    }
+  }
+
   @Override
   public Iterable<String> listSecrets() {
     ListSecretsPagedResponse response =
         callSecretManager(() -> csmClient.listSecrets(ProjectName.of(project)));
-    return Iterables.transform(
-        response.iterateAll(), secret -> SecretName.parse(secret.getName()).getSecret());
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(secret -> SecretName.parse(secret.getName()).getSecret())
+            .iterator();
   }
 
   @Override
@@ -70,8 +86,10 @@ public class SecretManagerClientImpl implements SecretManagerClient {
     checkNotNull(secretId, "secretId");
     ListSecretVersionsPagedResponse response =
         callSecretManager(() -> csmClient.listSecretVersions(SecretName.of(project, secretId)));
-    return Iterables.transform(
-        response.iterateAll(), SecretManagerClientImpl::toSecretVersionState);
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(SecretManagerClientImpl::toSecretVersionState)
+            .iterator();
   }
 
   private static SecretVersionState toSecretVersionState(SecretVersion secretVersion) {
@@ -108,6 +126,22 @@ public class SecretManagerClientImpl implements SecretManagerClient {
                 .toStringUtf8());
   }
 
+  @Override
+  public void enableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.enableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
+  @Override
+  public void disableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.disableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
   @Override
   public void destroySecretVersion(String secretId, String version) {
     checkNotNull(secretId, "secretId");

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerModule.java

@@ -14,41 +14,47 @@
 
 package google.registry.privileges.secretmanager;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
-import dagger.Component;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
-import google.registry.util.UtilsModule;
 import java.io.IOException;
 import javax.inject.Singleton;
 
 /** Provides bindings for {@link SecretManagerClient}. */
 @Module
-public class SecretManagerModule {
-
-  private final String project;
-
-  public SecretManagerModule(String project) {
-    this.project = checkNotNull(project, "project");
-  }
+public abstract class SecretManagerModule {
 
   @Provides
   @Singleton
-  SecretManagerClient provideSecretManagerClient(Retrier retrier) {
+  static SecretManagerServiceSettings provideSecretManagerSetting(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle) {
     try {
-      return new SecretManagerClientImpl(project, SecretManagerServiceClient.create(), retrier);
+      return SecretManagerServiceSettings.newBuilder()
+          .setCredentialsProvider(() -> credentialsBundle.getGoogleCredentials())
+          .build();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  @Provides
   @Singleton
-  @Component(modules = {ConfigModule.class, SecretManagerModule.class, UtilsModule.class})
-  public interface SecretManagerComponent {
-    SecretManagerClient secretManagerClient();
+  static SecretManagerClient provideSecretManagerClient(
+      SecretManagerServiceSettings serviceSettings,
+      @Config("projectId") String project,
+      Retrier retrier) {
+    try {
+      SecretManagerServiceClient stub = SecretManagerServiceClient.create(serviceSettings);
+      Runtime.getRuntime().addShutdownHook(new Thread(stub::close));
+      return new SecretManagerClientImpl(project, stub, retrier);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
   }
 }

core/src/main/java/google/registry/privileges/secretmanager/SqlCredential.java

@@ -0,0 +1,55 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static avro.shaded.com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.value.AutoValue;
+import java.util.List;
+
+/**
+ * Contains the login name and password of a Cloud SQL user.
+ *
+ * <p>User must take care not to include the {@link #SEPARATOR} in property values.
+ */
+@AutoValue
+public abstract class SqlCredential {
+
+  public static final Character SEPARATOR = ' ';
+
+  public abstract String login();
+
+  public abstract String password();
+
+  @Override
+  public final String toString() {
+    // Use Object.toString(), which does not show object data.
+    return super.toString();
+  }
+
+  public final String toFormattedString() {
+    return String.format("%s%c%s", login(), SEPARATOR, password());
+  }
+
+  public static SqlCredential fromFormattedString(String sqlCredential) {
+    List<String> items = com.google.common.base.Splitter.on(SEPARATOR).splitToList(sqlCredential);
+    checkState(items.size() == 2, "Invalid SqlCredential string.");
+    return of(items.get(0), items.get(1));
+  }
+
+  public static SqlCredential of(String login, String password) {
+    return new AutoValue_SqlCredential(login, password);
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlCredentialStore.java

@@ -0,0 +1,118 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.cloud.secretmanager.v1.SecretVersionName;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Storage of SQL users' login credentials, backed by Cloud Secret Manager.
+ *
+ * <p>A user's credential is stored with one level of indirection using two secret IDs: Each version
+ * of the <em>credential data</em> is stored as follows: its secret ID is determined by {@link
+ * #getCredentialDataSecretId(SqlUser, String dbInstance)}, and the value of each version is a
+ * {@link SqlCredential}, serialized using {@link SqlCredential#toFormattedString}. The 'live'
+ * version of the credential is saved under the 'live pointer' secret explained below.
+ *
+ * <p>The pointer to the 'live' version of the credential data is stored as follows: its secret ID
+ * is determined by {@link #getLiveLabelSecretId(SqlUser, String dbInstance)}; and the value of each
+ * version is a {@link SecretVersionName} in String form, pointing to a version of the credential
+ * data. Only the 'latest' version of this secret should be used. It is guaranteed to be valid.
+ *
+ * <p>The indirection in credential storage makes it easy to handle failures in the credential
+ * change process.
+ */
+public class SqlCredentialStore {
+  private final SecretManagerClient csmClient;
+  private final String dbInstance;
+
+  @Inject
+  SqlCredentialStore(
+      SecretManagerClient csmClient, @Config("cloudSqlDbInstanceName") String dbInstance) {
+    this.csmClient = csmClient;
+    this.dbInstance = dbInstance;
+  }
+
+  public SqlCredential getCredential(SqlUser user) {
+    SecretVersionName credentialName = getLiveCredentialSecretVersion(user);
+    return SqlCredential.fromFormattedString(
+        csmClient.getSecretData(
+            credentialName.getSecret(), Optional.of(credentialName.getSecretVersion())));
+  }
+
+  public void createOrUpdateCredential(SqlUser user, String password) {
+    SecretVersionName dataName = saveCredentialData(user, password);
+    saveLiveLabel(user, dataName);
+  }
+
+  public void deleteCredential(SqlUser user) {
+    try {
+      csmClient.deleteSecret(getCredentialDataSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok
+    }
+    try {
+      csmClient.deleteSecret(getLiveLabelSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok.
+    }
+  }
+
+  private void createSecretIfAbsent(String secretId) {
+    try {
+      csmClient.createSecret(secretId);
+    } catch (SecretAlreadyExistsException ignore) {
+      // Not a problem.
+    }
+  }
+
+  private SecretVersionName saveCredentialData(SqlUser user, String password) {
+    String credentialDataSecretId = getCredentialDataSecretId(user, dbInstance);
+    createSecretIfAbsent(credentialDataSecretId);
+    String credentialVersion =
+        csmClient.addSecretVersion(
+            credentialDataSecretId,
+            SqlCredential.of(createDatabaseLoginName(user), password).toFormattedString());
+    return SecretVersionName.of(csmClient.getProject(), credentialDataSecretId, credentialVersion);
+  }
+
+  private void saveLiveLabel(SqlUser user, SecretVersionName dataVersionName) {
+    String liveLabelSecretId = getLiveLabelSecretId(user, dbInstance);
+    createSecretIfAbsent(liveLabelSecretId);
+    csmClient.addSecretVersion(liveLabelSecretId, dataVersionName.toString());
+  }
+
+  private SecretVersionName getLiveCredentialSecretVersion(SqlUser user) {
+    return SecretVersionName.parse(
+        csmClient.getSecretData(getLiveLabelSecretId(user, dbInstance), Optional.empty()));
+  }
+
+  private static String getLiveLabelSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-live-label-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  private static String getCredentialDataSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-data-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  // WIP: when b/170230882 is complete, login will be versioned.
+  private static String createDatabaseLoginName(SqlUser user) {
+    return user.geUserName();
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlUser.java

@@ -0,0 +1,67 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.common.base.Ascii;
+
+/**
+ * SQL user information for privilege management purposes.
+ *
+ * <p>A {@link RobotUser} represents a software system accessing the database using its own
+ * credential. Robots are well known and enumerated in {@link RobotId}.
+ */
+public abstract class SqlUser {
+
+  private final UserType type;
+  private final String userName;
+
+  protected SqlUser(UserType type, String userName) {
+    this.type = type;
+    this.userName = userName;
+  }
+
+  public UserType getType() {
+    return type;
+  }
+
+  public String geUserName() {
+    return userName;
+  }
+
+  /** Cloud SQL user types. Please see class javadoc of {@link SqlUser} for more information. */
+  enum UserType {
+    // Work in progress. Human user will be added.
+    ROBOT
+  }
+
+  /** Enumerates the {@link RobotUser RobotUsers} in the system. */
+  public enum RobotId {
+    NOMULUS,
+    /**
+     * Credential for RegistryTool. This is temporary, and will be removed when tool users are
+     * assigned their personal credentials.
+     */
+    TOOL;
+  }
+
+  /** Information of a RobotUser for privilege management purposes. */
+  // Work in progress. Eventually will be provided based on configuration.
+  public static class RobotUser extends SqlUser {
+
+    public RobotUser(RobotId robot) {
+      super(UserType.ROBOT, Ascii.toLowerCase(robot.name()));
+    }
+  }
+}

core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java

@@ -14,9 +14,11 @@
 
 package google.registry.reporting.spec11;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
@@ -59,6 +61,25 @@ public class Spec11RegistrarThreatMatchesParser {
     return getFromFile(getGcsFilename(date));
   }
 
+  /** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */
+  public ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
+      throws IOException {
+    if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) {
+      return ImmutableSet.of();
+    }
+    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
+    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename);
+        InputStreamReader isr = new InputStreamReader(in, UTF_8)) {
+      // Skip the header at line 0
+      return Splitter.on("\n")
+          .omitEmptyStrings()
+          .splitToStream(CharStreams.toString(isr))
+          .skip(1)
+          .map(this::parseRegistrarThreatMatch)
+          .collect(toImmutableSet());
+    }
+  }
+
   public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
     LocalDate yesterday = date.minusDays(1);
     GcsFilename gcsFilename = getGcsFilename(yesterday);
@@ -82,20 +103,6 @@ public class Spec11RegistrarThreatMatchesParser {
     return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
   }
 
-  private ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
-      throws IOException, JSONException {
-    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
-    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
-      ImmutableList<String> reportLines =
-          ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
-      // Iterate from 1 to size() to skip the header at line 0.
-      for (int i = 1; i < reportLines.size(); i++) {
-        builder.add(parseRegistrarThreatMatch(reportLines.get(i)));
-      }
-      return builder.build();
-    }
-  }
-
   private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
     JSONObject reportJSON = new JSONObject(line);
     String clientId = reportJSON.getString(Spec11Pipeline.REGISTRAR_CLIENT_ID_FIELD);

core/src/main/java/google/registry/schema/cursor/Cursor.java

@@ -16,7 +16,6 @@ package google.registry.schema.cursor;
 
 import static com.google.appengine.api.search.checkers.Preconditions.checkNotNull;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.common.Cursor.CursorType;
@@ -26,6 +25,7 @@ import google.registry.schema.replay.SqlEntity;
 import google.registry.util.DateTimeUtils;
 import java.io.Serializable;
 import java.time.ZonedDateTime;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.EnumType;
@@ -104,8 +104,8 @@ public class Cursor implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // Cursors are not converted since they are ephemeral
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Cursors are not converted since they are ephemeral
   }
 
   static class CursorId extends ImmutableObject implements Serializable {

core/src/main/java/google/registry/schema/domain/RegistryLock.java

@@ -19,7 +19,6 @@ import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.toZonedDateTime;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.Buildable;
 import google.registry.model.CreateAutoTimestamp;
 import google.registry.model.ImmutableObject;
@@ -234,8 +233,8 @@ public final class RegistryLock extends ImmutableObject implements Buildable, Sq
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not stored in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not persisted in Datastore
   }
 
   /** Builder for {@link google.registry.schema.domain.RegistryLock}. */

core/src/main/java/google/registry/schema/replay/DatastoreAndSqlEntity.java

@@ -14,18 +14,18 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /** An entity that has the same Java object representation in SQL and Datastore. */
 public interface DatastoreAndSqlEntity extends DatastoreEntity, SqlEntity {
 
   @Override
-  default ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(this);
+  default Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.of(this);
   }
 
   @Override
-  default ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of(this);
+  default Optional<SqlEntity> toSqlEntity() {
+    return Optional.of(this);
   }
 }

core/src/main/java/google/registry/schema/replay/DatastoreEntity.java

@@ -14,7 +14,7 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /**
  * An object that can be stored in Datastore and serialized using Objectify's {@link
@@ -26,5 +26,5 @@ import com.google.common.collect.ImmutableList;
  */
 public interface DatastoreEntity {
 
-  ImmutableList<SqlEntity> toSqlEntities();
+  Optional<SqlEntity> toSqlEntity();
 }

core/src/main/java/google/registry/schema/replay/DatastoreOnlyEntity.java

@@ -0,0 +1,25 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.schema.replay;
+
+import java.util.Optional;
+
+/** An entity that is only stored in Datastore, that should not be replayed to SQL. */
+public interface DatastoreOnlyEntity extends DatastoreEntity {
+  @Override
+  default Optional<SqlEntity> toSqlEntity() {
+    return Optional.empty();
+  }
+}

core/src/main/java/google/registry/schema/replay/NonReplicatedEntity.java

@@ -14,7 +14,7 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /**
  * Represents an entity that should not participate in asynchronous replication.
@@ -24,12 +24,12 @@ import com.google.common.collect.ImmutableList;
 public interface NonReplicatedEntity extends DatastoreEntity, SqlEntity {
 
   @Override
-  default ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of();
+  default Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty();
   }
 
   @Override
-  default ImmutableList<SqlEntity> toSqlEntities() {
-    return ImmutableList.of();
+  default Optional<SqlEntity> toSqlEntity() {
+    return Optional.empty();
   }
 }

core/src/main/java/google/registry/schema/replay/SqlEntity.java

@@ -14,7 +14,7 @@
 
 package google.registry.schema.replay;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 
 /**
  * An object that can be stored in Cloud SQL using {@link
@@ -25,5 +25,5 @@ import com.google.common.collect.ImmutableList;
  */
 public interface SqlEntity {
 
-  ImmutableList<DatastoreEntity> toDatastoreEntities();
+  Optional<DatastoreEntity> toDatastoreEntity();
 }

core/src/main/java/google/registry/schema/replay/SqlReplayCheckpoint.java

@@ -18,7 +18,7 @@ import static google.registry.model.common.CrossTldSingleton.SINGLETON_ID;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 
-import com.google.common.collect.ImmutableList;
+import java.util.Optional;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.joda.time.DateTime;
@@ -32,8 +32,8 @@ public class SqlReplayCheckpoint implements SqlEntity {
   private DateTime lastReplayTime;
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // not necessary to persist in Datastore
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Not necessary to persist in Datastore
   }
 
   public static DateTime get() {

core/src/main/java/google/registry/schema/server/Lock.java

@@ -16,7 +16,6 @@ package google.registry.schema.server;
 
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
@@ -24,6 +23,7 @@ import google.registry.schema.server.Lock.LockId;
 import google.registry.util.DateTimeUtils;
 import java.io.Serializable;
 import java.time.ZonedDateTime;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -120,8 +120,8 @@ public class Lock implements SqlEntity {
   }
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // Locks are not converted since they are ephemeral
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // Locks are not converted since they are ephemeral
   }
 
   static class LockId extends ImmutableObject implements Serializable {

core/src/main/java/google/registry/schema/tld/PremiumEntry.java

@@ -14,13 +14,13 @@
 
 package google.registry.schema.tld;
 
-import com.google.common.collect.ImmutableList;
 import google.registry.model.ImmutableObject;
 import google.registry.model.registry.label.PremiumList;
 import google.registry.schema.replay.DatastoreEntity;
 import google.registry.schema.replay.SqlEntity;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.util.Optional;
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
@@ -47,7 +47,7 @@ public class PremiumEntry extends ImmutableObject implements Serializable, SqlEn
   private PremiumEntry() {}
 
   @Override
-  public ImmutableList<DatastoreEntity> toDatastoreEntities() {
-    return ImmutableList.of(); // PremiumList is dually-written
+  public Optional<DatastoreEntity> toDatastoreEntity() {
+    return Optional.empty(); // PremiumList is dually-written
   }
 }

core/src/main/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java

@@ -138,15 +138,6 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
       validateWith = PathParameter.InputFile.class)
   Path clientCertificateFilename;
 
-  @Nullable
-  @Parameter(
-    names = "--cert_hash",
-    description =
-        "Hash of client certificate (SHA256 base64 no padding). Do not use this unless "
-            + "you want to store ONLY the hash and not the full certificate"
-  )
-  String clientCertificateHash;
-
   @Nullable
   @Parameter(
       names = "--failover_cert_file",
@@ -375,14 +366,6 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
         }
         builder.setFailoverClientCertificate(asciiCert, now);
       }
-      if (!isNullOrEmpty(clientCertificateHash)) {
-        checkArgument(clientCertificateFilename == null,
-            "Can't specify both --cert_hash and --cert_file");
-        if ("null".equals(clientCertificateHash)) {
-          clientCertificateHash = null;
-        }
-        builder.setClientCertificateHash(clientCertificateHash);
-      }
       if (ianaId != null) {
         builder.setIanaIdentifier(ianaId.orElse(null));
       }

core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java

@@ -45,10 +45,10 @@ import google.registry.model.domain.DomainBase;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.OneTime entities with duplicate IDs.")
-public class DedupeOneTimeBillingEventIdsCommand extends DedupeEntityIdsCommand<OneTime> {
+public class DedupeOneTimeBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<OneTime> {
 
   @Override
-  void dedupe(OneTime entity) {
+  void process(OneTime entity) {
     Key<BillingEvent> key = Key.create(entity);
     Key<DomainBase> domainKey = getGrandParentAsDomain(key);
     DomainBase domain = ofy().load().key(domainKey).now();

core/src/main/java/google/registry/tools/DedupeRecurringBillingEventIdsCommand.java

@@ -54,11 +54,10 @@ import java.util.Set;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.Recurring entities with duplicate IDs.")
-public class DedupeRecurringBillingEventIdsCommand
-    extends DedupeEntityIdsCommand<BillingEvent.Recurring> {
+public class DedupeRecurringBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<Recurring> {
 
   @Override
-  void dedupe(Recurring recurring) {
+  void process(Recurring recurring) {
     // Loads the associated DomainBase and BillingEvent.OneTime entities that
     // may have reference to this BillingEvent.Recurring entity.
     Key<DomainBase> domainKey = getGrandParentAsDomain(Key.create(recurring));

core/src/main/java/google/registry/tools/GetSqlCredentialCommand.java

@@ -0,0 +1,73 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.io.FileOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to get a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(separators = " =", commandDescription = "Get the Cloud SQL Credential for a given user")
+public class GetSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"-o", "--output"},
+      description = "Name of output file for key data.",
+      validateWith = PathParameter.OutputFile.class)
+  private Path outputPath = null;
+
+  @Inject
+  GetSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+
+    SqlCredential credential;
+    try {
+      credential = store.getCredential(sqlUser);
+    } catch (SecretManagerException e) {
+      System.out.println(e.getMessage());
+      return;
+    }
+
+    if (outputPath == null) {
+      System.out.printf("[%s]\n", credential.toFormattedString());
+      return;
+    }
+    try (FileOutputStream out = new FileOutputStream(outputPath.toFile())) {
+      out.write(credential.toFormattedString().getBytes(StandardCharsets.UTF_8));
+    }
+  }
+}

core/src/main/java/google/registry/tools/ReadEntityFromKeyPathCommand.java

@@ -33,8 +33,13 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.List;
 
-/** Base Command to dedupe entities with duplicate IDs. */
-abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
+/**
+ * Base Command to read entities from Datastore by their key paths retrieved from BigQuery.
+ *
+ * <p>The key path is the value of column __key__.path of the entity's BigQuery table. Its value is
+ * converted from the entity's key.
+ */
+abstract class ReadEntityFromKeyPathCommand<T> extends MutatingCommand {
 
   @Parameter(
       names = "--key_paths_file",
@@ -48,7 +53,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
 
   private StringBuilder changeMessage = new StringBuilder();
 
-  abstract void dedupe(T entity);
+  abstract void process(T entity);
 
   @Override
   protected void init() throws Exception {
@@ -69,7 +74,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
       }
       Class<T> clazz = new TypeInstantiator<T>(getClass()) {}.getExactType();
       if (clazz.isInstance(entity)) {
-        dedupe((T) entity);
+        process((T) entity);
       } else {
         throw new IllegalArgumentException("Unsupported entity key: " + untypedKey);
       }

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
+import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 
@@ -32,6 +33,7 @@ public final class RegistryTool {
       new ImmutableMap.Builder<String, Class<? extends Command>>()
           .put("ack_poll_messages", AckPollMessagesCommand.class)
           .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
+          .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
@@ -78,6 +80,7 @@ public final class RegistryTool {
           .put("get_routing_map", GetRoutingMapCommand.class)
           .put("get_schema", GetSchemaCommand.class)
           .put("get_schema_tree", GetSchemaTreeCommand.class)
+          .put("get_sql_credential", GetSqlCredentialCommand.class)
           .put("get_tld", GetTldCommand.class)
           .put("ghostryde", GhostrydeCommand.class)
           .put("hash_certificate", HashCertificateCommand.class)
@@ -99,12 +102,15 @@ public final class RegistryTool {
           .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)
           .put("remove_ip_address", RemoveIpAddressCommand.class)
+          .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class)
           .put("renew_domain", RenewDomainCommand.class)
           .put("resave_entities", ResaveEntitiesCommand.class)
           .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
           .put("resave_epp_resource", ResaveEppResourceCommand.class)
+          .put("save_sql_credential", SaveSqlCredentialCommand.class)
           .put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class)
           .put("set_num_instances", SetNumInstancesCommand.class)
+          .put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class)
           .put("setup_ote", SetupOteCommand.class)
           .put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class)
           .put("unlock_domain", UnlockDomainCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -34,6 +34,7 @@ import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.NomulusToolJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
@@ -74,6 +75,7 @@ import javax.inject.Singleton;
       PersistenceModule.class,
       RdeModule.class,
       RequestFactoryModule.class,
+      SecretManagerModule.class,
       URLFetchServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
@@ -118,6 +120,8 @@ interface RegistryToolComponent {
 
   void inject(GetOperationStatusCommand command);
 
+  void inject(GetSqlCredentialCommand command);
+
   void inject(GhostrydeCommand command);
 
   void inject(ImportDatastoreCommand command);
@@ -138,10 +142,14 @@ interface RegistryToolComponent {
 
   void inject(RenewDomainCommand command);
 
+  void inject(SaveSqlCredentialCommand command);
+
   void inject(SendEscrowReportToIcannCommand command);
 
   void inject(SetNumInstancesCommand command);
 
+  void inject(SetSqlReplayCheckpointCommand command);
+
   void inject(SetupOteCommand command);
 
   void inject(UnlockDomainCommand command);

core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameters;
+import com.googlecode.objectify.Key;
+import google.registry.model.domain.DomainBase;
+import google.registry.persistence.VKey;
+import javax.annotation.Nullable;
+import org.joda.time.DateTime;
+
+/** Command to remove the Registry 1.0 key in {@link DomainBase} entity. */
+@Parameters(separators = " =", commandDescription = "Remove .")
+public class RemoveRegistryOneKeyCommand extends ReadEntityFromKeyPathCommand<DomainBase> {
+
+  @Override
+  void process(DomainBase entity) {
+    // Assert that the DomainBase entity must be deleted before 2017-08-01(most of the problematic
+    // entities were deleted before 2017, though there are still a few entities deleted in 2017-07).
+    // This is because we finished the Registry 2.0 migration in 2017 and should not generate any
+    // Registry 1.0 key after it.
+    if (!entity.getDeletionTime().isBefore(DateTime.parse("2017-08-01T00:00:00Z"))) {
+      throw new IllegalStateException(
+          String.format(
+              "Entity's deletion time %s is not before 2017-08-01T00:00:00Z",
+              entity.getDeletionTime()));
+    }
+    boolean hasChange = false;
+    DomainBase.Builder domainBuilder = entity.asBuilder();
+    // We only found the registry 1.0 key existed in fields autorenewBillingEvent,
+    // autorenewPollMessage and deletePollMessage so we just need to check these fields for each
+    // entity.
+    if (isRegistryOneKey(entity.getAutorenewBillingEvent())) {
+      domainBuilder.setAutorenewBillingEvent(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getAutorenewPollMessage())) {
+      domainBuilder.setAutorenewPollMessage(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getDeletePollMessage())) {
+      domainBuilder.setDeletePollMessage(null);
+      hasChange = true;
+    }
+    if (hasChange) {
+      stageEntityChange(entity, domainBuilder.build());
+    }
+  }
+
+  private static boolean isRegistryOneKey(@Nullable VKey<?> vKey) {
+    if (vKey == null || vKey.getOfyKey() == null || vKey.getOfyKey().getParent() == null) {
+      return false;
+    }
+    Key<?> parentKey = vKey.getOfyKey().getParent();
+    return parentKey.getKind().equals("EntityGroupRoot") && parentKey.getName().equals("per-tld");
+  }
+}

core/src/main/java/google/registry/tools/SaveSqlCredentialCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to create or update a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(
+    separators = " =",
+    commandDescription = "Create or update the Cloud SQL Credential for a given user")
+public class SaveSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"--input"},
+      description =
+          "Name of input file for the password. If absent, command will prompt for "
+              + "password in console.",
+      validateWith = PathParameter.InputFile.class)
+  private Path inputPath = null;
+
+  @Inject
+  SaveSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    String password = getPassword();
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+    store.createOrUpdateCredential(sqlUser, password);
+    System.out.printf("\nDone:[%s]\n", password);
+  }
+
+  private String getPassword() throws Exception {
+    if (inputPath != null) {
+      return Files.readAllLines(inputPath, StandardCharsets.UTF_8).get(0);
+    }
+    return System.console().readLine("Please enter the password: ").trim();
+  }
+}

core/src/main/java/google/registry/tools/SetSqlReplayCheckpointCommand.java

@@ -0,0 +1,48 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.Iterables;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import java.util.List;
+import org.joda.time.DateTime;
+
+/** Command to set {@link SqlReplayCheckpoint} to a particular, post-initial-population time. */
+@Parameters(separators = " =", commandDescription = "Set SqlReplayCheckpoint to a particular time")
+public class SetSqlReplayCheckpointCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  @Parameter(description = "Time to which SqlReplayCheckpoint will be set", required = true)
+  List<DateTime> mainParameters;
+
+  @Override
+  protected String prompt() {
+    checkArgument(mainParameters.size() == 1, "Must provide exactly one DateTime to set");
+    return String.format(
+        "Set SqlReplayCheckpoint to %s?", Iterables.getOnlyElement(mainParameters));
+  }
+
+  @Override
+  protected String execute() {
+    DateTime dateTime = Iterables.getOnlyElement(mainParameters);
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime));
+    return String.format("Set SqlReplayCheckpoint time to %s", dateTime);
+  }
+}

core/src/main/java/google/registry/tools/SetupOteCommand.java

@@ -65,13 +65,6 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo
       validateWith = PathParameter.InputFile.class)
   private Path certFile;
 
-  @Parameter(
-      names = {"-h", "--certhash"},
-      description =
-          "Hash of client certificate (SHA256 base64 no padding). Do not use this unless "
-              + "you want to store ONLY the hash and not the full certificate.")
-  private String certHash;
-
   @Parameter(
       names = {"--overwrite"},
       description = "Whether to replace existing entities if we encounter any, instead of failing.")
@@ -89,9 +82,7 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo
   /** Run any pre-execute command checks */
   @Override
   protected void init() throws Exception {
-    checkArgument(
-        certFile == null ^ certHash == null,
-        "Must specify exactly one of client certificate file or client certificate hash.");
+    checkArgument(certFile != null, "Must specify exactly one client certificate file.");
 
     password = passwordGenerator.createString(PASSWORD_LENGTH);
     oteAccountBuilder =
@@ -101,16 +92,10 @@ final class SetupOteCommand extends ConfirmingCommand implements CommandWithRemo
             .setIpAllowList(ipAllowList)
             .setReplaceExisting(overwrite);
 
-    if (certFile != null) {
       String asciiCert = MoreFiles.asCharSource(certFile, US_ASCII).read();
       // Don't wait for create_registrar to fail if it's a bad certificate file.
       loadCertificate(asciiCert);
       oteAccountBuilder.setCertificate(asciiCert, clock.nowUtc());
-    }
-
-    if (certHash != null) {
-      oteAccountBuilder.setCertificateHash(certHash);
-    }
   }
 
   @Override

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -0,0 +1,216 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.beam.spec11.ThreatMatch;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.model.reporting.Spec11ThreatMatchDao;
+import google.registry.reporting.spec11.RegistrarThreatMatches;
+import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.Clock;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+import javax.inject.Inject;
+import org.joda.time.LocalDate;
+
+/**
+ * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
+ *
+ * <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
+ * rough estimate of when we started using this format) and convert those RegistrarThreatMatches
+ * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
+ *
+ * <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
+ * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
+ * or if the domain objects were not active at the time of the scan.
+ */
+@Parameters(
+    commandDescription =
+        "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
+            + "Cloud SQL database.")
+public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
+
+  @Parameter(
+      names = {"-o", "--overwrite_existing_dates"},
+      description =
+          "Whether the command will overwrite data that already exists for dates that exist in the "
+              + "GCS bucket. Defaults to false.")
+  private boolean overrideExistingDates;
+
+  @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
+  // Inject the clock for testing purposes
+  @Inject Clock clock;
+
+  @Override
+  protected String prompt() {
+    return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
+  }
+
+  @Override
+  protected String execute() {
+    ImmutableList<LocalDate> dates = getDatesToBackfill();
+    ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
+        new ImmutableListMultimap.Builder<>();
+    for (LocalDate date : dates) {
+      try {
+        // It's OK if the file doesn't exist for a particular date; the result will be empty.
+        threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
+      } catch (IOException e) {
+        throw new RuntimeException(
+            String.format("Error parsing through file with date %s.", date), e);
+      }
+    }
+    ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
+        threatMatchesBuilder.build();
+    // Look up all possible DomainBases for these domain names, any of which can be in the past
+    ImmutableListMultimap<String, DomainBase> domainsByDomainName =
+        getDomainsByDomainName(threatMatches);
+
+    // For each date, convert all threat matches with the proper domain repo ID
+    int totalNumThreats = 0;
+    for (LocalDate date : threatMatches.keySet()) {
+      ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
+      for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
+        rtm.threatMatches().stream()
+            .map(
+                threatMatch ->
+                    threatMatchToCloudSqlObject(
+                        threatMatch, date, rtm.clientId(), domainsByDomainName))
+            .forEach(spec11ThreatsBuilder::add);
+      }
+      ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
+      jpaTm()
+          .transact(
+              () -> {
+                Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
+                jpaTm().putAll(spec11Threats);
+              });
+      totalNumThreats += spec11Threats.size();
+    }
+    return String.format(
+        "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
+  }
+
+  /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
+  private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
+      ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
+    return threatMatchesByDate.values().stream()
+        .map(RegistrarThreatMatches::threatMatches)
+        .flatMap(ImmutableList::stream)
+        .map(ThreatMatch::fullyQualifiedDomainName)
+        .distinct()
+        .collect(
+            flatteningToImmutableListMultimap(
+                Function.identity(),
+                (domainName) -> {
+                  List<DomainBase> domains =
+                      ofy()
+                          .load()
+                          .type(DomainBase.class)
+                          .filter("fullyQualifiedDomainName", domainName)
+                          .list();
+                  domains.sort(Comparator.comparing(DomainBase::getCreationTime).reversed());
+                  checkState(
+                      !domains.isEmpty(),
+                      "Domain name %s had no associated DomainBase objects.",
+                      domainName);
+                  return domains.stream();
+                }));
+  }
+
+  /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
+  private Spec11ThreatMatch threatMatchToCloudSqlObject(
+      ThreatMatch threatMatch,
+      LocalDate date,
+      String registrarId,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    DomainBase domain =
+        findDomainAsOfDateOrThrow(
+            threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
+    return new Spec11ThreatMatch.Builder()
+        .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
+        .setCheckDate(date)
+        .setRegistrarId(registrarId)
+        .setDomainName(threatMatch.fullyQualifiedDomainName())
+        .setDomainRepoId(domain.getRepoId())
+        .build();
+  }
+
+  /** Returns the DomainBase object as of the particular date, which is likely in the past. */
+  private DomainBase findDomainAsOfDateOrThrow(
+      String domainName,
+      LocalDate date,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
+    for (DomainBase domain : domains) {
+      // We only know the date (not datetime) of the threat scan, so we approximate
+      LocalDate creationDate = domain.getCreationTime().toLocalDate();
+      LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
+      if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
+        return domain;
+      }
+    }
+    throw new IllegalStateException(
+        String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
+  }
+
+  /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
+  private ImmutableList<LocalDate> getDatesToBackfill() {
+    ImmutableSet<LocalDate> datesToSkip =
+        overrideExistingDates ? ImmutableSet.of() : getExistingDates();
+    ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
+    LocalDate endDate = clock.nowUtc().toLocalDate();
+    for (LocalDate currentDate = START_DATE;
+        !currentDate.isAfter(endDate);
+        currentDate = currentDate.plusDays(1)) {
+      if (!datesToSkip.contains(currentDate)) {
+        result.add(currentDate);
+      }
+    }
+    return result.build();
+  }
+
+  private ImmutableSet<LocalDate> getExistingDates() {
+    return jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .getEntityManager()
+                    .createQuery(
+                        "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
+                    .getResultStream()
+                    .collect(toImmutableSet()));
+  }
+}

core/src/test/java/google/registry/backup/DeleteOldCommitLogsActionTest.java

@@ -95,7 +95,7 @@ public class DeleteOldCommitLogsActionTest
     // Before deleting the unneeded manifests - we have 11 of them (one for the first
     // creation, and 10 more for the mutateContacts)
     assertThat(ofy().load().type(CommitLogManifest.class).count()).isEqualTo(11);
-    // And each DatastoreHelper.persistResourceWithCommitLog creates 3 mutations
+    // And each DatabaseHelper.persistResourceWithCommitLog creates 3 mutations
     assertThat(ofy().load().type(CommitLogMutation.class).count()).isEqualTo(33);
   }
 
@@ -111,7 +111,7 @@ public class DeleteOldCommitLogsActionTest
     assertThat(ImmutableList.copyOf(ofy().load().type(CommitLogManifest.class).keys().iterable()))
         .containsExactlyElementsIn(contact.getRevisions().values());
 
-    // And each DatastoreHelper.persistResourceWithCommitLog creates 3 mutations
+    // And each DatabaseHelper.persistResourceWithCommitLog creates 3 mutations
     assertThat(ofyLoadType(CommitLogMutation.class)).hasSize(contact.getRevisions().size() * 3);
   }
 

core/src/test/java/google/registry/backup/ReplayCommitLogsToSqlActionTest.java

@@ -0,0 +1,458 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.backup;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
+import static google.registry.backup.RestoreCommitLogsActionTest.GCS_BUCKET;
+import static google.registry.backup.RestoreCommitLogsActionTest.createCheckpoint;
+import static google.registry.backup.RestoreCommitLogsActionTest.saveDiffFile;
+import static google.registry.backup.RestoreCommitLogsActionTest.saveDiffFileNotToRestore;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.model.ofy.CommitLogBucket.getBucketKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistActiveContact;
+import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.truth.Truth8;
+import com.googlecode.objectify.Key;
+import google.registry.config.RegistryConfig;
+import google.registry.model.common.Cursor;
+import google.registry.model.common.Cursor.CursorType;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.secdns.DelegationSignerData;
+import google.registry.model.ofy.CommitLogBucket;
+import google.registry.model.ofy.CommitLogManifest;
+import google.registry.model.ofy.CommitLogMutation;
+import google.registry.model.registrar.RegistrarContact;
+import google.registry.model.registry.label.ReservedList;
+import google.registry.model.server.Lock;
+import google.registry.model.tmch.ClaimsListShard;
+import google.registry.model.translators.VKeyTranslatorFactory;
+import google.registry.persistence.VKey;
+import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.FakeClock;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.TestObject;
+import google.registry.util.RequestStatusChecker;
+import java.io.IOException;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+/** Tests for {@link ReplayCommitLogsToSqlAction}. */
+@ExtendWith(MockitoExtension.class)
+public class ReplayCommitLogsToSqlActionTest {
+
+  private final FakeClock fakeClock = new FakeClock(DateTime.parse("2000-01-01TZ"));
+
+  @RegisterExtension
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withDatastoreAndCloudSql()
+          .withClock(fakeClock)
+          .withOfyTestEntities(TestObject.class)
+          .withJpaUnitTestEntities(
+              RegistrarContact.class,
+              TestObject.class,
+              SqlReplayCheckpoint.class,
+              ContactResource.class,
+              DomainBase.class,
+              GracePeriod.class,
+              DelegationSignerData.class)
+          .build();
+
+  /** Local GCS service. */
+  private final GcsService gcsService = GcsServiceFactory.createGcsService();
+
+  private final ReplayCommitLogsToSqlAction action = new ReplayCommitLogsToSqlAction();
+  private final FakeResponse response = new FakeResponse();
+  @Mock private RequestStatusChecker requestStatusChecker;
+
+  @BeforeAll
+  static void beforeAll() {
+    VKeyTranslatorFactory.addTestEntityClass(TestObject.class);
+  }
+
+  @BeforeEach
+  void beforeEach() {
+    action.gcsService = gcsService;
+    action.response = response;
+    action.requestStatusChecker = requestStatusChecker;
+    action.diffLister = new GcsDiffFileLister();
+    action.diffLister.gcsService = gcsService;
+    action.diffLister.gcsBucket = GCS_BUCKET;
+    action.diffLister.executor = newDirectExecutorService();
+    RegistryConfig.overrideCloudSqlReplayCommitLogs(true);
+  }
+
+  @Test
+  void testReplay_multipleDiffFiles() throws Exception {
+    jpaTm()
+        .transact(
+            () -> {
+              jpaTm().insertWithoutBackup(TestObject.create("previous to keep"));
+              jpaTm().insertWithoutBackup(TestObject.create("previous to delete"));
+            });
+    DateTime now = fakeClock.nowUtc();
+    // Create 3 transactions, across two diff files.
+    // Before: {"previous to keep", "previous to delete"}
+    // 1a: Add {"a", "b"}, Delete {"previous to delete"}
+    // 1b: Add {"c", "d"}, Delete {"a"}
+    // 2:  Add {"e", "f"}, Delete {"c"}
+    // After:  {"previous to keep", "b", "d", "e", "f"}
+    Key<CommitLogManifest> manifest1aKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(3));
+    Key<CommitLogManifest> manifest1bKey =
+        CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
+    Key<CommitLogManifest> manifest2Key =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now.minusMinutes(3),
+            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))),
+        CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
+        CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
+        CommitLogManifest.create(
+            getBucketKey(2),
+            now.minusMinutes(2),
+            ImmutableSet.of(Key.create(TestObject.create("a")))),
+        CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
+        CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now.minusMinutes(1),
+            ImmutableSet.of(Key.create(TestObject.create("c")))),
+        CommitLogMutation.create(manifest2Key, TestObject.create("e")),
+        CommitLogMutation.create(manifest2Key, TestObject.create("f")));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    fakeClock.advanceOneMilli();
+    runAndAssertSuccess(now);
+    assertExpectedIds("previous to keep", "b", "d", "e", "f");
+  }
+
+  @Test
+  void testReplay_noManifests() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    saveDiffFile(gcsService, createCheckpoint(now.minusMillis(2)));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMillis(1)));
+    runAndAssertSuccess(now.minusMillis(1));
+    assertExpectedIds("previous to keep");
+  }
+
+  @Test
+  void testReplay_manifestWithNoDeletions() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().insertWithoutBackup(TestObject.create("previous to keep")));
+    Key<CommitLogBucket> bucketKey = getBucketKey(1);
+    Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(bucketKey, now, null),
+        CommitLogMutation.create(manifestKey, TestObject.create("a")),
+        CommitLogMutation.create(manifestKey, TestObject.create("b")));
+    runAndAssertSuccess(now.minusMinutes(1));
+    assertExpectedIds("previous to keep", "a", "b");
+  }
+
+  @Test
+  void testReplay_manifestWithNoMutations() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm()
+        .transact(
+            () -> {
+              jpaTm().insertWithoutBackup(TestObject.create("previous to keep"));
+              jpaTm().insertWithoutBackup(TestObject.create("previous to delete"));
+            });
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now,
+            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    runAndAssertSuccess(now.minusMinutes(1));
+    assertExpectedIds("previous to keep");
+  }
+
+  @Test
+  void wtestReplay_mutateExistingEntity() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().put(TestObject.create("existing", "a")));
+    Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMillis(1)),
+        CommitLogManifest.create(getBucketKey(1), now, null),
+        CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
+    action.run();
+    TestObject fromDatabase =
+        jpaTm().transact(() -> jpaTm().load(VKey.createSql(TestObject.class, "existing")));
+    assertThat(fromDatabase.getField()).isEqualTo("b");
+  }
+
+  // This should be harmless
+  @Test
+  void testReplay_deleteMissingEntity() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm().transact(() -> jpaTm().put(TestObject.create("previous to keep", "a")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1).minusMillis(1));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1)));
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMillis(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now,
+            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    action.run();
+    assertExpectedIds("previous to keep");
+  }
+
+  @Test
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  void testReplay_properlyWeighted() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    Key<CommitLogManifest> manifestKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+    // Create (but don't save to SQL) a domain + contact
+    createTld("tld");
+    DomainBase domain = newDomainBase("example.tld");
+    CommitLogMutation domainMutation =
+        tm().transact(() -> CommitLogMutation.create(manifestKey, domain));
+    ContactResource contact = tm().transact(() -> tm().load(domain.getRegistrant()));
+    CommitLogMutation contactMutation =
+        tm().transact(() -> CommitLogMutation.create(manifestKey, contact));
+
+    // Create and save to SQL a registrar contact that we will delete
+    RegistrarContact toDelete = AppEngineExtension.makeRegistrarContact1();
+    jpaTm().transact(() -> jpaTm().put(toDelete));
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+
+    // spy the txn manager so we can see what order things were inserted/removed
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+    // Save in the commit logs the domain and contact (in that order) and the token deletion
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(toDelete))),
+        domainMutation,
+        contactMutation);
+
+    runAndAssertSuccess(now.minusMinutes(1));
+    // Verify two things:
+    // 1. that the contact insert occurred before the domain insert (necessary for FK ordering)
+    //    even though the domain came first in the file
+    // 2. that the allocation token delete occurred after the insertions
+    InOrder inOrder = Mockito.inOrder(spy);
+    inOrder.verify(spy).put(any(ContactResource.class));
+    inOrder.verify(spy).put(any(DomainBase.class));
+    inOrder.verify(spy).delete(toDelete.createVKey());
+    inOrder.verify(spy).put(any(SqlReplayCheckpoint.class));
+  }
+
+  @Test
+  @SuppressWarnings({"unchecked", "rawtypes"})
+  void testReplay_properlyWeighted_doesNotApplyCrossTransactions() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    Key<CommitLogManifest> manifestKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+
+    // Create and save the standard contact
+    ContactResource contact = persistActiveContact("contact1234");
+    jpaTm().transact(() -> jpaTm().put(contact));
+
+    // Simulate a Datastore transaction with a new version of the contact
+    ContactResource contactWithEdit =
+        contact.asBuilder().setEmailAddress("replay@example.tld").build();
+    CommitLogMutation contactMutation =
+        tm().transact(() -> CommitLogMutation.create(manifestKey, contactWithEdit));
+
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+
+    // spy the txn manager so we can see what order things were inserted
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+    // Save two commits -- the deletion, then the new version of the contact
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1).plusMillis(1)),
+        CommitLogManifest.create(
+            getBucketKey(1), now.minusMinutes(1), ImmutableSet.of(Key.create(contact))),
+        CommitLogManifest.create(
+            getBucketKey(1), now.minusMinutes(1).plusMillis(1), ImmutableSet.of()),
+        contactMutation);
+    runAndAssertSuccess(now.minusMinutes(1).plusMillis(1));
+    // Verify that the delete occurred first (because it was in the first transaction) even though
+    // deletes have higher weight
+    ArgumentCaptor<Object> putCaptor = ArgumentCaptor.forClass(Object.class);
+    InOrder inOrder = Mockito.inOrder(spy);
+    inOrder.verify(spy).delete(contact.createVKey());
+    inOrder.verify(spy).put(putCaptor.capture());
+    assertThat(putCaptor.getValue().getClass()).isEqualTo(ContactResource.class);
+    assertThat(jpaTm().transact(() -> jpaTm().load(contact.createVKey()).getEmailAddress()))
+        .isEqualTo("replay@example.tld");
+  }
+
+  @Test
+  void testSuccess_nonReplicatedEntity_isNotReplayed() {
+    DateTime now = fakeClock.nowUtc();
+
+    // spy the txn manager so we can verify it's never called
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    Key<CommitLogManifest> manifestKey =
+        CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
+
+    // Have a commit log with a couple objects that shouldn't be replayed
+    ReservedList reservedList =
+        new ReservedList.Builder().setReservedListMap(ImmutableMap.of()).setName("name").build();
+    Cursor cursor = Cursor.createGlobal(CursorType.RECURRING_BILLING, now.minusHours(1));
+    tm().transact(
+            () -> {
+              try {
+                saveDiffFile(
+                    gcsService,
+                    createCheckpoint(now.minusMinutes(1)),
+                    CommitLogManifest.create(
+                        getBucketKey(1), now.minusMinutes(1), ImmutableSet.of()),
+                    // Reserved list is dually-written non-replicated
+                    CommitLogMutation.create(manifestKey, reservedList),
+                    // Cursors aren't replayed to SQL at all
+                    CommitLogMutation.create(manifestKey, cursor));
+              } catch (IOException e) {
+                throw new RuntimeException(e);
+              }
+            });
+    runAndAssertSuccess(now.minusMinutes(1));
+    // jpaTm()::put should only have been called with the checkpoint
+    verify(spy, times(2)).put(any(SqlReplayCheckpoint.class));
+    verify(spy, times(2)).put(any());
+  }
+
+  @Test
+  void testSuccess_nonReplicatedEntity_isNotDeleted() throws Exception {
+    DateTime now = fakeClock.nowUtc();
+    // spy the txn manager so we can verify it's never called
+    JpaTransactionManager spy = spy(jpaTm());
+    TransactionManagerFactory.setJpaTm(() -> spy);
+
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(now.minusMinutes(1).minusMillis(1)));
+    // Save a couple deletes that aren't propagated to SQL (the objects deleted are irrelevant)
+    Key<ClaimsListShard> claimsListKey = Key.create(ClaimsListShard.class, 1L);
+    saveDiffFile(
+        gcsService,
+        createCheckpoint(now.minusMinutes(1)),
+        CommitLogManifest.create(
+            getBucketKey(1),
+            now.minusMinutes(1),
+            // one object only exists in Datastore, one is dually-written (so isn't replicated)
+            ImmutableSet.of(getCrossTldKey(), claimsListKey)));
+
+    runAndAssertSuccess(now.minusMinutes(1));
+    verify(spy, times(0)).delete(any(VKey.class));
+  }
+
+  @Test
+  void testFailure_notEnabled() {
+    RegistryConfig.overrideCloudSqlReplayCommitLogs(false);
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
+    assertThat(response.getPayload())
+        .isEqualTo("ReplayCommitLogsToSqlAction was called but disabled in the config.");
+  }
+
+  @Test
+  void testFailure_cannotAcquireLock() {
+    Truth8.assertThat(
+            Lock.acquire(
+                ReplayCommitLogsToSqlAction.class.getSimpleName(),
+                null,
+                Duration.standardHours(1),
+                requestStatusChecker,
+                false))
+        .isPresent();
+    fakeClock.advanceOneMilli();
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_NO_CONTENT);
+    assertThat(response.getPayload())
+        .isEqualTo("Can't acquire SQL commit log replay lock, aborting.");
+  }
+
+  private void runAndAssertSuccess(DateTime expectedCheckpointTime) {
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(expectedCheckpointTime);
+  }
+
+  private void assertExpectedIds(String... expectedIds) {
+    ImmutableList<String> actualIds =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAll(TestObject.class).stream()
+                        .map(TestObject::getId)
+                        .collect(toImmutableList()));
+    assertThat(actualIds).containsExactlyElementsIn(expectedIds);
+  }
+}

core/src/test/java/google/registry/backup/RestoreCommitLogsActionTest.java

@@ -61,7 +61,7 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 /** Unit tests for {@link RestoreCommitLogsAction}. */
 public class RestoreCommitLogsActionTest {
 
-  private static final String GCS_BUCKET = "gcs bucket";
+  static final String GCS_BUCKET = "gcs bucket";
 
   private final DateTime now = DateTime.now(UTC);
   private final RestoreCommitLogsAction action = new RestoreCommitLogsAction();
@@ -89,9 +89,10 @@ public class RestoreCommitLogsActionTest {
 
   @Test
   void testRestore_multipleDiffFiles() throws Exception {
-    ofy().saveWithoutBackup().entities(
-        TestObject.create("previous to keep"),
-        TestObject.create("previous to delete")).now();
+    ofy()
+        .saveWithoutBackup()
+        .entities(TestObject.create("previous to keep"), TestObject.create("previous to delete"))
+        .now();
     // Create 3 transactions, across two diff files.
     // Before: {"previous to keep", "previous to delete"}
     // 1a: Add {"a", "b"}, Delete {"previous to delete"}
@@ -104,29 +105,33 @@ public class RestoreCommitLogsActionTest {
         CommitLogManifest.createKey(getBucketKey(2), now.minusMinutes(2));
     Key<CommitLogManifest> manifest2Key =
         CommitLogManifest.createKey(getBucketKey(1), now.minusMinutes(1));
-    saveDiffFileNotToRestore(now.minusMinutes(2));
-    Iterable<ImmutableObject> file1CommitLogs = saveDiffFile(
-        createCheckpoint(now.minusMinutes(1)),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now.minusMinutes(3),
-            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))),
-        CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
-        CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
-        CommitLogManifest.create(
-            getBucketKey(2),
-            now.minusMinutes(2),
-            ImmutableSet.of(Key.create(TestObject.create("a")))),
-        CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
-        CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
-    Iterable<ImmutableObject> file2CommitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now.minusMinutes(1),
-            ImmutableSet.of(Key.create(TestObject.create("c")))),
-        CommitLogMutation.create(manifest2Key, TestObject.create("e")),
-        CommitLogMutation.create(manifest2Key, TestObject.create("f")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(2));
+    Iterable<ImmutableObject> file1CommitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now.minusMinutes(1)),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now.minusMinutes(3),
+                ImmutableSet.of(Key.create(TestObject.create("previous to delete")))),
+            CommitLogMutation.create(manifest1aKey, TestObject.create("a")),
+            CommitLogMutation.create(manifest1aKey, TestObject.create("b")),
+            CommitLogManifest.create(
+                getBucketKey(2),
+                now.minusMinutes(2),
+                ImmutableSet.of(Key.create(TestObject.create("a")))),
+            CommitLogMutation.create(manifest1bKey, TestObject.create("c")),
+            CommitLogMutation.create(manifest1bKey, TestObject.create("d")));
+    Iterable<ImmutableObject> file2CommitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now.minusMinutes(1),
+                ImmutableSet.of(Key.create(TestObject.create("c")))),
+            CommitLogMutation.create(manifest2Key, TestObject.create("e")),
+            CommitLogMutation.create(manifest2Key, TestObject.create("f")));
     action.fromTime = now.minusMinutes(1).minusMillis(1);
     action.run();
     ofy().clearSessionCache();
@@ -139,10 +144,9 @@ public class RestoreCommitLogsActionTest {
 
   @Test
   void testRestore_noManifests() throws Exception {
-    ofy().saveWithoutBackup().entity(
-        TestObject.create("previous to keep")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(createCheckpoint(now));
+    ofy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs = saveDiffFile(gcsService, createCheckpoint(now));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -156,32 +160,37 @@ public class RestoreCommitLogsActionTest {
     ofy().saveWithoutBackup().entity(TestObject.create("previous to keep")).now();
     Key<CommitLogBucket> bucketKey = getBucketKey(1);
     Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(bucketKey, now);
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(bucketKey, now, null),
-        CommitLogMutation.create(manifestKey, TestObject.create("a")),
-        CommitLogMutation.create(manifestKey, TestObject.create("b")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(bucketKey, now, null),
+            CommitLogMutation.create(manifestKey, TestObject.create("a")),
+            CommitLogMutation.create(manifestKey, TestObject.create("b")));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep", "a", "b");
     assertInDatastore(commitLogs);
     assertInDatastore(CommitLogCheckpointRoot.create(now));
     assertCommitLogBuckets(ImmutableMap.of(1, now));
-}
+  }
 
   @Test
   void testRestore_manifestWithNoMutations() throws Exception {
-    ofy().saveWithoutBackup().entities(
-        TestObject.create("previous to keep"),
-        TestObject.create("previous to delete")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now,
-            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    ofy()
+        .saveWithoutBackup()
+        .entities(TestObject.create("previous to keep"), TestObject.create("previous to delete"))
+        .now();
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now,
+                ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -193,12 +202,13 @@ public class RestoreCommitLogsActionTest {
   // This is a pathological case that shouldn't be possible, but we should be robust to it.
   @Test
   void testRestore_manifestWithNoMutationsOrDeletions() throws Exception {
-    ofy().saveWithoutBackup().entities(
-        TestObject.create("previous to keep")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(getBucketKey(1), now, null));
+    ofy().saveWithoutBackup().entities(TestObject.create("previous to keep")).now();
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(getBucketKey(1), now, null));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -211,11 +221,13 @@ public class RestoreCommitLogsActionTest {
   void testRestore_mutateExistingEntity() throws Exception {
     ofy().saveWithoutBackup().entity(TestObject.create("existing", "a")).now();
     Key<CommitLogManifest> manifestKey = CommitLogManifest.createKey(getBucketKey(1), now);
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(getBucketKey(1), now, null),
-        CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(getBucketKey(1), now, null),
+            CommitLogMutation.create(manifestKey, TestObject.create("existing", "b")));
     action.run();
     ofy().clearSessionCache();
     assertThat(ofy().load().entity(TestObject.create("existing")).now().getField()).isEqualTo("b");
@@ -228,13 +240,15 @@ public class RestoreCommitLogsActionTest {
   @Test
   void testRestore_deleteMissingEntity() throws Exception {
     ofy().saveWithoutBackup().entity(TestObject.create("previous to keep", "a")).now();
-    saveDiffFileNotToRestore(now.minusMinutes(1));
-    Iterable<ImmutableObject> commitLogs = saveDiffFile(
-        createCheckpoint(now),
-        CommitLogManifest.create(
-            getBucketKey(1),
-            now,
-            ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
+    saveDiffFileNotToRestore(gcsService, now.minusMinutes(1));
+    Iterable<ImmutableObject> commitLogs =
+        saveDiffFile(
+            gcsService,
+            createCheckpoint(now),
+            CommitLogManifest.create(
+                getBucketKey(1),
+                now,
+                ImmutableSet.of(Key.create(TestObject.create("previous to delete")))));
     action.run();
     ofy().clearSessionCache();
     assertExpectedIds("previous to keep");
@@ -243,12 +257,13 @@ public class RestoreCommitLogsActionTest {
     assertInDatastore(CommitLogCheckpointRoot.create(now));
   }
 
-  private CommitLogCheckpoint createCheckpoint(DateTime now) {
+  static CommitLogCheckpoint createCheckpoint(DateTime now) {
     return CommitLogCheckpoint.create(now, toMap(getBucketIds(), x -> now));
   }
 
-  private Iterable<ImmutableObject> saveDiffFile(
-      CommitLogCheckpoint checkpoint, ImmutableObject... entities) throws IOException {
+  static Iterable<ImmutableObject> saveDiffFile(
+      GcsService gcsService, CommitLogCheckpoint checkpoint, ImmutableObject... entities)
+      throws IOException {
     DateTime now = checkpoint.getCheckpointTime();
     List<ImmutableObject> allEntities = Lists.asList(checkpoint, entities);
     ByteArrayOutputStream output = new ByteArrayOutputStream();
@@ -264,8 +279,9 @@ public class RestoreCommitLogsActionTest {
     return allEntities;
   }
 
-  private void saveDiffFileNotToRestore(DateTime now) throws Exception {
+  static void saveDiffFileNotToRestore(GcsService gcsService, DateTime now) throws Exception {
     saveDiffFile(
+        gcsService,
         createCheckpoint(now),
         CommitLogManifest.create(getBucketKey(1), now, null),
         CommitLogMutation.create(

core/src/test/java/google/registry/beam/initsql/WriteToSqlTest.java

@@ -29,8 +29,8 @@ import google.registry.model.registrar.Registrar;
 import google.registry.persistence.transaction.JpaTestRules;
 import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationTestExtension;
 import google.registry.testing.AppEngineExtension;
-import google.registry.testing.DatastoreEntityExtension;
 import google.registry.testing.DatabaseHelper;
+import google.registry.testing.DatastoreEntityExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.InjectExtension;
 import java.io.Serializable;

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -20,7 +20,6 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.withSettings;
 
@@ -285,8 +284,7 @@ class Spec11PipelineTest {
             .build();
 
     verify(mockJpaTm).transact(any(Runnable.class));
-    verify(mockJpaTm).insert(expected);
-    verifyNoMoreInteractions(mockJpaTm);
+    verify(mockJpaTm).putAll(ImmutableList.of(expected));
   }
 
   /**

core/src/test/java/google/registry/flows/EppLifecycleDomainTest.java

@@ -501,7 +501,7 @@ class EppLifecycleDomainTest extends EppTestCase {
 
   @Test
   void testEapDomainDeletion_withinAddGracePeriod_eapFeeIsNotRefunded() throws Exception {
-    assertThatCommand("login_valid_fee_extension.xml").hasResponse("generic_success_response.xml");
+    assertThatCommand("login_valid_fee_extension.xml").hasSuccessfulLogin();
     createContacts(DateTime.parse("2000-06-01T00:00:00Z"));
 
     // Set the EAP schedule.
@@ -718,7 +718,7 @@ class EppLifecycleDomainTest extends EppTestCase {
             START_OF_TIME, PREDELEGATION,
             gaDate, GENERAL_AVAILABILITY));
 
-    assertThatCommand("login_valid_fee_extension.xml").hasResponse("generic_success_response.xml");
+    assertThatCommand("login_valid_fee_extension.xml").hasSuccessfulLogin();
 
     assertThatCommand("domain_check_fee_premium.xml")
         .atTime(gaDate.plusDays(1))
@@ -1196,7 +1196,7 @@ class EppLifecycleDomainTest extends EppTestCase {
 
     assertThatLogin("NewRegistrar", "foo-BAR2")
         .atTime(sunriseDate.minusDays(3))
-        .hasResponse("generic_success_response.xml");
+        .hasSuccessfulLogin();
 
     createContactsAndHosts();
 
@@ -1292,7 +1292,7 @@ class EppLifecycleDomainTest extends EppTestCase {
 
     assertThatLogin("NewRegistrar", "foo-BAR2")
         .atTime(sunriseDate.minusDays(3))
-        .hasResponse("generic_success_response.xml");
+        .hasSuccessfulLogin();
 
     createContactsAndHosts();
 

core/src/test/java/google/registry/flows/EppLoginTlsTest.java

@@ -44,13 +44,13 @@ class EppLoginTlsTest extends EppTestCase {
     persistResource(
         loadRegistrar("NewRegistrar")
             .asBuilder()
-            .setClientCertificateHash(CertificateSamples.SAMPLE_CERT_HASH)
+            .setClientCertificate(CertificateSamples.SAMPLE_CERT, DateTime.now(UTC))
             .build());
     // Set a cert for the second registrar, or else any cert will be allowed for login.
     persistResource(
         loadRegistrar("TheRegistrar")
             .asBuilder()
-            .setClientCertificateHash(CertificateSamples.SAMPLE_CERT2_HASH)
+            .setClientCertificate(CertificateSamples.SAMPLE_CERT2, DateTime.now(UTC))
             .build());
   }
 

core/src/test/java/google/registry/flows/EppTestCase.java

@@ -121,6 +121,10 @@ public class EppTestCase {
       return assertCommandAndResponse(
           inputFilename, inputSubstitutions, outputFilename, outputSubstitutions, now);
     }
+
+    public String hasSuccessfulLogin() throws Exception {
+      return assertLoginCommandAndResponse(inputFilename, inputSubstitutions, null, now);
+    }
   }
 
   protected CommandAsserter assertThatCommand(String inputFilename) {
@@ -137,13 +141,33 @@ public class EppTestCase {
   }
 
   protected void assertThatLoginSucceeds(String clientId, String password) throws Exception {
-    assertThatLogin(clientId, password).hasResponse("generic_success_response.xml");
+    assertThatLogin(clientId, password).hasSuccessfulLogin();
   }
 
   protected void assertThatLogoutSucceeds() throws Exception {
     assertThatCommand("logout.xml").hasResponse("logout_response.xml");
   }
 
+  private String assertLoginCommandAndResponse(
+      String inputFilename,
+      @Nullable Map<String, String> inputSubstitutions,
+      @Nullable Map<String, String> outputSubstitutions,
+      DateTime now)
+      throws Exception {
+    String outputFilename = "generic_success_response.xml";
+    clock.setTo(now);
+    String input = loadFile(EppTestCase.class, inputFilename, inputSubstitutions);
+    String expectedOutput = loadFile(EppTestCase.class, outputFilename, outputSubstitutions);
+    setUpSession();
+    FakeResponse response = executeXmlCommand(input);
+
+    // Check that the logged-in header was added to the response
+    assertThat(response.getHeaders()).isEqualTo(ImmutableMap.of("Logged-In", "true"));
+
+    return verifyAndReturnOutput(
+        response.getPayload(), expectedOutput, inputFilename, outputFilename);
+  }
+
   private String assertCommandAndResponse(
       String inputFilename,
       @Nullable Map<String, String> inputSubstitutions,
@@ -154,6 +178,18 @@ public class EppTestCase {
     clock.setTo(now);
     String input = loadFile(EppTestCase.class, inputFilename, inputSubstitutions);
     String expectedOutput = loadFile(EppTestCase.class, outputFilename, outputSubstitutions);
+    setUpSession();
+    FakeResponse response = executeXmlCommand(input);
+
+    // Checks that the Logged-In header is not in the response. If testing the login command, use
+    // assertLoginCommandAndResponse instead of this method.
+    assertThat(response.getHeaders()).doesNotContainEntry("Logged-In", "true");
+
+    return verifyAndReturnOutput(
+        response.getPayload(), expectedOutput, inputFilename, outputFilename);
+  }
+
+  private void setUpSession() {
     if (sessionMetadata == null) {
       sessionMetadata =
           new HttpSessionMetadata(new FakeHttpSession()) {
@@ -165,7 +201,13 @@ public class EppTestCase {
             }
           };
     }
-    String actualOutput = executeXmlCommand(input);
+  }
+
+  private String verifyAndReturnOutput(
+      String actualOutput, String expectedOutput, String inputFilename, String outputFilename)
+      throws Exception {
+    // Run the resulting xml through the unmarshaller to verify that it was valid.
+    EppXmlTransformer.validateOutput(actualOutput);
     assertXmlEqualsWithMessage(
         expectedOutput,
         actualOutput,
@@ -176,7 +218,7 @@ public class EppTestCase {
     return actualOutput;
   }
 
-  private String executeXmlCommand(String inputXml) throws Exception {
+  private FakeResponse executeXmlCommand(String inputXml) throws Exception {
     EppRequestHandler handler = new EppRequestHandler();
     FakeResponse response = new FakeResponse();
     handler.response = response;
@@ -195,10 +237,7 @@ public class EppTestCase {
         inputXml.getBytes(UTF_8));
     assertThat(response.getStatus()).isEqualTo(SC_OK);
     assertThat(response.getContentType()).isEqualTo(APPLICATION_EPP_XML_UTF8);
-    String result = response.getPayload();
-    // Run the resulting xml through the unmarshaller to verify that it was valid.
-    EppXmlTransformer.validateOutput(result);
-    return result;
+    return response;
   }
 
   EppMetric getRecordedEppMetric() {

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -96,17 +96,24 @@ import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferResponse;
 import google.registry.model.transfer.TransferStatus;
 import google.registry.persistence.VKey;
+import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.util.Map;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link DomainDeleteFlow}. */
 class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, DomainBase> {
 
+  @Order(value = Order.DEFAULT - 2)
+  @RegisterExtension
+  final ReplayExtension replayExtension = new ReplayExtension(clock);
+
   private DomainBase domain;
   private HistoryEntry earlierHistoryEntry;
 
@@ -137,6 +144,7 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
         persistResource(createAutorenewBillingEvent("TheRegistrar").build());
     PollMessage.Autorenew autorenewPollMessage =
         persistResource(createAutorenewPollMessage("TheRegistrar").build());
+
     domain =
         persistResource(
             domain
@@ -144,6 +152,7 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
                 .setAutorenewBillingEvent(autorenewBillingEvent.createVKey())
                 .setAutorenewPollMessage(autorenewPollMessage.createVKey())
                 .build());
+
     assertTransactionalFlow(true);
   }
 
@@ -151,15 +160,20 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
     // Persist a linked contact.
     ContactResource contact = persistActiveContact("sh8013");
     domain =
-        newDomainBase(getUniqueIdFromCommand())
-            .asBuilder()
-            .setCreationTimeForTest(TIME_BEFORE_FLOW)
-            .setRegistrant(contact.createVKey())
-            .setRegistrationExpirationTime(expirationTime)
-            .build();
+        persistResource(
+            newDomainBase(getUniqueIdFromCommand())
+                .asBuilder()
+                .setCreationTimeForTest(TIME_BEFORE_FLOW)
+                .setRegistrant(contact.createVKey())
+                .setRegistrationExpirationTime(expirationTime)
+                .build());
     earlierHistoryEntry =
         persistResource(
-            new HistoryEntry.Builder().setType(DOMAIN_CREATE).setParent(domain).build());
+            new HistoryEntry.Builder()
+                .setType(DOMAIN_CREATE)
+                .setParent(domain)
+                .setModificationTime(clock.nowUtc())
+                .build());
   }
 
   private void setUpGracePeriods(GracePeriod... gracePeriods) {
@@ -208,7 +222,14 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
 
   private void assertAutorenewClosedAndCancellationCreatedFor(
       BillingEvent.OneTime graceBillingEvent, HistoryEntry historyEntryDomainDelete) {
-    DateTime eventTime = clock.nowUtc();
+    assertAutorenewClosedAndCancellationCreatedFor(
+        graceBillingEvent, historyEntryDomainDelete, clock.nowUtc());
+  }
+
+  private void assertAutorenewClosedAndCancellationCreatedFor(
+      BillingEvent.OneTime graceBillingEvent,
+      HistoryEntry historyEntryDomainDelete,
+      DateTime eventTime) {
     assertBillingEvents(
         createAutorenewBillingEvent("TheRegistrar").setRecurrenceEndTime(eventTime).build(),
         graceBillingEvent,
@@ -293,7 +314,11 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
     setUpSuccessfulTest();
     setUpGracePeriods(
         GracePeriod.create(
-            GracePeriodStatus.ADD, domain.getRepoId(), TIME_BEFORE_FLOW.plusDays(1), "foo", null));
+            GracePeriodStatus.ADD,
+            domain.getRepoId(),
+            TIME_BEFORE_FLOW.plusDays(1),
+            "TheRegistrar",
+            null));
     dryRunFlowAssertResponse(loadFile("generic_success_response.xml"));
   }
 
@@ -398,7 +423,7 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
             GracePeriodStatus.TRANSFER,
             domain.getRepoId(),
             TIME_BEFORE_FLOW.plusDays(1),
-            "foo",
+            "NewRegistrar",
             null));
     // We should see exactly one poll message, which is for the autorenew 1 month in the future.
     assertPollMessages(createAutorenewPollMessage("TheRegistrar").build());
@@ -730,11 +755,11 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
             .setNameservers(ImmutableSet.of(host.createVKey()))
             .setDeletionTime(START_OF_TIME)
             .build());
-    clock.advanceOneMilli();
+    DateTime eventTime = clock.nowUtc();
     runFlowAssertResponse(loadFile("generic_success_response.xml"));
     assertDnsTasksEnqueued("example.tld");
     assertAutorenewClosedAndCancellationCreatedFor(
-        graceBillingEvent, getOnlyHistoryEntryOfType(domain, DOMAIN_DELETE));
+        graceBillingEvent, getOnlyHistoryEntryOfType(domain, DOMAIN_DELETE), eventTime);
   }
 
   @Test

core/src/test/java/google/registry/flows/domain/DomainRenewFlowTest.java

@@ -26,6 +26,7 @@ import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
 import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.DatabaseHelper.persistResources;
 import static google.registry.testing.DomainBaseSubject.assertAboutDomains;
 import static google.registry.testing.EppExceptionSubject.assertAboutEppExceptions;
 import static google.registry.testing.HistoryEntrySubject.assertAboutHistoryEntries;
@@ -69,12 +70,17 @@ import google.registry.model.registry.Registry;
 import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.testing.ReplayExtension;
 import java.util.Map;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link DomainRenewFlow}. */
 class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBase> {
@@ -96,50 +102,66 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
 
   private final DateTime expirationTime = DateTime.parse("2000-04-03T22:00:00.0Z");
 
+  @Order(value = Order.DEFAULT - 3)
+  @RegisterExtension
+  final SetClockExtension setClockExtension = new SetClockExtension();
+
+  @Order(value = Order.DEFAULT - 2)
+  @RegisterExtension
+  final ReplayExtension replayExtension = new ReplayExtension(clock);
+
   @BeforeEach
   void initDomainTest() {
     createTld("tld");
-    clock.setTo(expirationTime.minusMillis(2));
     setEppInput("domain_renew.xml", ImmutableMap.of("DOMAIN", "example.tld", "YEARS", "5"));
   }
 
   private void persistDomain(StatusValue... statusValues) throws Exception {
     DomainBase domain = newDomainBase(getUniqueIdFromCommand());
-    HistoryEntry historyEntryDomainCreate =
-        persistResource(
-            new HistoryEntry.Builder()
-                .setParent(domain)
-                .setType(HistoryEntry.Type.DOMAIN_CREATE)
-                .build());
-    BillingEvent.Recurring autorenewEvent =
-        persistResource(
-            new BillingEvent.Recurring.Builder()
-                .setReason(Reason.RENEW)
-                .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
-                .setTargetId(getUniqueIdFromCommand())
-                .setClientId("TheRegistrar")
-                .setEventTime(expirationTime)
-                .setRecurrenceEndTime(END_OF_TIME)
-                .setParent(historyEntryDomainCreate)
-                .build());
-    PollMessage.Autorenew autorenewPollMessage =
-        persistResource(
-            new PollMessage.Autorenew.Builder()
-                .setTargetId(getUniqueIdFromCommand())
-                .setClientId("TheRegistrar")
-                .setEventTime(expirationTime)
-                .setAutorenewEndTime(END_OF_TIME)
-                .setMsg("Domain was auto-renewed.")
-                .setParent(historyEntryDomainCreate)
-                .build());
-    persistResource(
-        domain
-            .asBuilder()
-            .setRegistrationExpirationTime(expirationTime)
-            .setStatusValues(ImmutableSet.copyOf(statusValues))
-            .setAutorenewBillingEvent(autorenewEvent.createVKey())
-            .setAutorenewPollMessage(autorenewPollMessage.createVKey())
-            .build());
+    tm().transact(
+            () -> {
+              try {
+                HistoryEntry historyEntryDomainCreate =
+                    new HistoryEntry.Builder()
+                        .setParent(domain)
+                        .setType(HistoryEntry.Type.DOMAIN_CREATE)
+                        .setModificationTime(clock.nowUtc())
+                        .build();
+                BillingEvent.Recurring autorenewEvent =
+                    new BillingEvent.Recurring.Builder()
+                        .setReason(Reason.RENEW)
+                        .setFlags(ImmutableSet.of(Flag.AUTO_RENEW))
+                        .setTargetId(getUniqueIdFromCommand())
+                        .setClientId("TheRegistrar")
+                        .setEventTime(expirationTime)
+                        .setRecurrenceEndTime(END_OF_TIME)
+                        .setParent(historyEntryDomainCreate)
+                        .build();
+                PollMessage.Autorenew autorenewPollMessage =
+                    new PollMessage.Autorenew.Builder()
+                        .setTargetId(getUniqueIdFromCommand())
+                        .setClientId("TheRegistrar")
+                        .setEventTime(expirationTime)
+                        .setAutorenewEndTime(END_OF_TIME)
+                        .setMsg("Domain was auto-renewed.")
+                        .setParent(historyEntryDomainCreate)
+                        .build();
+                DomainBase newDomain =
+                    domain
+                        .asBuilder()
+                        .setRegistrationExpirationTime(expirationTime)
+                        .setStatusValues(ImmutableSet.copyOf(statusValues))
+                        .setAutorenewBillingEvent(autorenewEvent.createVKey())
+                        .setAutorenewPollMessage(autorenewPollMessage.createVKey())
+                        .build();
+                persistResources(
+                    ImmutableSet.of(
+                        historyEntryDomainCreate, autorenewEvent,
+                        autorenewPollMessage, newDomain));
+              } catch (Exception e) {
+                throw new RuntimeException("Error persisting domain", e);
+              }
+            });
     clock.advanceOneMilli();
   }
 
@@ -255,6 +277,7 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
 
   @Test
   void testSuccess() throws Exception {
+    clock.advanceOneMilli();
     persistDomain();
     doSuccessfulTest(
         "domain_renew_response.xml",
@@ -789,4 +812,21 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
                 TransactionReportField.netRenewsFieldFromYears(5),
                 1));
   }
+
+  /**
+   * Local extension so we can set the clock correctly prior to using it to define basic entities in
+   * AppEngineExtension.
+   *
+   * <p>This has to happen first, we'll get timestamp inversions if we try to set the clock after
+   * these objects are created.
+   */
+  class SetClockExtension implements BeforeEachCallback {
+    @Override
+    public void beforeEach(ExtensionContext context) {
+      // we have to go far enough back before the expiration time so that the clock advances we do
+      // after each persist don't move us into a grace period.  The current value is likely beyond
+      // what is necessary, but this doesn't hurt.
+      clock.setTo(expirationTime.minusMillis(20));
+    }
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -90,11 +90,14 @@ import google.registry.model.host.HostResource;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.persistence.VKey;
+import google.registry.testing.ReplayExtension;
 import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link DomainUpdateFlow}. */
 class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, DomainBase> {
@@ -112,6 +115,10 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   private ContactResource mak21Contact;
   private ContactResource unusedContact;
 
+  @Order(value = Order.DEFAULT - 2)
+  @RegisterExtension
+  final ReplayExtension replayExtension = new ReplayExtension(clock);
+
   @BeforeEach
   void initDomainTest() {
     createTld("tld");
@@ -146,6 +153,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     persistResource(
         new HistoryEntry.Builder()
             .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setModificationTime(clock.nowUtc())
             .setParent(domain)
             .build());
     clock.advanceOneMilli();
@@ -168,6 +176,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     persistResource(
         new HistoryEntry.Builder()
             .setType(HistoryEntry.Type.DOMAIN_CREATE)
+            .setModificationTime(clock.nowUtc())
             .setParent(domain)
             .build());
     clock.advanceOneMilli();

core/src/test/java/google/registry/flows/session/LoginFlowTestCase.java

@@ -14,6 +14,7 @@
 
 package google.registry.flows.session;
 
+import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.DatabaseHelper.deleteResource;
 import static google.registry.testing.DatabaseHelper.loadRegistrar;
 import static google.registry.testing.DatabaseHelper.persistResource;
@@ -32,6 +33,7 @@ import google.registry.flows.session.LoginFlow.PasswordChangesNotSupportedExcept
 import google.registry.flows.session.LoginFlow.RegistrarAccountNotActiveException;
 import google.registry.flows.session.LoginFlow.TooManyFailedLoginsException;
 import google.registry.flows.session.LoginFlow.UnsupportedLanguageException;
+import google.registry.model.eppoutput.EppOutput;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.Registrar.State;
 import org.junit.jupiter.api.BeforeEach;
@@ -74,6 +76,14 @@ public abstract class LoginFlowTestCase extends FlowTestCase<LoginFlow> {
     doSuccessfulTest("login_valid.xml");
   }
 
+  @Test
+  void testSuccess_setsIsLoginResponse() throws Exception {
+    setEppInput("login_valid.xml");
+    assertTransactionalFlow(false);
+    EppOutput output = runFlow();
+    assertThat(output.getResponse().isLoginResponse()).isTrue();
+  }
+
   @Test
   void testSuccess_suspendedRegistrar() throws Exception {
     persistResource(getRegistrarBuilder().setState(State.SUSPENDED).build());

core/src/test/java/google/registry/flows/session/LoginFlowViaTlsTest.java

@@ -15,6 +15,7 @@
 package google.registry.flows.session;
 
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.net.InetAddresses;
@@ -26,6 +27,7 @@ import google.registry.model.registrar.Registrar;
 import google.registry.testing.CertificateSamples;
 import google.registry.util.CidrAddressBlock;
 import java.util.Optional;
+import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link LoginFlow} when accessed via a TLS transport. */
@@ -41,7 +43,7 @@ public class LoginFlowViaTlsTest extends LoginFlowTestCase {
   @Override
   protected Registrar.Builder getRegistrarBuilder() {
     return super.getRegistrarBuilder()
-        .setClientCertificateHash(GOOD_CERT)
+        .setClientCertificate(CertificateSamples.SAMPLE_CERT, DateTime.now(UTC))
         .setIpAddressAllowList(
             ImmutableList.of(CidrAddressBlock.create(InetAddresses.forString(GOOD_IP.get()), 32)));
   }