Validate SQL credentials in Secret Manager (#907)

* Validate SQL credentials in Secret Manager

Load SQL credentials from the SecretManager and compare them with the
ones currently in use in Nomulus server, beam pipeline, and the registry
tool. Normal operations are not affected by failures related to the
SecretManager, be it IOException, insufficient permission , or wrong or
missing credential.

The appengine and compute engine default service accounts must be
granted the permission to access the secret data. In the short term, we
will grant the secretmanager.secretAccessor role to these accounts. In
the long term, with the proposed privilege service, access will be granted
on per-secret basis.

config/presubmits.py

@@ -65,7 +65,7 @@ class PresubmitCheck:
     for pattern in self.skipped_patterns:
       if pattern in file:
         return False
-    with open(file, "r") as f:
+    with open(file, "r", encoding='utf8') as f:
       file_content = f.read()
       matches = re.match(self.regex, file_content, re.DOTALL)
       if self.regex_type == FORBIDDEN:
@@ -241,7 +241,7 @@ def verify_flyway_index():
 
   # Remove the sequence numbers and compare against the index file contents.
   files = [filename[1] for filename in sorted(files)]
-  with open('db/src/main/resources/sql/flyway.txt') as index:
+  with open('db/src/main/resources/sql/flyway.txt', encoding='utf8') as index:
     indexed_files = index.read().splitlines()
   if files != indexed_files:
     unindexed = set(files) - set(indexed_files)

core/src/main/java/google/registry/batch/ResaveAllEppResourcesAction.java

@@ -50,11 +50,21 @@ public class ResaveAllEppResourcesAction implements Runnable {
   @Inject Response response;
   @Inject ResaveAllEppResourcesAction() {}
 
+  /**
+   * The number of shards to run the map-only mapreduce on.
+   *
+   * <p>This is less than the default of 100 because we only run this action monthly and can afford
+   * it being slower, but we don't want to write out lots of large commit logs in a short period of
+   * time because they make the Cloud SQL migration tougher.
+   */
+  private static final int NUM_SHARDS = 10;
+
   @Override
   public void run() {
     mrRunner
         .setJobName("Re-save all EPP resources")
         .setModuleName("backend")
+        .setDefaultMapShards(NUM_SHARDS)
         .runMapOnly(
             new ResaveAllEppResourcesActionMapper(),
             ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))

core/src/main/java/google/registry/beam/initsql/BeamJpaModule.java

@@ -31,6 +31,7 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.JdbcJpaTm;
 import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -168,6 +169,7 @@ public class BeamJpaModule {
         BeamJpaModule.class,
         KmsModule.class,
         PersistenceModule.class,
+        SecretManagerModule.class,
         UtilsModule.class
       })
   public interface JpaTransactionManagerComponent {

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Throwables.throwIfUnchecked;
 import static google.registry.beam.initsql.BackupPaths.getCommitLogTimestamp;
 import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
+import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.JpaRetries.isFailedTxnRetriable;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.setJpaTm;
@@ -68,6 +69,7 @@ import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.ProcessFunction;
+import org.apache.beam.sdk.transforms.SerializableFunction;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -264,9 +266,9 @@ public final class Transforms {
   }
 
   /**
-   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
-   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
-   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   * Returns a {@link PTransform} that writes a {@link PCollection} of {@link VersionedEntity}s to a
+   * SQL database. and outputs an empty {@code PCollection<Void>}. This allows other operations to
+   * {@link org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
    *
    * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
    * job, we will not add features unless proven necessary.
@@ -282,16 +284,53 @@ public final class Transforms {
       int maxWriters,
       int batchSize,
       SerializableSupplier<JpaTransactionManager> jpaSupplier) {
-    return new PTransform<PCollection<VersionedEntity>, PCollection<Void>>() {
+    return writeToSql(
+        transformId,
+        maxWriters,
+        batchSize,
+        jpaSupplier,
+        (e) -> ofy().toPojo(e.getEntity().get()),
+        TypeDescriptor.of(VersionedEntity.class));
+  }
+
+  /**
+   * Returns a {@link PTransform} that writes a {@link PCollection} of entities to a SQL database.
+   * and outputs an empty {@code PCollection<Void>}. This allows other operations to {@link
+   * org.apache.beam.sdk.transforms.Wait wait} for the completion of this transform.
+   *
+   * <p>The converter and type descriptor are generics so that we can convert any type of entity to
+   * an object to be placed in SQL.
+   *
+   * <p>Errors are handled according to the pipeline runner's default policy. As part of a one-time
+   * job, we will not add features unless proven necessary.
+   *
+   * @param transformId a unique ID for an instance of the returned transform
+   * @param maxWriters the max number of concurrent writes to SQL, which also determines the max
+   *     number of connection pools created
+   * @param batchSize the number of entities to write in each operation
+   * @param jpaSupplier supplier of a {@link JpaTransactionManager}
+   * @param jpaConverter the function that converts the input object to a JPA entity
+   * @param objectDescriptor the type descriptor of the input object
+   */
+  public static <T> PTransform<PCollection<T>, PCollection<Void>> writeToSql(
+      String transformId,
+      int maxWriters,
+      int batchSize,
+      SerializableSupplier<JpaTransactionManager> jpaSupplier,
+      SerializableFunction<T, Object> jpaConverter,
+      TypeDescriptor<T> objectDescriptor) {
+    return new PTransform<PCollection<T>, PCollection<Void>>() {
       @Override
-      public PCollection<Void> expand(PCollection<VersionedEntity> input) {
+      public PCollection<Void> expand(PCollection<T> input) {
         return input
             .apply(
                 "Shard data for " + transformId,
-                MapElements.into(kvs(integers(), TypeDescriptor.of(VersionedEntity.class)))
+                MapElements.into(kvs(integers(), objectDescriptor))
                     .via(ve -> KV.of(ThreadLocalRandom.current().nextInt(maxWriters), ve)))
             .apply("Batch output by shard " + transformId, GroupIntoBatches.ofSize(batchSize))
-            .apply("Write in batch for " + transformId, ParDo.of(new SqlBatchWriter(jpaSupplier)));
+            .apply(
+                "Write in batch for " + transformId,
+                ParDo.of(new SqlBatchWriter<T>(jpaSupplier, jpaConverter)));
       }
     };
   }
@@ -385,18 +424,22 @@ public final class Transforms {
    * to hold the {@code JpaTransactionManager} instance, we must ensure that JpaTransactionManager
    * is not changed or torn down while being used by some instance.
    */
-  private static class SqlBatchWriter extends DoFn<KV<Integer, Iterable<VersionedEntity>>, Void> {
+  private static class SqlBatchWriter<T> extends DoFn<KV<Integer, Iterable<T>>, Void> {
 
     private static int instanceCount = 0;
     private static JpaTransactionManager originalJpa;
 
     private final SerializableSupplier<JpaTransactionManager> jpaSupplier;
+    private final SerializableFunction<T, Object> jpaConverter;
 
     private transient Ofy ofy;
     private transient SystemSleeper sleeper;
 
-    SqlBatchWriter(SerializableSupplier<JpaTransactionManager> jpaSupplier) {
+    SqlBatchWriter(
+        SerializableSupplier<JpaTransactionManager> jpaSupplier,
+        SerializableFunction<T, Object> jpaConverter) {
       this.jpaSupplier = jpaSupplier;
+      this.jpaConverter = jpaConverter;
     }
 
     @Setup
@@ -429,13 +472,11 @@ public final class Transforms {
     }
 
     @ProcessElement
-    public void processElement(@Element KV<Integer, Iterable<VersionedEntity>> kv) {
+    public void processElement(@Element KV<Integer, Iterable<T>> kv) {
       try (AppEngineEnvironment env = new AppEngineEnvironment()) {
         ImmutableList<Object> ofyEntities =
             Streams.stream(kv.getValue())
-                .map(VersionedEntity::getEntity)
-                .map(Optional::get)
-                .map(ofy::toPojo)
+                .map(this.jpaConverter::apply)
                 .collect(ImmutableList.toImmutableList());
         retry(() -> jpaTm().transact(() -> jpaTm().putAll(ofyEntities)));
       }

core/src/main/java/google/registry/beam/spec11/Spec11Pipeline.java

@@ -20,7 +20,7 @@ import static google.registry.beam.BeamUtils.getQueryFromFile;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.auto.value.AutoValue;
 import com.google.common.collect.ImmutableSet;
-import google.registry.backup.AppEngineEnvironment;
+import google.registry.beam.initsql.Transforms;
 import google.registry.beam.initsql.Transforms.SerializableSupplier;
 import google.registry.beam.spec11.SafeBrowsingTransforms.EvaluateSafeBrowsingFn;
 import google.registry.config.CredentialModule.LocalCredential;
@@ -43,7 +43,6 @@ import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.ValueProvider;
 import org.apache.beam.sdk.options.ValueProvider.NestedValueProvider;
-import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.GroupByKey;
 import org.apache.beam.sdk.transforms.MapElements;
 import org.apache.beam.sdk.transforms.ParDo;
@@ -191,34 +190,27 @@ public class Spec11Pipeline implements Serializable {
       PCollection<Subdomain> domains,
       EvaluateSafeBrowsingFn evaluateSafeBrowsingFn,
       ValueProvider<String> dateProvider) {
-
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsSql =
         domains.apply("Run through SafeBrowsing API", ParDo.of(evaluateSafeBrowsingFn));
-    /* Store ThreatMatch objects in SQL. */
+    TypeDescriptor<KV<Subdomain, ThreatMatch>> descriptor =
+        new TypeDescriptor<KV<Subdomain, ThreatMatch>>() {};
     subdomainsSql.apply(
-        ParDo.of(
-            new DoFn<KV<Subdomain, ThreatMatch>, Void>() {
-              @ProcessElement
-              public void processElement(ProcessContext context) {
-                // create the Spec11ThreatMatch from Subdomain and ThreatMatch
-                try (AppEngineEnvironment env = new AppEngineEnvironment()) {
-                  Subdomain subdomain = context.element().getKey();
-                  Spec11ThreatMatch threatMatch =
-                      new Spec11ThreatMatch.Builder()
-                          .setThreatTypes(
-                              ImmutableSet.of(
-                                  ThreatType.valueOf(context.element().getValue().threatType())))
-                          .setCheckDate(
-                              LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
-                          .setDomainName(subdomain.domainName())
-                          .setDomainRepoId(subdomain.domainRepoId())
-                          .setRegistrarId(subdomain.registrarId())
-                          .build();
-                  JpaTransactionManager jpaTransactionManager = jpaSupplierFactory.get();
-                  jpaTransactionManager.transact(() -> jpaTransactionManager.insert(threatMatch));
-                }
-              }
-            }));
+        Transforms.writeToSql(
+            "Spec11ThreatMatch",
+            4,
+            4,
+            jpaSupplierFactory,
+            (kv) -> {
+              Subdomain subdomain = kv.getKey();
+              return new Spec11ThreatMatch.Builder()
+                  .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
+                  .setCheckDate(LocalDate.parse(dateProvider.get(), ISODateTimeFormat.date()))
+                  .setDomainName(subdomain.domainName())
+                  .setDomainRepoId(subdomain.domainRepoId())
+                  .setRegistrarId(subdomain.registrarId())
+                  .build();
+            },
+            descriptor));
 
     /* Store ThreatMatch objects in JSON. */
     PCollection<KV<Subdomain, ThreatMatch>> subdomainsJson =

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -415,6 +415,14 @@ public final class RegistryConfig {
       return config.cloudSql.instanceConnectionName;
     }
 
+    @Provides
+    @Config("cloudSqlDbInstanceName")
+    public static String providesCloudSqlDbInstance(RegistryConfigSettings config) {
+      // Format of instanceConnectionName: project-id:region:instance-name
+      int lastColonIndex = config.cloudSql.instanceConnectionName.lastIndexOf(':');
+      return config.cloudSql.instanceConnectionName.substring(lastColonIndex + 1);
+    }
+
     @Provides
     @Config("cloudDnsRootUrl")
     public static Optional<String> getCloudDnsRootUrl(RegistryConfigSettings config) {

core/src/main/java/google/registry/model/DatabaseMigrationUtils.java

@@ -0,0 +1,38 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryEnvironment;
+
+/** Utility methods related to migrating dual-read/dual-write entities. */
+public class DatabaseMigrationUtils {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Throws exceptions only in unit tests, otherwise only logs exceptions. */
+  public static void suppressExceptionUnlessInTest(Runnable work, String message) {
+    try {
+      work.run();
+    } catch (Exception e) {
+      if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)) {
+        throw e;
+      }
+      logger.atWarning().withCause(e).log(message);
+    }
+  }
+
+  private DatabaseMigrationUtils() {}
+}

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -28,6 +28,11 @@ import org.joda.time.DateTime;
  */
 public class UpdateAutoTimestamp extends ImmutableObject {
 
+  // When set to true, database converters/translators should do tha auto update.  When set to
+  // false, auto update should be suspended (this exists to allow us to preserve the original value
+  // during a replay).
+  static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
+
   DateTime timestamp;
 
   /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
@@ -40,4 +45,30 @@ public class UpdateAutoTimestamp extends ImmutableObject {
     instance.timestamp = timestamp;
     return instance;
   }
+
+  // TODO(b/175610935): Remove the auto-update disabling code below after migration.
+
+  /** Class to allow us to safely disable auto-update in a try-with-resources block. */
+  public static class DisableAutoUpdateResource implements AutoCloseable {
+    DisableAutoUpdateResource() {
+      autoUpdateEnabled.set(false);
+    }
+
+    @Override
+    public void close() {
+      autoUpdateEnabled.set(true);
+    }
+  }
+
+  /**
+   * Resturns a resource that disables auto-updates on all {@link UpdateAutoTimestamp}s in the
+   * current thread, suitable for use with in a try-with-resources block.
+   */
+  public static DisableAutoUpdateResource disableAutoUpdate() {
+    return new DisableAutoUpdateResource();
+  }
+
+  public static boolean autoUpdateEnabled() {
+    return autoUpdateEnabled.get();
+  }
 }

core/src/main/java/google/registry/model/billing/BillingEvent.java

@@ -290,7 +290,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "allocationToken")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_event_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billable value. */
@@ -464,7 +464,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "recurrence_time_of_year")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_recurrence_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Recurring extends BillingEvent implements DatastoreAndSqlEntity {
 
     /**
@@ -559,7 +559,7 @@ public abstract class BillingEvent extends ImmutableObject
         @javax.persistence.Index(columnList = "billingTime")
       })
   @AttributeOverride(name = "id", column = @Column(name = "billing_cancellation_id"))
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Cancellation extends BillingEvent implements DatastoreAndSqlEntity {
 
     /** The billing time of the charge that is being cancelled. */
@@ -680,7 +680,7 @@ public abstract class BillingEvent extends ImmutableObject
   /** An event representing a modification of an existing one-time billing event. */
   @ReportedOn
   @Entity
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Modification extends BillingEvent implements DatastoreOnlyEntity {
 
     /** The change in cost that should be applied to the original billing event. */

core/src/main/java/google/registry/model/poll/PollMessage.java

@@ -279,7 +279,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("ONE_TIME")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class OneTime extends PollMessage {
 
     // Response data. Objectify cannot persist a base class type, so we must have a separate field
@@ -432,7 +432,7 @@ public abstract class PollMessage extends ImmutableObject
   @EntitySubclass(index = false)
   @javax.persistence.Entity
   @DiscriminatorValue("AUTORENEW")
-  @WithLongVKey
+  @WithLongVKey(compositeKey = true)
   public static class Autorenew extends PollMessage {
 
     /** The target id of the autorenew event. */

core/src/main/java/google/registry/model/reporting/Spec11ThreatMatchDao.java

@@ -21,10 +21,10 @@ import javax.persistence.TemporalType;
 import org.joda.time.LocalDate;
 
 /**
- * Data access object for {@link google.registry.model.reporting.Spec11ThreatMatch}.
+ * Data access object for {@link Spec11ThreatMatch}.
  *
- * <p>A JpaTransactionManager is passed into each static method because they are called from a BEAM
- * pipeline and we don't know where it's coming from.
+ * <p>The transaction manager is passed as a parameter because this could be called either from a
+ * BEAM pipeline or standard non-BEAM code.
  */
 public class Spec11ThreatMatchDao {
 

core/src/main/java/google/registry/model/smd/SignedMarkRevocationList.java

@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.isEmpty;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -32,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EmbedMap;
 import com.googlecode.objectify.annotation.Entity;
@@ -82,8 +82,6 @@ import org.joda.time.DateTime;
 @NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 public class SignedMarkRevocationList extends ImmutableObject implements NonReplicatedEntity {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
   @VisibleForTesting static final int SHARD_SIZE = 10000;
 
   /** Common ancestor for queries. */
@@ -121,12 +119,11 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
       memoizeWithShortExpiration(
           () -> {
             SignedMarkRevocationList datastoreList = loadFromDatastore();
-            // Also load the list from Cloud SQL, compare the two lists, and log if different.
-            try {
-              loadAndCompareCloudSqlList(datastoreList);
-            } catch (Throwable t) {
-              logger.atSevere().withCause(t).log("Error comparing signed mark revocation lists.");
-            }
+            suppressExceptionUnlessInTest(
+                () -> {
+                  loadAndCompareCloudSqlList(datastoreList);
+                },
+                "Error comparing signed mark revocation lists.");
             return datastoreList;
           });
 
@@ -229,11 +226,12 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
           Maps.difference(datastoreList.revokes, cloudSqlList.revokes);
       if (!diff.areEqual()) {
         if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
+          String message =
               String.format(
                   "Unequal SM revocation lists detected, Cloud SQL list with revision id %d has %d"
                       + " different records than the current Datastore list.",
-                  cloudSqlList.revisionId, diff.entriesDiffering().size()));
+                  cloudSqlList.revisionId, diff.entriesDiffering().size());
+          throw new RuntimeException(message);
         } else {
           StringBuilder diffMessage = new StringBuilder("Unequal SM revocation lists detected:\n");
           diff.entriesDiffering()
@@ -243,11 +241,13 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
                           String.format(
                               "SMD %s has key %s in Datastore and key %s in Cloud SQL.\n",
                               label, valueDiff.leftValue(), valueDiff.rightValue())));
-          logger.atWarning().log(diffMessage.toString());
+          throw new RuntimeException(diffMessage.toString());
         }
       }
     } else {
-      logger.atWarning().log("Signed mark revocation list in Cloud SQL is empty.");
+      if (datastoreList.size() != 0) {
+        throw new RuntimeException("Signed mark revocation list in Cloud SQL is empty.");
+      }
     }
   }
 

core/src/main/java/google/registry/model/smd/SignedMarkRevocationListDao.java

@@ -15,6 +15,7 @@
 package google.registry.model.smd;
 
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 
 import com.google.common.base.Supplier;
@@ -60,14 +61,14 @@ public class SignedMarkRevocationListDao {
    * the authoritative database.
    */
   static void trySave(SignedMarkRevocationList signedMarkRevocationList) {
-    try {
-      SignedMarkRevocationListDao.save(signedMarkRevocationList);
-      logger.atInfo().log(
-          "Inserted %,d signed mark revocations into Cloud SQL",
-          signedMarkRevocationList.revokes.size());
-    } catch (Throwable e) {
-      logger.atSevere().withCause(e).log("Error inserting signed mark revocations into Cloud SQL");
-    }
+    suppressExceptionUnlessInTest(
+        () -> {
+          SignedMarkRevocationListDao.save(signedMarkRevocationList);
+          logger.atInfo().log(
+              "Inserted %,d signed mark revocations into Cloud SQL.",
+              signedMarkRevocationList.revokes.size());
+        },
+        "Error inserting signed mark revocations into Cloud SQL.");
   }
 
   private static void save(SignedMarkRevocationList signedMarkRevocationList) {

core/src/main/java/google/registry/model/translators/UpdateAutoTimestampTranslatorFactory.java

@@ -46,7 +46,10 @@ public class UpdateAutoTimestampTranslatorFactory
       /** Save a timestamp, setting it to the current time. */
       @Override
       public Date saveValue(UpdateAutoTimestamp pojoValue) {
-        return tm().getTransactionTime().toDate();
-      }};
+        return UpdateAutoTimestamp.autoUpdateEnabled()
+            ? tm().getTransactionTime().toDate()
+            : pojoValue.getTimestamp().toDate();
+      }
+    };
   }
 }

core/src/main/java/google/registry/persistence/PersistenceComponent.java

@@ -20,6 +20,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import javax.inject.Singleton;
 import javax.persistence.EntityManagerFactory;
@@ -32,6 +33,7 @@ import javax.persistence.EntityManagerFactory;
       CredentialModule.class,
       KmsModule.class,
       PersistenceModule.class,
+      SecretManagerModule.class,
       UtilsModule.class
     })
 public interface PersistenceComponent {

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -26,6 +26,7 @@ import com.google.api.client.auth.oauth2.Credential;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
@@ -33,6 +34,11 @@ import google.registry.keyring.kms.KmsKeyring;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotId;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
 import google.registry.tools.AuthModule.CloudSqlClientCredential;
 import google.registry.util.Clock;
 import java.lang.annotation.Documented;
@@ -47,6 +53,8 @@ import org.hibernate.cfg.Environment;
 /** Dagger module class for the persistence layer. */
 @Module
 public class PersistenceModule {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
   // This name must be the same as the one defined in persistence.xml.
   public static final String PERSISTENCE_UNIT_NAME = "nomulus";
   public static final String HIKARI_CONNECTION_TIMEOUT = "hibernate.hikari.connectionTimeout";
@@ -122,11 +130,17 @@ public class PersistenceModule {
   static JpaTransactionManager provideAppEngineJpaTm(
       @Config("cloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       Clock clock) {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -136,6 +150,7 @@ public class PersistenceModule {
   static JpaTransactionManager provideNomulusToolJpaTm(
       @Config("toolsCloudSqlUsername") String username,
       KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
       @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
       @CloudSqlClientCredential Credential credential,
       Clock clock) {
@@ -143,6 +158,11 @@ public class PersistenceModule {
     HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, kmsKeyring.getToolsCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.TOOL),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -150,6 +170,7 @@ public class PersistenceModule {
   @Singleton
   @SocketFactoryJpaTm
   static JpaTransactionManager provideSocketFactoryJpaTm(
+      SqlCredentialStore credentialStore,
       @Config("beamCloudSqlUsername") String username,
       @Config("beamCloudSqlPassword") String password,
       @Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
@@ -159,6 +180,12 @@ public class PersistenceModule {
     overrides.put(Environment.USER, username);
     overrides.put(Environment.PASS, password);
     overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
+    // TODO(b/175700623): consider assigning different logins to pipelines
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -203,6 +230,30 @@ public class PersistenceModule {
     return emf;
   }
 
+  /** Verifies that the credential from the Secret Manager matches the one currently in use.
+   *
+   * <p>This is a helper for the transition to the Secret Manager, and will be removed once data
+   * and permissions are properly set up for all projects.
+   **/
+  private static void validateCredentialStore(
+      SqlCredentialStore credentialStore, SqlUser sqlUser, String login, String password) {
+    try {
+      SqlCredential credential = credentialStore.getCredential(sqlUser);
+      if (!credential.login().equals(login)) {
+        logger.atWarning().log(
+            "Wrong login for %s. Expecting %s, found %s.",
+            sqlUser.geUserName(), login, credential.login());
+        return;
+      }
+      if (!credential.password().equals(password)) {
+        logger.atWarning().log("Wrong password for %s.", sqlUser.geUserName());
+      }
+      logger.atWarning().log("Credentials in the kerying and the secret manager match.");
+    } catch (Throwable e) {
+      logger.atWarning().log(e.getMessage());
+    }
+  }
+
   /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
   @Qualifier
   @Documented

core/src/main/java/google/registry/persistence/WithLongVKey.java

@@ -31,4 +31,12 @@ public @interface WithLongVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/WithStringVKey.java

@@ -31,4 +31,12 @@ public @interface WithStringVKey {
    * class name will be "VKeyConverter_" concatenated with the suffix.
    */
   String classNameSuffix() default "";
+
+  /**
+   * Set to true if this is a composite vkey.
+   *
+   * <p>For composite VKeys, we don't attempt to define an objectify key when loading from SQL: the
+   * enclosing class has to take care of that.
+   */
+  boolean compositeKey() default false;
 }

core/src/main/java/google/registry/persistence/converter/UpdateAutoTimestampConverter.java

@@ -31,7 +31,14 @@ public class UpdateAutoTimestampConverter
 
   @Override
   public Timestamp convertToDatabaseColumn(UpdateAutoTimestamp entity) {
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(jpaTm().getTransactionTime()).toInstant());
+    return Timestamp.from(
+        DateTimeUtils.toZonedDateTime(
+                UpdateAutoTimestamp.autoUpdateEnabled()
+                        || entity == null
+                        || entity.getTimestamp() == null
+                    ? jpaTm().getTransactionTime()
+                    : entity.getTimestamp())
+            .toInstant());
   }
 
   @Override

core/src/main/java/google/registry/persistence/converter/VKeyConverter.java

@@ -14,6 +14,7 @@
 
 package google.registry.persistence.converter;
 
+import com.googlecode.objectify.Key;
 import google.registry.persistence.VKey;
 import javax.annotation.Nullable;
 import javax.persistence.AttributeConverter;
@@ -29,7 +30,28 @@ public abstract class VKeyConverter<T, C> implements AttributeConverter<VKey<? e
   @Override
   @Nullable
   public VKey<? extends T> convertToEntityAttribute(@Nullable C dbData) {
-    return dbData == null ? null : VKey.createSql(getAttributeClass(), dbData);
+    if (dbData == null) {
+      return null;
+    }
+    Class<? extends T> clazz = getAttributeClass();
+    Key ofyKey = null;
+    if (!hasCompositeOfyKey()) {
+      // If this isn't a composite key, we can create the Ofy key from the SQL key.
+      ofyKey =
+          dbData instanceof String
+              ? Key.create(clazz, (String) dbData)
+              : Key.create(clazz, (Long) dbData);
+      return VKey.create(clazz, dbData, ofyKey);
+    } else {
+      // We don't know how to create the Ofy key and probably don't have everything necessary to do
+      // it anyway, so just create an asymmetric key - the containing object will have to convert it
+      // into a symmetric key.
+      return VKey.createSql(clazz, dbData);
+    }
+  }
+
+  protected boolean hasCompositeOfyKey() {
+    return false;
   }
 
   /** Returns the class of the attribute. */

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClient.java

@@ -22,6 +22,9 @@ import java.util.Optional;
 /** A Cloud Secret Manager client for Nomulus, bound to a specific GCP project. */
 public interface SecretManagerClient {
 
+  /** Returns the project name with which this client is associated. */
+  String getProject();
+
   /**
    * Creates a new secret in the Cloud Secret Manager with no data.
    *
@@ -32,6 +35,9 @@ public interface SecretManagerClient {
    */
   void createSecret(String secretId);
 
+  /** Checks if a secret with the given {@code secretId} already exists. */
+  boolean secretExists(String secretId);
+
   /** Returns all secret IDs in the Cloud Secret Manager. */
   Iterable<String> listSecrets();
 
@@ -67,6 +73,24 @@ public interface SecretManagerClient {
    */
   String getSecretData(String secretId, Optional<String> version);
 
+  /**
+   * Enables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void enableSecretVersion(String secretId, String version);
+
+  /**
+   * Disables a secret version.
+   *
+   * @param secretId The ID of the secret
+   * @param version The version of the secret to fetch. If not provided, the {@code latest} version
+   *     will be returned
+   */
+  void disableSecretVersion(String secretId, String version);
+
   /**
    * Destroys a secret version.
    *

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerClientImpl.java

@@ -29,12 +29,11 @@ import com.google.cloud.secretmanager.v1.SecretName;
 import com.google.cloud.secretmanager.v1.SecretPayload;
 import com.google.cloud.secretmanager.v1.SecretVersion;
 import com.google.cloud.secretmanager.v1.SecretVersionName;
-import com.google.common.collect.Iterables;
+import com.google.common.collect.Streams;
 import com.google.protobuf.ByteString;
 import google.registry.util.Retrier;
 import java.util.Optional;
 import java.util.concurrent.Callable;
-import javax.inject.Inject;
 
 /** Implements {@link SecretManagerClient} on Google Cloud Platform. */
 public class SecretManagerClientImpl implements SecretManagerClient {
@@ -42,13 +41,17 @@ public class SecretManagerClientImpl implements SecretManagerClient {
   private final SecretManagerServiceClient csmClient;
   private final Retrier retrier;
 
-  @Inject
   SecretManagerClientImpl(String project, SecretManagerServiceClient csmClient, Retrier retrier) {
     this.project = project;
     this.csmClient = csmClient;
     this.retrier = retrier;
   }
 
+  @Override
+  public String getProject() {
+    return project;
+  }
+
   @Override
   public void createSecret(String secretId) {
     checkNotNull(secretId, "secretId");
@@ -57,12 +60,25 @@ public class SecretManagerClientImpl implements SecretManagerClient {
         () -> csmClient.createSecret(ProjectName.of(project), secretId, secretSettings));
   }
 
+  @Override
+  public boolean secretExists(String secretId) {
+    checkNotNull(secretId, "secretId");
+    try {
+      callSecretManager(() -> csmClient.getSecret(SecretName.of(project, secretId)));
+      return true;
+    } catch (NoSuchSecretResourceException e) {
+      return false;
+    }
+  }
+
   @Override
   public Iterable<String> listSecrets() {
     ListSecretsPagedResponse response =
         callSecretManager(() -> csmClient.listSecrets(ProjectName.of(project)));
-    return Iterables.transform(
-        response.iterateAll(), secret -> SecretName.parse(secret.getName()).getSecret());
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(secret -> SecretName.parse(secret.getName()).getSecret())
+            .iterator();
   }
 
   @Override
@@ -70,8 +86,10 @@ public class SecretManagerClientImpl implements SecretManagerClient {
     checkNotNull(secretId, "secretId");
     ListSecretVersionsPagedResponse response =
         callSecretManager(() -> csmClient.listSecretVersions(SecretName.of(project, secretId)));
-    return Iterables.transform(
-        response.iterateAll(), SecretManagerClientImpl::toSecretVersionState);
+    return () ->
+        Streams.stream(response.iterateAll())
+            .map(SecretManagerClientImpl::toSecretVersionState)
+            .iterator();
   }
 
   private static SecretVersionState toSecretVersionState(SecretVersion secretVersion) {
@@ -108,6 +126,22 @@ public class SecretManagerClientImpl implements SecretManagerClient {
                 .toStringUtf8());
   }
 
+  @Override
+  public void enableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.enableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
+  @Override
+  public void disableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    callSecretManager(
+        () -> csmClient.disableSecretVersion(SecretVersionName.of(project, secretId, version)));
+  }
+
   @Override
   public void destroySecretVersion(String secretId, String version) {
     checkNotNull(secretId, "secretId");

core/src/main/java/google/registry/privileges/secretmanager/SecretManagerModule.java

@@ -14,41 +14,47 @@
 
 package google.registry.privileges.secretmanager;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
-import dagger.Component;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import dagger.Module;
 import dagger.Provides;
-import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.config.CredentialModule.DefaultCredential;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
-import google.registry.util.UtilsModule;
 import java.io.IOException;
 import javax.inject.Singleton;
 
 /** Provides bindings for {@link SecretManagerClient}. */
 @Module
-public class SecretManagerModule {
-
-  private final String project;
-
-  public SecretManagerModule(String project) {
-    this.project = checkNotNull(project, "project");
-  }
+public abstract class SecretManagerModule {
 
   @Provides
   @Singleton
-  SecretManagerClient provideSecretManagerClient(Retrier retrier) {
+  static SecretManagerServiceSettings provideSecretManagerSetting(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle) {
     try {
-      return new SecretManagerClientImpl(project, SecretManagerServiceClient.create(), retrier);
+      return SecretManagerServiceSettings.newBuilder()
+          .setCredentialsProvider(() -> credentialsBundle.getGoogleCredentials())
+          .build();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
   }
 
+  @Provides
   @Singleton
-  @Component(modules = {ConfigModule.class, SecretManagerModule.class, UtilsModule.class})
-  public interface SecretManagerComponent {
-    SecretManagerClient secretManagerClient();
+  static SecretManagerClient provideSecretManagerClient(
+      SecretManagerServiceSettings serviceSettings,
+      @Config("projectId") String project,
+      Retrier retrier) {
+    try {
+      SecretManagerServiceClient stub = SecretManagerServiceClient.create(serviceSettings);
+      Runtime.getRuntime().addShutdownHook(new Thread(stub::close));
+      return new SecretManagerClientImpl(project, stub, retrier);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
   }
 }

core/src/main/java/google/registry/privileges/secretmanager/SqlCredential.java

@@ -0,0 +1,55 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static avro.shaded.com.google.common.base.Preconditions.checkState;
+
+import com.google.auto.value.AutoValue;
+import java.util.List;
+
+/**
+ * Contains the login name and password of a Cloud SQL user.
+ *
+ * <p>User must take care not to include the {@link #SEPARATOR} in property values.
+ */
+@AutoValue
+public abstract class SqlCredential {
+
+  public static final Character SEPARATOR = ' ';
+
+  public abstract String login();
+
+  public abstract String password();
+
+  @Override
+  public final String toString() {
+    // Use Object.toString(), which does not show object data.
+    return super.toString();
+  }
+
+  public final String toFormattedString() {
+    return String.format("%s%c%s", login(), SEPARATOR, password());
+  }
+
+  public static SqlCredential fromFormattedString(String sqlCredential) {
+    List<String> items = com.google.common.base.Splitter.on(SEPARATOR).splitToList(sqlCredential);
+    checkState(items.size() == 2, "Invalid SqlCredential string.");
+    return of(items.get(0), items.get(1));
+  }
+
+  public static SqlCredential of(String login, String password) {
+    return new AutoValue_SqlCredential(login, password);
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlCredentialStore.java

@@ -0,0 +1,118 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.cloud.secretmanager.v1.SecretVersionName;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
+import java.util.Optional;
+import javax.inject.Inject;
+
+/**
+ * Storage of SQL users' login credentials, backed by Cloud Secret Manager.
+ *
+ * <p>A user's credential is stored with one level of indirection using two secret IDs: Each version
+ * of the <em>credential data</em> is stored as follows: its secret ID is determined by {@link
+ * #getCredentialDataSecretId(SqlUser, String dbInstance)}, and the value of each version is a
+ * {@link SqlCredential}, serialized using {@link SqlCredential#toFormattedString}. The 'live'
+ * version of the credential is saved under the 'live pointer' secret explained below.
+ *
+ * <p>The pointer to the 'live' version of the credential data is stored as follows: its secret ID
+ * is determined by {@link #getLiveLabelSecretId(SqlUser, String dbInstance)}; and the value of each
+ * version is a {@link SecretVersionName} in String form, pointing to a version of the credential
+ * data. Only the 'latest' version of this secret should be used. It is guaranteed to be valid.
+ *
+ * <p>The indirection in credential storage makes it easy to handle failures in the credential
+ * change process.
+ */
+public class SqlCredentialStore {
+  private final SecretManagerClient csmClient;
+  private final String dbInstance;
+
+  @Inject
+  SqlCredentialStore(
+      SecretManagerClient csmClient, @Config("cloudSqlDbInstanceName") String dbInstance) {
+    this.csmClient = csmClient;
+    this.dbInstance = dbInstance;
+  }
+
+  public SqlCredential getCredential(SqlUser user) {
+    SecretVersionName credentialName = getLiveCredentialSecretVersion(user);
+    return SqlCredential.fromFormattedString(
+        csmClient.getSecretData(
+            credentialName.getSecret(), Optional.of(credentialName.getSecretVersion())));
+  }
+
+  public void createOrUpdateCredential(SqlUser user, String password) {
+    SecretVersionName dataName = saveCredentialData(user, password);
+    saveLiveLabel(user, dataName);
+  }
+
+  public void deleteCredential(SqlUser user) {
+    try {
+      csmClient.deleteSecret(getCredentialDataSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok
+    }
+    try {
+      csmClient.deleteSecret(getLiveLabelSecretId(user, dbInstance));
+    } catch (NoSuchSecretResourceException e) {
+      // ok.
+    }
+  }
+
+  private void createSecretIfAbsent(String secretId) {
+    try {
+      csmClient.createSecret(secretId);
+    } catch (SecretAlreadyExistsException ignore) {
+      // Not a problem.
+    }
+  }
+
+  private SecretVersionName saveCredentialData(SqlUser user, String password) {
+    String credentialDataSecretId = getCredentialDataSecretId(user, dbInstance);
+    createSecretIfAbsent(credentialDataSecretId);
+    String credentialVersion =
+        csmClient.addSecretVersion(
+            credentialDataSecretId,
+            SqlCredential.of(createDatabaseLoginName(user), password).toFormattedString());
+    return SecretVersionName.of(csmClient.getProject(), credentialDataSecretId, credentialVersion);
+  }
+
+  private void saveLiveLabel(SqlUser user, SecretVersionName dataVersionName) {
+    String liveLabelSecretId = getLiveLabelSecretId(user, dbInstance);
+    createSecretIfAbsent(liveLabelSecretId);
+    csmClient.addSecretVersion(liveLabelSecretId, dataVersionName.toString());
+  }
+
+  private SecretVersionName getLiveCredentialSecretVersion(SqlUser user) {
+    return SecretVersionName.parse(
+        csmClient.getSecretData(getLiveLabelSecretId(user, dbInstance), Optional.empty()));
+  }
+
+  private static String getLiveLabelSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-live-label-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  private static String getCredentialDataSecretId(SqlUser user, String dbInstance) {
+    return String.format("sql-cred-data-%s-%s", user.geUserName(), dbInstance);
+  }
+
+  // WIP: when b/170230882 is complete, login will be versioned.
+  private static String createDatabaseLoginName(SqlUser user) {
+    return user.geUserName();
+  }
+}

core/src/main/java/google/registry/privileges/secretmanager/SqlUser.java

@@ -0,0 +1,67 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import com.google.common.base.Ascii;
+
+/**
+ * SQL user information for privilege management purposes.
+ *
+ * <p>A {@link RobotUser} represents a software system accessing the database using its own
+ * credential. Robots are well known and enumerated in {@link RobotId}.
+ */
+public abstract class SqlUser {
+
+  private final UserType type;
+  private final String userName;
+
+  protected SqlUser(UserType type, String userName) {
+    this.type = type;
+    this.userName = userName;
+  }
+
+  public UserType getType() {
+    return type;
+  }
+
+  public String geUserName() {
+    return userName;
+  }
+
+  /** Cloud SQL user types. Please see class javadoc of {@link SqlUser} for more information. */
+  enum UserType {
+    // Work in progress. Human user will be added.
+    ROBOT
+  }
+
+  /** Enumerates the {@link RobotUser RobotUsers} in the system. */
+  public enum RobotId {
+    NOMULUS,
+    /**
+     * Credential for RegistryTool. This is temporary, and will be removed when tool users are
+     * assigned their personal credentials.
+     */
+    TOOL;
+  }
+
+  /** Information of a RobotUser for privilege management purposes. */
+  // Work in progress. Eventually will be provided based on configuration.
+  public static class RobotUser extends SqlUser {
+
+    public RobotUser(RobotId robot) {
+      super(UserType.ROBOT, Ascii.toLowerCase(robot.name()));
+    }
+  }
+}

core/src/main/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParser.java

@@ -14,9 +14,11 @@
 
 package google.registry.reporting.spec11;
 
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
@@ -59,6 +61,25 @@ public class Spec11RegistrarThreatMatchesParser {
     return getFromFile(getGcsFilename(date));
   }
 
+  /** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */
+  public ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
+      throws IOException {
+    if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) {
+      return ImmutableSet.of();
+    }
+    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
+    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename);
+        InputStreamReader isr = new InputStreamReader(in, UTF_8)) {
+      // Skip the header at line 0
+      return Splitter.on("\n")
+          .omitEmptyStrings()
+          .splitToStream(CharStreams.toString(isr))
+          .skip(1)
+          .map(this::parseRegistrarThreatMatch)
+          .collect(toImmutableSet());
+    }
+  }
+
   public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
     LocalDate yesterday = date.minusDays(1);
     GcsFilename gcsFilename = getGcsFilename(yesterday);
@@ -82,20 +103,6 @@ public class Spec11RegistrarThreatMatchesParser {
     return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
   }
 
-  private ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
-      throws IOException, JSONException {
-    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
-    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
-      ImmutableList<String> reportLines =
-          ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
-      // Iterate from 1 to size() to skip the header at line 0.
-      for (int i = 1; i < reportLines.size(); i++) {
-        builder.add(parseRegistrarThreatMatch(reportLines.get(i)));
-      }
-      return builder.build();
-    }
-  }
-
   private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
     JSONObject reportJSON = new JSONObject(line);
     String clientId = reportJSON.getString(Spec11Pipeline.REGISTRAR_CLIENT_ID_FIELD);

core/src/main/java/google/registry/tools/DedupeOneTimeBillingEventIdsCommand.java

@@ -45,10 +45,10 @@ import google.registry.model.domain.DomainBase;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.OneTime entities with duplicate IDs.")
-public class DedupeOneTimeBillingEventIdsCommand extends DedupeEntityIdsCommand<OneTime> {
+public class DedupeOneTimeBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<OneTime> {
 
   @Override
-  void dedupe(OneTime entity) {
+  void process(OneTime entity) {
     Key<BillingEvent> key = Key.create(entity);
     Key<DomainBase> domainKey = getGrandParentAsDomain(key);
     DomainBase domain = ofy().load().key(domainKey).now();

core/src/main/java/google/registry/tools/DedupeRecurringBillingEventIdsCommand.java

@@ -54,11 +54,10 @@ import java.util.Set;
 @Parameters(
     separators = " =",
     commandDescription = "Dedupe BillingEvent.Recurring entities with duplicate IDs.")
-public class DedupeRecurringBillingEventIdsCommand
-    extends DedupeEntityIdsCommand<BillingEvent.Recurring> {
+public class DedupeRecurringBillingEventIdsCommand extends ReadEntityFromKeyPathCommand<Recurring> {
 
   @Override
-  void dedupe(Recurring recurring) {
+  void process(Recurring recurring) {
     // Loads the associated DomainBase and BillingEvent.OneTime entities that
     // may have reference to this BillingEvent.Recurring entity.
     Key<DomainBase> domainKey = getGrandParentAsDomain(Key.create(recurring));

core/src/main/java/google/registry/tools/GetSqlCredentialCommand.java

@@ -0,0 +1,73 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.io.FileOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to get a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(separators = " =", commandDescription = "Get the Cloud SQL Credential for a given user")
+public class GetSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"-o", "--output"},
+      description = "Name of output file for key data.",
+      validateWith = PathParameter.OutputFile.class)
+  private Path outputPath = null;
+
+  @Inject
+  GetSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+
+    SqlCredential credential;
+    try {
+      credential = store.getCredential(sqlUser);
+    } catch (SecretManagerException e) {
+      System.out.println(e.getMessage());
+      return;
+    }
+
+    if (outputPath == null) {
+      System.out.printf("[%s]\n", credential.toFormattedString());
+      return;
+    }
+    try (FileOutputStream out = new FileOutputStream(outputPath.toFile())) {
+      out.write(credential.toFormattedString().getBytes(StandardCharsets.UTF_8));
+    }
+  }
+}

core/src/main/java/google/registry/tools/ReadEntityFromKeyPathCommand.java

@@ -33,8 +33,13 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.List;
 
-/** Base Command to dedupe entities with duplicate IDs. */
-abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
+/**
+ * Base Command to read entities from Datastore by their key paths retrieved from BigQuery.
+ *
+ * <p>The key path is the value of column __key__.path of the entity's BigQuery table. Its value is
+ * converted from the entity's key.
+ */
+abstract class ReadEntityFromKeyPathCommand<T> extends MutatingCommand {
 
   @Parameter(
       names = "--key_paths_file",
@@ -48,7 +53,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
 
   private StringBuilder changeMessage = new StringBuilder();
 
-  abstract void dedupe(T entity);
+  abstract void process(T entity);
 
   @Override
   protected void init() throws Exception {
@@ -69,7 +74,7 @@ abstract class DedupeEntityIdsCommand<T> extends MutatingCommand {
       }
       Class<T> clazz = new TypeInstantiator<T>(getClass()) {}.getExactType();
       if (clazz.isInstance(entity)) {
-        dedupe((T) entity);
+        process((T) entity);
       } else {
         throw new IllegalArgumentException("Unsupported entity key: " + untypedKey);
       }

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
+import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 
@@ -32,6 +33,7 @@ public final class RegistryTool {
       new ImmutableMap.Builder<String, Class<? extends Command>>()
           .put("ack_poll_messages", AckPollMessagesCommand.class)
           .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
+          .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
@@ -78,6 +80,7 @@ public final class RegistryTool {
           .put("get_routing_map", GetRoutingMapCommand.class)
           .put("get_schema", GetSchemaCommand.class)
           .put("get_schema_tree", GetSchemaTreeCommand.class)
+          .put("get_sql_credential", GetSqlCredentialCommand.class)
           .put("get_tld", GetTldCommand.class)
           .put("ghostryde", GhostrydeCommand.class)
           .put("hash_certificate", HashCertificateCommand.class)
@@ -99,12 +102,15 @@ public final class RegistryTool {
           .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)
           .put("remove_ip_address", RemoveIpAddressCommand.class)
+          .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class)
           .put("renew_domain", RenewDomainCommand.class)
           .put("resave_entities", ResaveEntitiesCommand.class)
           .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
           .put("resave_epp_resource", ResaveEppResourceCommand.class)
+          .put("save_sql_credential", SaveSqlCredentialCommand.class)
           .put("send_escrow_report_to_icann", SendEscrowReportToIcannCommand.class)
           .put("set_num_instances", SetNumInstancesCommand.class)
+          .put("set_sql_replay_checkpoint", SetSqlReplayCheckpointCommand.class)
           .put("setup_ote", SetupOteCommand.class)
           .put("uniform_rapid_suspension", UniformRapidSuspensionCommand.class)
           .put("unlock_domain", UnlockDomainCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -34,6 +34,7 @@ import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.NomulusToolJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
@@ -74,6 +75,7 @@ import javax.inject.Singleton;
       PersistenceModule.class,
       RdeModule.class,
       RequestFactoryModule.class,
+      SecretManagerModule.class,
       URLFetchServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
@@ -118,6 +120,8 @@ interface RegistryToolComponent {
 
   void inject(GetOperationStatusCommand command);
 
+  void inject(GetSqlCredentialCommand command);
+
   void inject(GhostrydeCommand command);
 
   void inject(ImportDatastoreCommand command);
@@ -138,10 +142,14 @@ interface RegistryToolComponent {
 
   void inject(RenewDomainCommand command);
 
+  void inject(SaveSqlCredentialCommand command);
+
   void inject(SendEscrowReportToIcannCommand command);
 
   void inject(SetNumInstancesCommand command);
 
+  void inject(SetSqlReplayCheckpointCommand command);
+
   void inject(SetupOteCommand command);
 
   void inject(UnlockDomainCommand command);

core/src/main/java/google/registry/tools/RemoveRegistryOneKeyCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameters;
+import com.googlecode.objectify.Key;
+import google.registry.model.domain.DomainBase;
+import google.registry.persistence.VKey;
+import javax.annotation.Nullable;
+import org.joda.time.DateTime;
+
+/** Command to remove the Registry 1.0 key in {@link DomainBase} entity. */
+@Parameters(separators = " =", commandDescription = "Remove .")
+public class RemoveRegistryOneKeyCommand extends ReadEntityFromKeyPathCommand<DomainBase> {
+
+  @Override
+  void process(DomainBase entity) {
+    // Assert that the DomainBase entity must be deleted before 2017-08-01(most of the problematic
+    // entities were deleted before 2017, though there are still a few entities deleted in 2017-07).
+    // This is because we finished the Registry 2.0 migration in 2017 and should not generate any
+    // Registry 1.0 key after it.
+    if (!entity.getDeletionTime().isBefore(DateTime.parse("2017-08-01T00:00:00Z"))) {
+      throw new IllegalStateException(
+          String.format(
+              "Entity's deletion time %s is not before 2017-08-01T00:00:00Z",
+              entity.getDeletionTime()));
+    }
+    boolean hasChange = false;
+    DomainBase.Builder domainBuilder = entity.asBuilder();
+    // We only found the registry 1.0 key existed in fields autorenewBillingEvent,
+    // autorenewPollMessage and deletePollMessage so we just need to check these fields for each
+    // entity.
+    if (isRegistryOneKey(entity.getAutorenewBillingEvent())) {
+      domainBuilder.setAutorenewBillingEvent(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getAutorenewPollMessage())) {
+      domainBuilder.setAutorenewPollMessage(null);
+      hasChange = true;
+    }
+    if (isRegistryOneKey(entity.getDeletePollMessage())) {
+      domainBuilder.setDeletePollMessage(null);
+      hasChange = true;
+    }
+    if (hasChange) {
+      stageEntityChange(entity, domainBuilder.build());
+    }
+  }
+
+  private static boolean isRegistryOneKey(@Nullable VKey<?> vKey) {
+    if (vKey == null || vKey.getOfyKey() == null || vKey.getOfyKey().getParent() == null) {
+      return false;
+    }
+    Key<?> parentKey = vKey.getOfyKey().getParent();
+    return parentKey.getKind().equals("EntityGroupRoot") && parentKey.getName().equals("per-tld");
+  }
+}

core/src/main/java/google/registry/tools/SaveSqlCredentialCommand.java

@@ -0,0 +1,69 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Ascii;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import google.registry.tools.params.PathParameter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.inject.Inject;
+
+/**
+ * Command to create or update a Cloud SQL credential in the Secret Manager.
+ *
+ * <p>This command is a short-term tool that will be deprecated by the planned privilege server.
+ */
+@Parameters(
+    separators = " =",
+    commandDescription = "Create or update the Cloud SQL Credential for a given user")
+public class SaveSqlCredentialCommand implements Command {
+
+  @Inject SqlCredentialStore store;
+
+  @Parameter(names = "--user", description = "The Cloud SQL user.", required = true)
+  private String user;
+
+  @Parameter(
+      names = {"--input"},
+      description =
+          "Name of input file for the password. If absent, command will prompt for "
+              + "password in console.",
+      validateWith = PathParameter.InputFile.class)
+  private Path inputPath = null;
+
+  @Inject
+  SaveSqlCredentialCommand() {}
+
+  @Override
+  public void run() throws Exception {
+    String password = getPassword();
+    SqlUser sqlUser = new RobotUser(SqlUser.RobotId.valueOf(Ascii.toUpperCase(user)));
+    store.createOrUpdateCredential(sqlUser, password);
+    System.out.printf("\nDone:[%s]\n", password);
+  }
+
+  private String getPassword() throws Exception {
+    if (inputPath != null) {
+      return Files.readAllLines(inputPath, StandardCharsets.UTF_8).get(0);
+    }
+    return System.console().readLine("Please enter the password: ").trim();
+  }
+}

core/src/main/java/google/registry/tools/SetSqlReplayCheckpointCommand.java

@@ -0,0 +1,48 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.Iterables;
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import java.util.List;
+import org.joda.time.DateTime;
+
+/** Command to set {@link SqlReplayCheckpoint} to a particular, post-initial-population time. */
+@Parameters(separators = " =", commandDescription = "Set SqlReplayCheckpoint to a particular time")
+public class SetSqlReplayCheckpointCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  @Parameter(description = "Time to which SqlReplayCheckpoint will be set", required = true)
+  List<DateTime> mainParameters;
+
+  @Override
+  protected String prompt() {
+    checkArgument(mainParameters.size() == 1, "Must provide exactly one DateTime to set");
+    return String.format(
+        "Set SqlReplayCheckpoint to %s?", Iterables.getOnlyElement(mainParameters));
+  }
+
+  @Override
+  protected String execute() {
+    DateTime dateTime = Iterables.getOnlyElement(mainParameters);
+    jpaTm().transact(() -> SqlReplayCheckpoint.set(dateTime));
+    return String.format("Set SqlReplayCheckpoint time to %s", dateTime);
+  }
+}

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -0,0 +1,216 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.beam.spec11.ThreatMatch;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.model.reporting.Spec11ThreatMatchDao;
+import google.registry.reporting.spec11.RegistrarThreatMatches;
+import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.Clock;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+import javax.inject.Inject;
+import org.joda.time.LocalDate;
+
+/**
+ * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
+ *
+ * <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
+ * rough estimate of when we started using this format) and convert those RegistrarThreatMatches
+ * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
+ *
+ * <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
+ * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
+ * or if the domain objects were not active at the time of the scan.
+ */
+@Parameters(
+    commandDescription =
+        "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
+            + "Cloud SQL database.")
+public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
+
+  @Parameter(
+      names = {"-o", "--overwrite_existing_dates"},
+      description =
+          "Whether the command will overwrite data that already exists for dates that exist in the "
+              + "GCS bucket. Defaults to false.")
+  private boolean overrideExistingDates;
+
+  @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
+  // Inject the clock for testing purposes
+  @Inject Clock clock;
+
+  @Override
+  protected String prompt() {
+    return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
+  }
+
+  @Override
+  protected String execute() {
+    ImmutableList<LocalDate> dates = getDatesToBackfill();
+    ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
+        new ImmutableListMultimap.Builder<>();
+    for (LocalDate date : dates) {
+      try {
+        // It's OK if the file doesn't exist for a particular date; the result will be empty.
+        threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
+      } catch (IOException e) {
+        throw new RuntimeException(
+            String.format("Error parsing through file with date %s.", date), e);
+      }
+    }
+    ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
+        threatMatchesBuilder.build();
+    // Look up all possible DomainBases for these domain names, any of which can be in the past
+    ImmutableListMultimap<String, DomainBase> domainsByDomainName =
+        getDomainsByDomainName(threatMatches);
+
+    // For each date, convert all threat matches with the proper domain repo ID
+    int totalNumThreats = 0;
+    for (LocalDate date : threatMatches.keySet()) {
+      ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
+      for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
+        rtm.threatMatches().stream()
+            .map(
+                threatMatch ->
+                    threatMatchToCloudSqlObject(
+                        threatMatch, date, rtm.clientId(), domainsByDomainName))
+            .forEach(spec11ThreatsBuilder::add);
+      }
+      ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
+      jpaTm()
+          .transact(
+              () -> {
+                Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
+                jpaTm().putAll(spec11Threats);
+              });
+      totalNumThreats += spec11Threats.size();
+    }
+    return String.format(
+        "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
+  }
+
+  /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
+  private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
+      ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
+    return threatMatchesByDate.values().stream()
+        .map(RegistrarThreatMatches::threatMatches)
+        .flatMap(ImmutableList::stream)
+        .map(ThreatMatch::fullyQualifiedDomainName)
+        .distinct()
+        .collect(
+            flatteningToImmutableListMultimap(
+                Function.identity(),
+                (domainName) -> {
+                  List<DomainBase> domains =
+                      ofy()
+                          .load()
+                          .type(DomainBase.class)
+                          .filter("fullyQualifiedDomainName", domainName)
+                          .list();
+                  domains.sort(Comparator.comparing(DomainBase::getCreationTime).reversed());
+                  checkState(
+                      !domains.isEmpty(),
+                      "Domain name %s had no associated DomainBase objects.",
+                      domainName);
+                  return domains.stream();
+                }));
+  }
+
+  /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
+  private Spec11ThreatMatch threatMatchToCloudSqlObject(
+      ThreatMatch threatMatch,
+      LocalDate date,
+      String registrarId,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    DomainBase domain =
+        findDomainAsOfDateOrThrow(
+            threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
+    return new Spec11ThreatMatch.Builder()
+        .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
+        .setCheckDate(date)
+        .setRegistrarId(registrarId)
+        .setDomainName(threatMatch.fullyQualifiedDomainName())
+        .setDomainRepoId(domain.getRepoId())
+        .build();
+  }
+
+  /** Returns the DomainBase object as of the particular date, which is likely in the past. */
+  private DomainBase findDomainAsOfDateOrThrow(
+      String domainName,
+      LocalDate date,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
+    for (DomainBase domain : domains) {
+      // We only know the date (not datetime) of the threat scan, so we approximate
+      LocalDate creationDate = domain.getCreationTime().toLocalDate();
+      LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
+      if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
+        return domain;
+      }
+    }
+    throw new IllegalStateException(
+        String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
+  }
+
+  /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
+  private ImmutableList<LocalDate> getDatesToBackfill() {
+    ImmutableSet<LocalDate> datesToSkip =
+        overrideExistingDates ? ImmutableSet.of() : getExistingDates();
+    ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
+    LocalDate endDate = clock.nowUtc().toLocalDate();
+    for (LocalDate currentDate = START_DATE;
+        !currentDate.isAfter(endDate);
+        currentDate = currentDate.plusDays(1)) {
+      if (!datesToSkip.contains(currentDate)) {
+        result.add(currentDate);
+      }
+    }
+    return result.build();
+  }
+
+  private ImmutableSet<LocalDate> getExistingDates() {
+    return jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .getEntityManager()
+                    .createQuery(
+                        "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
+                    .getResultStream()
+                    .collect(toImmutableSet()));
+  }
+}

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -20,7 +20,6 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.withSettings;
 
@@ -285,8 +284,7 @@ class Spec11PipelineTest {
             .build();
 
     verify(mockJpaTm).transact(any(Runnable.class));
-    verify(mockJpaTm).insert(expected);
-    verifyNoMoreInteractions(mockJpaTm);
+    verify(mockJpaTm).putAll(ImmutableList.of(expected));
   }
 
   /**

core/src/test/java/google/registry/model/UpdateAutoTimestampTest.java

@@ -15,64 +15,106 @@
 package google.registry.model;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.joda.time.DateTimeZone.UTC;
 
+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.persistence.VKey;
 import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.FakeClock;
+import google.registry.testing.TestOfyAndSql;
 import org.joda.time.DateTime;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link UpdateAutoTimestamp}. */
+@DualDatabaseTest
 public class UpdateAutoTimestampTest {
 
+  FakeClock clock = new FakeClock();
+
   @RegisterExtension
   public final AppEngineExtension appEngine =
       AppEngineExtension.builder()
           .withDatastoreAndCloudSql()
+          .withJpaUnitTestEntities(UpdateAutoTimestampTestObject.class)
           .withOfyTestEntities(UpdateAutoTimestampTestObject.class)
+          .withClock(clock)
           .build();
 
   /** Timestamped class. */
   @Entity(name = "UatTestEntity")
+  @javax.persistence.Entity
   @EntityForTesting
   public static class UpdateAutoTimestampTestObject extends CrossTldSingleton {
+    @Ignore @javax.persistence.Id long id = SINGLETON_ID;
     UpdateAutoTimestamp updateTime = UpdateAutoTimestamp.create(null);
   }
 
   private UpdateAutoTimestampTestObject reload() {
-    return ofy().load().entity(new UpdateAutoTimestampTestObject()).now();
+    return tm().transact(
+            () ->
+                tm().load(
+                        VKey.create(
+                            UpdateAutoTimestampTestObject.class,
+                            1L,
+                            Key.create(new UpdateAutoTimestampTestObject()))));
   }
 
-  @Test
+  @TestOfyAndSql
   void testSaveSetsTime() {
     DateTime transactionTime =
         tm().transact(
                 () -> {
                   UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                   assertThat(object.updateTime.timestamp).isNull();
-                  ofy().save().entity(object);
+                  tm().insert(object);
                   return tm().getTransactionTime();
                 });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
   }
 
-  @Test
+  @TestOfyAndSql
+  void testDisabledUpdates() throws Exception {
+    DateTime initialTime =
+        tm().transact(
+                () -> {
+                  tm().insert(new UpdateAutoTimestampTestObject());
+                  return tm().getTransactionTime();
+                });
+
+    UpdateAutoTimestampTestObject object = reload();
+    clock.advanceOneMilli();
+
+    try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+        new UpdateAutoTimestamp.DisableAutoUpdateResource()) {
+      DateTime secondTransactionTime =
+          tm().transact(
+                  () -> {
+                    tm().put(object);
+                    return tm().getTransactionTime();
+                  });
+      assertThat(secondTransactionTime).isGreaterThan(initialTime);
+    }
+    assertThat(reload().updateTime.timestamp).isEqualTo(initialTime);
+  }
+
+  @TestOfyAndSql
   void testResavingOverwritesOriginalTime() {
     DateTime transactionTime =
         tm().transact(
                 () -> {
                   UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                   object.updateTime = UpdateAutoTimestamp.create(DateTime.now(UTC).minusDays(1));
-                  ofy().save().entity(object);
+                  tm().insert(object);
                   return tm().getTransactionTime();
                 });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
     assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
   }
 }

core/src/test/java/google/registry/model/domain/DomainBaseSqlTest.java

@@ -84,10 +84,10 @@ public class DomainBaseSqlTest {
     saveRegistrar("registrar1");
     saveRegistrar("registrar2");
     saveRegistrar("registrar3");
-    contactKey = VKey.createSql(ContactResource.class, "contact_id1");
-    contact2Key = VKey.createSql(ContactResource.class, "contact_id2");
+    contactKey = createKey(ContactResource.class, "contact_id1");
+    contact2Key = createKey(ContactResource.class, "contact_id2");
 
-    host1VKey = VKey.createSql(HostResource.class, "host1");
+    host1VKey = createKey(HostResource.class, "host1");
 
     domain =
         new DomainBase.Builder()
@@ -696,6 +696,10 @@ public class DomainBaseSqlTest {
     assertThat(domain.getGracePeriods()).isEqualTo(gracePeriods);
   }
 
+  private <T> VKey<T> createKey(Class<T> clazz, String name) {
+    return VKey.create(clazz, name, Key.create(clazz, name));
+  }
+
   private <T> VKey<T> createLegacyVKey(Class<T> clazz, long id) {
     return VKey.create(
         clazz, id, Key.create(Key.create(EntityGroupRoot.class, "per-tld"), clazz, id));

core/src/test/java/google/registry/model/history/DomainHistoryTest.java

@@ -14,7 +14,6 @@
 
 package google.registry.model.history;
 
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
@@ -92,10 +91,7 @@ public class DomainHistoryTest extends EntityTestCase {
               assertDomainHistoriesEqual(fromDatabase, domainHistory);
               assertThat(fromDatabase.getParentVKey()).isEqualTo(domainHistory.getParentVKey());
               assertThat(fromDatabase.getNsHosts())
-                  .containsExactlyElementsIn(
-                      domainHistory.getNsHosts().stream()
-                          .map(key -> VKey.createSql(HostResource.class, key.getSqlKey()))
-                          .collect(toImmutableSet()));
+                  .containsExactlyElementsIn(domainHistory.getNsHosts());
             });
   }
 

core/src/test/java/google/registry/model/host/HostResourceTest.java

@@ -17,13 +17,17 @@ package google.registry.model.host;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
+import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.cloneAndSetAutoTimestamps;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistNewRegistrars;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.HostResourceSubject.assertAboutHosts;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.net.InetAddresses;
 import google.registry.model.EntityTestCase;
@@ -32,11 +36,14 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.eppcommon.Trid;
 import google.registry.model.transfer.DomainTransferData;
 import google.registry.model.transfer.TransferStatus;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link HostResource}. */
+@DualDatabaseTest
 class HostResourceTest extends EntityTestCase {
 
   private final DateTime day3 = fakeClock.nowUtc();
@@ -49,6 +56,7 @@ class HostResourceTest extends EntityTestCase {
   @BeforeEach
   void setUp() {
     createTld("com");
+    persistNewRegistrars("gaining", "losing", "thisRegistrar", "thatRegistrar");
     // Set up a new persisted registrar entity.
     domain =
         persistResource(
@@ -71,9 +79,9 @@ class HostResourceTest extends EntityTestCase {
                 new HostResource.Builder()
                     .setRepoId("DEADBEEF-COM")
                     .setHostName("ns1.example.com")
-                    .setCreationClientId("a registrar")
+                    .setCreationClientId("thisRegistrar")
                     .setLastEppUpdateTime(fakeClock.nowUtc())
-                    .setLastEppUpdateClientId("another registrar")
+                    .setLastEppUpdateClientId("thatRegistrar")
                     .setLastTransferTime(fakeClock.nowUtc())
                     .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
                     .setStatusValues(ImmutableSet.of(StatusValue.OK))
@@ -81,13 +89,22 @@ class HostResourceTest extends EntityTestCase {
                     .build()));
   }
 
-  @Test
+  @TestOfyAndSql
   void testPersistence() {
+    HostResource newHost = host.asBuilder().setRepoId("NEWHOST").build();
+    tm().transact(() -> tm().insert(newHost));
+    assertThat(ImmutableList.of(tm().transact(() -> tm().load(newHost.createVKey()))))
+        .comparingElementsUsing(immutableObjectCorrespondence("revisions"))
+        .containsExactly(newHost);
+  }
+
+  @TestOfyOnly
+  void testLoadingByForeignKey() {
     assertThat(loadByForeignKey(HostResource.class, host.getForeignKey(), fakeClock.nowUtc()))
         .hasValue(host);
   }
 
-  @Test
+  @TestOfyOnly
   void testIndexing() throws Exception {
     // Clone it and save it before running the indexing test so that its transferData fields are
     // populated from the superordinate domain.
@@ -100,7 +117,7 @@ class HostResourceTest extends EntityTestCase {
         "currentSponsorClientId");
   }
 
-  @Test
+  @TestOfyAndSql
   void testEmptyStringsBecomeNull() {
     assertThat(
             new HostResource.Builder()
@@ -122,7 +139,7 @@ class HostResourceTest extends EntityTestCase {
         .isNotNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testEmptySetsBecomeNull() {
     assertThat(new HostResource.Builder().setInetAddresses(null).build().inetAddresses).isNull();
     assertThat(new HostResource.Builder().setInetAddresses(ImmutableSet.of()).build().inetAddresses)
@@ -135,7 +152,7 @@ class HostResourceTest extends EntityTestCase {
         .isNotNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testImplicitStatusValues() {
     // OK is implicit if there's no other statuses.
     assertAboutHosts()
@@ -157,13 +174,13 @@ class HostResourceTest extends EntityTestCase {
         .hasExactlyStatusValues(StatusValue.CLIENT_HOLD);
   }
 
-  @Test
+  @TestOfyAndSql
   void testToHydratedString_notCircular() {
     // If there are circular references, this will overflow the stack.
     host.toHydratedString();
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_uppercaseHostName() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -173,7 +190,7 @@ class HostResourceTest extends EntityTestCase {
         .contains("Host name must be in puny-coded, lower-case form");
   }
 
-  @Test
+  @TestOfyAndSql
   void testFailure_utf8HostName() {
     IllegalArgumentException thrown =
         assertThrows(
@@ -183,14 +200,14 @@ class HostResourceTest extends EntityTestCase {
         .contains("Host name must be in puny-coded, lower-case form");
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostNeverSwitchedDomains_domainWasNeverTransferred() {
     domain = domain.asBuilder().setLastTransferTime(null).build();
     host = host.asBuilder().setLastTransferTime(null).setLastSuperordinateChange(null).build();
     assertThat(host.computeLastTransferTime(domain)).isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostNeverSwitchedDomains_domainWasTransferred() {
     // Host was created on Day 1.
     // Domain was transferred on Day 2.
@@ -205,7 +222,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isEqualTo(day2);
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostCreatedAfterDomainWasTransferred() {
     // Domain was transferred on Day 1.
     // Host was created subordinate to domain on Day 2.
@@ -217,9 +234,9 @@ class HostResourceTest extends EntityTestCase {
                     .setCreationTime(day2)
                     .setRepoId("DEADBEEF-COM")
                     .setHostName("ns1.example.com")
-                    .setCreationClientId("a registrar")
+                    .setCreationClientId("thisRegistrar")
                     .setLastEppUpdateTime(fakeClock.nowUtc())
-                    .setLastEppUpdateClientId("another registrar")
+                    .setLastEppUpdateClientId("thatRegistrar")
                     .setInetAddresses(ImmutableSet.of(InetAddresses.forString("127.0.0.1")))
                     .setStatusValues(ImmutableSet.of(StatusValue.OK))
                     .setSuperordinateDomain(domain.createVKey())
@@ -227,7 +244,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isNull();
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_hostWasTransferred_domainWasNeverTransferred() {
     // Host was transferred on Day 1.
     // Host was made subordinate to domain on Day 2.
@@ -237,7 +254,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isEqualTo(day1);
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_domainWasTransferredBeforeHostBecameSubordinate() {
     // Host was transferred on Day 1.
     // Domain was transferred on Day 2.
@@ -247,7 +264,7 @@ class HostResourceTest extends EntityTestCase {
     assertThat(host.computeLastTransferTime(domain)).isEqualTo(day1);
   }
 
-  @Test
+  @TestOfyAndSql
   void testComputeLastTransferTime_domainWasTransferredAfterHostBecameSubordinate() {
     // Host was transferred on Day 1.
     // Host was made subordinate to domain on Day 2.

core/src/test/java/google/registry/model/smd/SignedMarkRevocationListDaoTest.java

@@ -50,21 +50,6 @@ public class SignedMarkRevocationListDaoTest {
     assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
   }
 
-  @Test
-  void trySave_failureIsSwallowed() {
-    SignedMarkRevocationList list =
-        SignedMarkRevocationList.create(
-            fakeClock.nowUtc(), ImmutableMap.of("mark", fakeClock.nowUtc().minusHours(1)));
-    SignedMarkRevocationListDao.trySave(list);
-    SignedMarkRevocationList fromDb = SignedMarkRevocationListDao.getLatestRevision().get();
-    assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
-
-    // This should throw an exception, which is swallowed and nothing changed
-    SignedMarkRevocationListDao.trySave(list);
-    SignedMarkRevocationList secondFromDb = SignedMarkRevocationListDao.getLatestRevision().get();
-    assertAboutImmutableObjects().that(secondFromDb).isEqualExceptFields(fromDb);
-  }
-
   @Test
   void testRetrieval_notPresent() {
     assertThat(SignedMarkRevocationListDao.getLatestRevision().isPresent()).isFalse();

core/src/test/java/google/registry/model/smd/SignedMarkRevocationListTest.java

@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.smd.SignedMarkRevocationList.SHARD_SIZE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.time.Duration.standardDays;
@@ -138,6 +139,33 @@ public class SignedMarkRevocationListTest {
         .isEqualTo(DateTime.parse("2000-01-01T00:00:00Z"));
   }
 
+  @Test
+  void test_getCreationTime_missingInCloudSQL() {
+    clock.setTo(DateTime.parse("2000-01-01T00:00:00Z"));
+    createSaveGetHelper(1);
+    jpaTm().transact(() -> jpaTm().delete(SignedMarkRevocationListDao.getLatestRevision().get()));
+    RuntimeException thrown =
+        assertThrows(RuntimeException.class, () -> SignedMarkRevocationList.get());
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("Signed mark revocation list in Cloud SQL is empty.");
+  }
+
+  @Test
+  void test_getCreationTime_unequalListsInDatabases() {
+    clock.setTo(DateTime.parse("2000-01-01T00:00:00Z"));
+    createSaveGetHelper(1);
+    ImmutableMap.Builder<String, DateTime> revokes = new ImmutableMap.Builder<>();
+    for (int i = 0; i < 3; i++) {
+      revokes.put(Integer.toString(i), clock.nowUtc());
+    }
+    SignedMarkRevocationListDao.trySave(
+        SignedMarkRevocationList.create(clock.nowUtc(), revokes.build()));
+    RuntimeException thrown =
+        assertThrows(RuntimeException.class, () -> SignedMarkRevocationList.get());
+    assertThat(thrown).hasMessageThat().contains("Unequal SM revocation lists detected:");
+  }
+
   @Test
   void test_isSmdRevoked_present() {
     final int rows = SHARD_SIZE + 1;

core/src/test/java/google/registry/persistence/converter/LongVKeyConverterTest.java

@@ -19,8 +19,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithLongVKey;
-import google.registry.persistence.transaction.JpaTestRules;
-import google.registry.persistence.transaction.JpaTestRules.JpaUnitTestExtension;
+import google.registry.testing.AppEngineExtension;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.junit.jupiter.api.Test;
@@ -30,33 +29,56 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 public class LongVKeyConverterTest {
 
   @RegisterExtension
-  public final JpaUnitTestExtension jpaExtension =
-      new JpaTestRules.Builder()
-          .withEntityClass(TestEntity.class, VKeyConverter_LongType.class)
-          .buildUnitTestRule();
+  public final AppEngineExtension appEngineExtension =
+      new AppEngineExtension.Builder()
+          .withDatastoreAndCloudSql()
+          .withoutCannedData()
+          .withJpaUnitTestEntities(
+              TestLongEntity.class,
+              VKeyConverter_LongType.class,
+              VKeyConverter_CompositeLongType.class)
+          .withOfyTestEntities(TestLongEntity.class, CompositeKeyTestLongEntity.class)
+          .build();
 
   @Test
   void testRoundTrip() {
-    TestEntity original = new TestEntity(VKey.createSql(TestEntity.class, 10L));
+    TestLongEntity original =
+        new TestLongEntity(
+            VKey.createSql(TestLongEntity.class, 10L),
+            VKey.createSql(CompositeKeyTestLongEntity.class, 20L));
     jpaTm().transact(() -> jpaTm().getEntityManager().persist(original));
 
-    TestEntity retrieved =
-        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "id"));
+    TestLongEntity retrieved =
+        jpaTm().transact(() -> jpaTm().getEntityManager().find(TestLongEntity.class, "id"));
     assertThat(retrieved.number.getSqlKey()).isEqualTo(10L);
+    assertThat(retrieved.number.getOfyKey().getId()).isEqualTo(10L);
+
+    assertThat(retrieved.composite.getSqlKey()).isEqualTo(20L);
+    assertThat(retrieved.composite.maybeGetOfyKey().isPresent()).isFalse();
   }
 
-  @Entity(name = "TestEntity")
+  @Entity(name = "TestLongEntity")
+  @com.googlecode.objectify.annotation.Entity
   @WithLongVKey(classNameSuffix = "LongType")
-  static class TestEntity {
-    @Id String id = "id";
+  static class TestLongEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id = "id";
 
-    VKey<TestEntity> number;
+    VKey<TestLongEntity> number;
+    VKey<CompositeKeyTestLongEntity> composite;
 
-    TestEntity(VKey<TestEntity> number) {
+    TestLongEntity(VKey<TestLongEntity> number, VKey<CompositeKeyTestLongEntity> composite) {
       this.number = number;
+      this.composite = composite;
     }
 
     /** Default constructor, needed for hibernate. */
-    public TestEntity() {}
+    public TestLongEntity() {}
+  }
+
+  @Entity(name = "CompositeKeyTestLongEntity")
+  @com.googlecode.objectify.annotation.Entity
+  @WithLongVKey(classNameSuffix = "CompositeLongType", compositeKey = true)
+  static class CompositeKeyTestLongEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id = "id";
   }
 }

core/src/test/java/google/registry/persistence/converter/StringVKeyConverterTest.java

@@ -19,8 +19,7 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import google.registry.persistence.VKey;
 import google.registry.persistence.WithStringVKey;
-import google.registry.persistence.transaction.JpaTestRules;
-import google.registry.persistence.transaction.JpaTestRules.JpaUnitTestExtension;
+import google.registry.testing.AppEngineExtension;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import org.junit.jupiter.api.Test;
@@ -30,36 +29,61 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 public class StringVKeyConverterTest {
 
   @RegisterExtension
-  public final JpaUnitTestExtension jpaExtension =
-      new JpaTestRules.Builder()
-          .withEntityClass(TestEntity.class, VKeyConverter_StringType.class)
-          .buildUnitTestRule();
+  public final AppEngineExtension appEngineExtension =
+      new AppEngineExtension.Builder()
+          .withDatastoreAndCloudSql()
+          .withoutCannedData()
+          .withJpaUnitTestEntities(
+              TestStringEntity.class,
+              VKeyConverter_StringType.class,
+              VKeyConverter_CompositeStringType.class)
+          .withOfyTestEntities(TestStringEntity.class, CompositeKeyTestStringEntity.class)
+          .build();
 
   @Test
   void testRoundTrip() {
-    TestEntity original =
-        new TestEntity("TheRealSpartacus", VKey.createSql(TestEntity.class, "ImSpartacus!"));
+    TestStringEntity original =
+        new TestStringEntity(
+            "TheRealSpartacus",
+            VKey.createSql(TestStringEntity.class, "ImSpartacus!"),
+            VKey.createSql(CompositeKeyTestStringEntity.class, "NoImSpartacus!"));
     jpaTm().transact(() -> jpaTm().getEntityManager().persist(original));
 
-    TestEntity retrieved =
+    TestStringEntity retrieved =
         jpaTm()
-            .transact(() -> jpaTm().getEntityManager().find(TestEntity.class, "TheRealSpartacus"));
+            .transact(
+                () -> jpaTm().getEntityManager().find(TestStringEntity.class, "TheRealSpartacus"));
     assertThat(retrieved.other.getSqlKey()).isEqualTo("ImSpartacus!");
+    assertThat(retrieved.other.getOfyKey().getName()).isEqualTo("ImSpartacus!");
+
+    assertThat(retrieved.composite.getSqlKey()).isEqualTo("NoImSpartacus!");
+    assertThat(retrieved.composite.maybeGetOfyKey().isPresent()).isFalse();
   }
 
-  @Entity(name = "TestEntity")
+  @Entity(name = "TestStringEntity")
+  @com.googlecode.objectify.annotation.Entity
   @WithStringVKey(classNameSuffix = "StringType")
-  static class TestEntity {
-    @Id String id;
+  static class TestStringEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id;
 
-    VKey<TestEntity> other;
+    VKey<TestStringEntity> other;
+    VKey<CompositeKeyTestStringEntity> composite;
 
-    TestEntity(String id, VKey<TestEntity> other) {
+    TestStringEntity(
+        String id, VKey<TestStringEntity> other, VKey<CompositeKeyTestStringEntity> composite) {
       this.id = id;
       this.other = other;
+      this.composite = composite;
     }
 
     /** Default constructor, needed for hibernate. */
-    public TestEntity() {}
+    public TestStringEntity() {}
+  }
+
+  @Entity(name = "CompositeKeyTestStringEntity")
+  @com.googlecode.objectify.annotation.Entity
+  @WithStringVKey(classNameSuffix = "CompositeStringType", compositeKey = true)
+  static class CompositeKeyTestStringEntity {
+    @com.googlecode.objectify.annotation.Id @Id String id = "id";
   }
 }

core/src/test/java/google/registry/privileges/secretmanager/FakeSecretManagerClient.java

@@ -32,6 +32,11 @@ public class FakeSecretManagerClient implements SecretManagerClient {
   @Inject
   FakeSecretManagerClient() {}
 
+  @Override
+  public String getProject() {
+    return "fake_project";
+  }
+
   @Override
   public void createSecret(String secretId) {
     checkNotNull(secretId, "secretId");
@@ -41,6 +46,12 @@ public class FakeSecretManagerClient implements SecretManagerClient {
     secrets.put(secretId, new SecretEntry(secretId));
   }
 
+  @Override
+  public boolean secretExists(String secretId) {
+    checkNotNull(secretId, "secretId");
+    return secrets.containsKey(secretId);
+  }
+
   @Override
   public Iterable<String> listSecrets() {
     return ImmutableSet.copyOf(secrets.keySet());
@@ -78,6 +89,28 @@ public class FakeSecretManagerClient implements SecretManagerClient {
     return secretEntry.getVersion(version).getData();
   }
 
+  @Override
+  public void enableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    secretEntry.enableVersion(version);
+  }
+
+  @Override
+  public void disableSecretVersion(String secretId, String version) {
+    checkNotNull(secretId, "secretId");
+    checkNotNull(version, "version");
+    SecretEntry secretEntry = secrets.get(secretId);
+    if (secretEntry == null) {
+      throw new NoSuchSecretResourceException(null);
+    }
+    secretEntry.disableVersion(version);
+  }
+
   @Override
   public void destroySecretVersion(String secretId, String version) {
     checkNotNull(secretId, "secretId");
@@ -118,6 +151,20 @@ public class FakeSecretManagerClient implements SecretManagerClient {
       return state;
     }
 
+    void enable() {
+      if (state.equals(State.DESTROYED)) {
+        throw new SecretManagerException(null);
+      }
+      state = State.ENABLED;
+    }
+
+    void disable() {
+      if (state.equals(State.DESTROYED)) {
+        throw new SecretManagerException(null);
+      }
+      state = State.DISABLED;
+    }
+
     void destroy() {
       data = null;
       state = State.DESTROYED;
@@ -145,6 +192,8 @@ public class FakeSecretManagerClient implements SecretManagerClient {
         return versions.get(index);
       } catch (NumberFormatException e) {
         throw new IllegalArgumentException("Invalid version " + version.get());
+      } catch (ArrayIndexOutOfBoundsException e) {
+        throw new NoSuchSecretResourceException(null);
       }
     }
 
@@ -156,15 +205,16 @@ public class FakeSecretManagerClient implements SecretManagerClient {
       return builder.build();
     }
 
+    void enableVersion(String version) {
+      getVersion(Optional.of(version)).enable();
+    }
+
+    void disableVersion(String version) {
+      getVersion(Optional.of(version)).disable();
+    }
+
     void destroyVersion(String version) {
-      try {
-        int index = Integer.valueOf(version);
-        versions.get(index).destroy();
-      } catch (NumberFormatException e) {
-        throw new IllegalArgumentException("Invalid version " + version);
-      } catch (ArrayIndexOutOfBoundsException e) {
-        throw new NoSuchSecretResourceException(null);
-      }
+      getVersion(Optional.of(version)).destroy();
     }
   }
 }

core/src/test/java/google/registry/privileges/secretmanager/SecretManagerClientTest.java

@@ -17,13 +17,20 @@ package google.registry.privileges.secretmanager;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import com.google.cloud.secretmanager.v1.SecretVersion.State;
+import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
 import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
 import google.registry.privileges.secretmanager.SecretManagerClient.SecretManagerException;
+import google.registry.util.Retrier;
+import google.registry.util.SystemSleeper;
+import java.io.IOException;
 import java.util.Optional;
 import java.util.UUID;
-import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -43,52 +50,53 @@ public class SecretManagerClientTest {
   private static final String SECRET_ID_PREFIX = "TEST_" + UUID.randomUUID() + "_";
   // Used for unique secret id generation.
   private static int seqno = 0;
-
   private static SecretManagerClient secretManagerClient;
   private static boolean isUnitTest = true;
 
-  private static String nextSecretId() {
-    return SECRET_ID_PREFIX + seqno++;
-  }
+  private String secretId;
 
   @BeforeAll
-  static void beforeAll() {
+  static void beforeAll() throws IOException {
     String environmentName = System.getProperty("test.gcp_integration.env");
     if (environmentName != null) {
       secretManagerClient =
-          DaggerSecretManagerModule_SecretManagerComponent.builder()
-              .secretManagerModule(
-                  new SecretManagerModule(String.format("domain-registry-%s", environmentName)))
-              .build()
-              .secretManagerClient();
+          SecretManagerModule.provideSecretManagerClient(
+              SecretManagerServiceSettings.newBuilder()
+                  .setCredentialsProvider(() -> GoogleCredentials.getApplicationDefault())
+                  .build(),
+              String.format("domain-registry-%s", environmentName),
+              new Retrier(new SystemSleeper(), 1));
       isUnitTest = false;
     } else {
       secretManagerClient = new FakeSecretManagerClient();
     }
   }
 
-  @AfterAll
-  static void afterAll() {
+  @BeforeEach
+  void beforeEach() {
+    secretId = SECRET_ID_PREFIX + seqno++;
+  }
+
+  @AfterEach
+  void afterEach() throws IOException {
     if (isUnitTest) {
       return;
     }
-    for (String secretId : secretManagerClient.listSecrets()) {
-      if (secretId.startsWith(SECRET_ID_PREFIX)) {
-        secretManagerClient.deleteSecret(secretId);
-      }
+    try {
+      secretManagerClient.deleteSecret(secretId);
+    } catch (NoSuchSecretResourceException e) {
+      // deleteSecret() deleted it already.
     }
   }
 
   @Test
   void createSecret_success() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     assertThat(secretManagerClient.listSecrets()).contains(secretId);
   }
 
   @Test
   void createSecret_duplicate() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     assertThrows(
         SecretAlreadyExistsException.class, () -> secretManagerClient.createSecret(secretId));
@@ -96,16 +104,25 @@ public class SecretManagerClientTest {
 
   @Test
   void addSecretVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     String version = secretManagerClient.addSecretVersion(secretId, "mydata");
     assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED))
         .containsExactly(version);
   }
 
+  @Test
+  void secretExists_true() {
+    secretManagerClient.createSecret(secretId);
+    assertThat(secretManagerClient.secretExists(secretId)).isTrue();
+  }
+
+  @Test
+  void secretExists_False() {
+    assertThat(secretManagerClient.secretExists(secretId)).isFalse();
+  }
+
   @Test
   void getSecretData_byVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     String version = secretManagerClient.addSecretVersion(secretId, "mydata");
     assertThat(secretManagerClient.getSecretData(secretId, Optional.of(version)))
@@ -114,15 +131,70 @@ public class SecretManagerClientTest {
 
   @Test
   void getSecretData_latestVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     secretManagerClient.addSecretVersion(secretId, "mydata");
     assertThat(secretManagerClient.getSecretData(secretId, Optional.empty())).isEqualTo("mydata");
   }
 
+  @Test
+  void disableSecretVersion() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.disableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
+  }
+
+  @Test
+  void disableSecretVersion_ignoreAlreadyDisabled() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.disableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
+    secretManagerClient.disableSecretVersion(secretId, version);
+  }
+
+  @Test
+  void disableSecretVersion_destroyed() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.destroySecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DESTROYED)).contains(version);
+    assertThrows(
+        SecretManagerException.class,
+        () -> secretManagerClient.disableSecretVersion(secretId, version));
+  }
+
+  @Test
+  void enableSecretVersion_ignoreAlreadyEnabled() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED)).contains(version);
+    secretManagerClient.enableSecretVersion(secretId, version);
+  }
+
+  @Test
+  void enableSecretVersion() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.disableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DISABLED)).contains(version);
+    secretManagerClient.enableSecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.ENABLED)).contains(version);
+  }
+
+  @Test
+  void enableSecretVersion_destroyed() {
+    secretManagerClient.createSecret(secretId);
+    String version = secretManagerClient.addSecretVersion(secretId, "mydata");
+    secretManagerClient.destroySecretVersion(secretId, version);
+    assertThat(secretManagerClient.listSecretVersions(secretId, State.DESTROYED)).contains(version);
+    assertThrows(
+        SecretManagerException.class,
+        () -> secretManagerClient.enableSecretVersion(secretId, version));
+  }
+
   @Test
   void destroySecretVersion() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     String version = secretManagerClient.addSecretVersion(secretId, "mydata");
     secretManagerClient.destroySecretVersion(secretId, version);
@@ -134,7 +206,6 @@ public class SecretManagerClientTest {
 
   @Test
   void deleteSecret() {
-    String secretId = nextSecretId();
     secretManagerClient.createSecret(secretId);
     assertThat(secretManagerClient.listSecrets()).contains(secretId);
     secretManagerClient.deleteSecret(secretId);

core/src/test/java/google/registry/privileges/secretmanager/SqlCredentialStoreTest.java

@@ -0,0 +1,63 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.privileges.secretmanager;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import com.google.cloud.secretmanager.v1.SecretVersionName;
+import google.registry.privileges.secretmanager.SqlUser.RobotId;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
+import java.util.Optional;
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link SqlCredentialStore}. */
+public class SqlCredentialStoreTest {
+
+  private final SecretManagerClient client = new FakeSecretManagerClient();
+  private final SqlCredentialStore credentialStore = new SqlCredentialStore(client, "db");
+  private SqlUser user = new RobotUser(RobotId.NOMULUS);
+
+  @Test
+  void createSecret() {
+    credentialStore.createOrUpdateCredential(user, "password");
+    assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isTrue();
+    assertThat(
+            SecretVersionName.parse(
+                    client.getSecretData("sql-cred-live-label-nomulus-db", Optional.empty()))
+                .getSecret())
+        .isEqualTo("sql-cred-data-nomulus-db");
+    assertThat(client.secretExists("sql-cred-data-nomulus-db")).isTrue();
+    assertThat(client.getSecretData("sql-cred-data-nomulus-db", Optional.empty()))
+        .isEqualTo("nomulus password");
+  }
+
+  @Test
+  void getCredential() {
+    credentialStore.createOrUpdateCredential(user, "password");
+    SqlCredential credential = credentialStore.getCredential(user);
+    assertThat(credential.login()).isEqualTo("nomulus");
+    assertThat(credential.password()).isEqualTo("password");
+  }
+
+  @Test
+  void deleteCredential() {
+    credentialStore.createOrUpdateCredential(user, "password");
+    assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isTrue();
+    assertThat(client.secretExists("sql-cred-data-nomulus-db")).isTrue();
+    credentialStore.deleteCredential(user);
+    assertThat(client.secretExists("sql-cred-live-label-nomulus-db")).isFalse();
+    assertThat(client.secretExists("sql-cred-data-nomulus-db")).isFalse();
+  }
+}

core/src/test/java/google/registry/reporting/spec11/Spec11RegistrarThreatMatchesParserTest.java

@@ -34,7 +34,7 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /** Unit tests for {@link Spec11RegistrarThreatMatchesParser}. */
-class Spec11RegistrarThreatMatchesParserTest {
+public class Spec11RegistrarThreatMatchesParserTest {
 
   private static final String TODAY = "2018-07-21";
   private static final String YESTERDAY = "2018-07-20";
@@ -66,6 +66,12 @@ class Spec11RegistrarThreatMatchesParserTest {
     assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))).isEmpty();
   }
 
+  @Test
+  void testNonexistent_returnsEmpty() throws Exception {
+    assertThat(parser.getRegistrarThreatMatches(LocalDate.parse(YESTERDAY).minusYears(1)))
+        .isEmpty();
+  }
+
   @Test
   void testFindPrevious_olderThanYesterdayFound() throws Exception {
     setupFile("spec11_fake_report_previous_day", "2018-07-14");
@@ -95,7 +101,7 @@ class Spec11RegistrarThreatMatchesParserTest {
   }
 
   /** The expected contents of the sample spec11 report file */
-  static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
+  public static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
     return ImmutableSet.of(getMatchA(), getMatchB());
   }
 

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -750,6 +750,20 @@ public class DatabaseHelper {
             .build());
   }
 
+  /** Persists and returns a {@link Registrar} with the specified registrarId. */
+  public static Registrar persistNewRegistrar(String registrarId) {
+    return persistNewRegistrar(registrarId, registrarId + " name", Registrar.Type.REAL, 100L);
+  }
+
+  /** Persists and returns a list of {@link Registrar}s with the specified registrarIds. */
+  public static ImmutableList<Registrar> persistNewRegistrars(String... registrarIds) {
+    ImmutableList.Builder<Registrar> newRegistrars = new ImmutableList.Builder<>();
+    for (String registrarId : registrarIds) {
+      newRegistrars.add(persistNewRegistrar(registrarId));
+    }
+    return newRegistrars.build();
+  }
+
   private static Iterable<BillingEvent> getBillingEvents() {
     return transactIfJpaTm(
         () ->

core/src/test/java/google/registry/tools/RemoveRegistryOneKeyCommandTest.java

@@ -0,0 +1,122 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistResource;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.truth.Correspondence;
+import com.googlecode.objectify.Key;
+import google.registry.model.ImmutableObject;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.EntityGroupRoot;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.poll.PollMessage;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Unit test for {@link RemoveRegistryOneKeyCommand}. */
+public class RemoveRegistryOneKeyCommandTest extends CommandTestCase<RemoveRegistryOneKeyCommand> {
+  DomainBase domain;
+  HistoryEntry historyEntry;
+
+  @BeforeEach
+  void beforeEach() {
+    createTld("foobar");
+    domain =
+        newDomainBase("foo.foobar")
+            .asBuilder()
+            .setDeletionTime(DateTime.parse("2016-01-01T00:00:00Z"))
+            .setAutorenewBillingEvent(createRegistryOneVKey(BillingEvent.Recurring.class, 100L))
+            .setAutorenewPollMessage(createRegistryOneVKey(PollMessage.Autorenew.class, 200L))
+            .setDeletePollMessage(createRegistryOneVKey(PollMessage.OneTime.class, 300L))
+            .build();
+  }
+
+  @Test
+  void removeRegistryOneKeyInDomainBase_succeeds() throws Exception {
+    DomainBase origin = persistResource(domain);
+
+    runCommand(
+        "--force",
+        "--key_paths_file",
+        writeToNamedTmpFile("keypath.txt", getKeyPathLiteral(domain)));
+
+    DomainBase persisted = ofy().load().key(domain.createVKey().getOfyKey()).now();
+    assertThat(ImmutableList.of(persisted))
+        .comparingElementsUsing(getDomainBaseCorrespondence())
+        .containsExactly(origin);
+    assertThat(persisted.getAutorenewBillingEvent()).isNull();
+    assertThat(persisted.getAutorenewPollMessage()).isNull();
+    assertThat(persisted.getDeletePollMessage()).isNull();
+  }
+
+  @Test
+  void removeRegistryOneKeyInDomainBase_notModifyRegistryTwoKey() throws Exception {
+    DomainBase origin =
+        persistResource(
+            domain
+                .asBuilder()
+                .setAutorenewBillingEvent(
+                    createRegistryTwoVKey(BillingEvent.Recurring.class, domain, 300L))
+                .build());
+
+    runCommand(
+        "--force",
+        "--key_paths_file",
+        writeToNamedTmpFile("keypath.txt", getKeyPathLiteral(domain)));
+
+    DomainBase persisted = ofy().load().key(domain.createVKey().getOfyKey()).now();
+    assertThat(ImmutableList.of(persisted))
+        .comparingElementsUsing(getDomainBaseCorrespondence())
+        .containsExactly(origin);
+    assertThat(persisted.getAutorenewBillingEvent())
+        .isEqualTo(createRegistryTwoVKey(BillingEvent.Recurring.class, domain, 300L));
+    assertThat(persisted.getAutorenewPollMessage()).isNull();
+    assertThat(persisted.getDeletePollMessage()).isNull();
+  }
+
+  private static String getKeyPathLiteral(Object entity) {
+    Key<?> key = Key.create(entity);
+    return String.format("\"DomainBase\", \"%s\"", key.getName());
+  }
+
+  private static <T> VKey<T> createRegistryOneVKey(Class<T> clazz, long id) {
+    Key<?> parent = Key.create(EntityGroupRoot.class, "per-tld");
+    return VKey.create(clazz, id, Key.create(parent, clazz, id));
+  }
+
+  private static <T> VKey<T> createRegistryTwoVKey(Class<T> clazz, DomainBase domain, long id) {
+    Key<?> parent = Key.create(domain.createVKey().getOfyKey(), HistoryEntry.class, 1000L);
+    return VKey.create(clazz, id, Key.create(parent, clazz, id));
+  }
+
+  private static Correspondence<ImmutableObject, ImmutableObject> getDomainBaseCorrespondence() {
+    return immutableObjectCorrespondence(
+        "revisions",
+        "updateTimestamp",
+        "autorenewBillingEvent",
+        "autorenewPollMessage",
+        "deletePollMessage");
+  }
+}

core/src/test/java/google/registry/tools/SetSqlReplayCheckpointCommandTest.java

@@ -0,0 +1,42 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static org.junit.Assert.assertThrows;
+
+import google.registry.schema.replay.SqlReplayCheckpoint;
+import org.joda.time.DateTime;
+import org.junit.jupiter.api.Test;
+
+class SetSqlReplayCheckpointCommandTest extends CommandTestCase<SetSqlReplayCheckpointCommand> {
+
+  @Test
+  void testSuccess() throws Exception {
+    assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(START_OF_TIME);
+    DateTime timeToSet = DateTime.parse("2000-06-06T22:00:00.0Z");
+    runCommandForced(timeToSet.toString());
+    assertThat(jpaTm().transact(SqlReplayCheckpoint::get)).isEqualTo(timeToSet);
+  }
+
+  @Test
+  void testFailure_multipleParams() throws Exception {
+    DateTime one = DateTime.parse("2000-06-06T22:00:00.0Z");
+    DateTime two = DateTime.parse("2001-06-06T22:00:00.0Z");
+    assertThrows(IllegalArgumentException.class, () -> runCommand(one.toString(), two.toString()));
+  }
+}

core/src/test/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommandTest.java

@@ -0,0 +1,271 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
+import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.deleteResource;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistActiveDomain;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.tools.CommandTestCase;
+import java.io.IOException;
+import org.joda.time.DateTime;
+import org.joda.time.LocalDate;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Tests for {@link BackfillSpec11ThreatMatchesCommand}. */
+public class BackfillSpec11ThreatMatchesCommandTest
+    extends CommandTestCase<BackfillSpec11ThreatMatchesCommand> {
+
+  private static final LocalDate CURRENT_DATE = DateTime.parse("2020-11-22").toLocalDate();
+  private final Spec11RegistrarThreatMatchesParser threatMatchesParser =
+      mock(Spec11RegistrarThreatMatchesParser.class);
+
+  private DomainBase domainA;
+
+  @BeforeEach
+  void beforeEach() throws Exception {
+    createTld("com");
+    domainA = persistActiveDomain("a.com");
+    persistActiveDomain("b.com");
+    persistActiveDomain("c.com");
+    fakeClock.setTo(CURRENT_DATE.toDateTimeAtStartOfDay());
+    command.threatMatchesParser = threatMatchesParser;
+    command.clock = fakeClock;
+    when(threatMatchesParser.getRegistrarThreatMatches(any(LocalDate.class)))
+        .thenReturn(ImmutableSet.of());
+  }
+
+  @Test
+  void testSuccess_singleFile() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    assertInStdout("Backfill Spec11 results from 692 files?");
+    assertInStdout("Successfully parsed through 692 files with 3 threats.");
+    verifyExactlyThreeEntriesInDbFromLastDay();
+  }
+
+  @Test
+  void testSuccess_sameDomain_multipleDays() throws Exception {
+    // If the same domains show up on multiple days, there should be multiple entries for them
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    when(threatMatchesParser.getRegistrarThreatMatches(LocalDate.parse("2019-01-01")))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    assertInStdout("Backfill Spec11 results from 692 files?");
+    assertInStdout("Successfully parsed through 692 files with 6 threats.");
+    jpaTm()
+        .transact(
+            () -> {
+              ImmutableList<Spec11ThreatMatch> threatMatches =
+                  jpaTm().loadAll(Spec11ThreatMatch.class);
+              assertThat(threatMatches).hasSize(6);
+              assertThat(
+                      threatMatches.stream()
+                          .map(Spec11ThreatMatch::getDomainName)
+                          .collect(toImmutableSet()))
+                  .containsExactly("a.com", "b.com", "c.com");
+              assertThat(
+                      threatMatches.stream()
+                          .map(Spec11ThreatMatch::getCheckDate)
+                          .collect(toImmutableSet()))
+                  .containsExactly(CURRENT_DATE, LocalDate.parse("2019-01-01"));
+            });
+  }
+
+  @Test
+  void testSuccess_empty() throws Exception {
+    runCommandForced();
+    assertInStdout("Backfill Spec11 results from 692 files?");
+    assertInStdout("Successfully parsed through 692 files with 0 threats.");
+  }
+
+  @Test
+  void testSuccess_sameDayTwice() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    runCommandForced();
+    verifyExactlyThreeEntriesInDbFromLastDay();
+  }
+
+  @Test
+  void testSuccess_threeDomainsForDomainName() throws Exception {
+    // We should use the repo ID from the proper DomainBase object at the scan's point in time.
+    // First, domain was created at START_OF_TIME and deleted one year ago
+    DateTime now = fakeClock.nowUtc();
+    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
+
+    // Next, domain was created six months ago and deleted two months ago
+    DomainBase secondSave =
+        persistResource(
+            newDomainBase("a.com")
+                .asBuilder()
+                .setCreationTimeForTest(now.minusMonths(6))
+                .setDeletionTime(now.minusMonths(2))
+                .build());
+
+    // Lastly, domain was created one month ago and is still valid
+    DomainBase thirdSave =
+        persistResource(
+            newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
+
+    // If the scan result was from three months ago, we should use the second save
+    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    String threatMatchRepoId =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAll(Spec11ThreatMatch.class).stream()
+                        .filter((match) -> match.getDomainName().equals("a.com"))
+                        .findFirst()
+                        .get()
+                        .getDomainRepoId());
+    assertThat(threatMatchRepoId).isNotEqualTo(domainA.getRepoId());
+    assertThat(threatMatchRepoId).isEqualTo(secondSave.getRepoId());
+    assertThat(threatMatchRepoId).isNotEqualTo(thirdSave.getRepoId());
+  }
+
+  @Test
+  void testSuccess_skipsExistingDatesWithoutOverwrite() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    Spec11ThreatMatch previous =
+        new Spec11ThreatMatch.Builder()
+            .setCheckDate(CURRENT_DATE)
+            .setDomainName("previous.tld")
+            .setDomainRepoId("1-DOMAIN")
+            .setRegistrarId("TheRegistrar")
+            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(previous));
+
+    runCommandForced();
+    ImmutableList<Spec11ThreatMatch> threatMatches =
+        jpaTm().transact(() -> jpaTm().loadAll(Spec11ThreatMatch.class));
+    assertAboutImmutableObjects()
+        .that(Iterables.getOnlyElement(threatMatches))
+        .isEqualExceptFields(previous, "id");
+  }
+
+  @Test
+  void testSuccess_overwritesExistingDatesWhenSpecified() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    Spec11ThreatMatch previous =
+        new Spec11ThreatMatch.Builder()
+            .setCheckDate(CURRENT_DATE)
+            .setDomainName("previous.tld")
+            .setDomainRepoId("1-DOMAIN")
+            .setRegistrarId("TheRegistrar")
+            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(previous));
+
+    runCommandForced("--overwrite_existing_dates");
+    verifyExactlyThreeEntriesInDbFromLastDay();
+  }
+
+  @Test
+  void testFailure_oneFileFails() throws Exception {
+    // If there are any exceptions, we should fail loud and fast
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE.minusDays(1)))
+        .thenThrow(new IOException("hi"));
+    RuntimeException runtimeException =
+        assertThrows(RuntimeException.class, this::runCommandForced);
+    assertThat(runtimeException.getCause().getClass()).isEqualTo(IOException.class);
+    assertThat(runtimeException).hasCauseThat().hasMessageThat().isEqualTo("hi");
+    jpaTm().transact(() -> assertThat(jpaTm().loadAll(Spec11ThreatMatch.class)).isEmpty());
+  }
+
+  @Test
+  void testFailure_noDomainForDomainName() throws Exception {
+    deleteResource(domainA);
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
+        .hasMessageThat()
+        .isEqualTo("Domain name a.com had no associated DomainBase objects.");
+  }
+
+  @Test
+  void testFailure_noDomainAtTimeOfScan() throws Exception {
+    // If the domain existed at some point(s) in time but not the time of the scan, fail.
+    // First, domain was created at START_OF_TIME and deleted one year ago
+    DateTime now = fakeClock.nowUtc();
+    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
+
+    // Second, domain was created one month ago and is still valid
+    persistResource(
+        newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
+
+    // If we have a result for this domain from 3 months ago when it didn't exist, fail.
+    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
+        .thenReturn(sampleThreatMatches());
+    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
+        .hasMessageThat()
+        .isEqualTo("Could not find a DomainBase valid for a.com on day 2020-08-22.");
+  }
+
+  private void verifyExactlyThreeEntriesInDbFromLastDay() {
+    jpaTm()
+        .transact(
+            () -> {
+              ImmutableList<Spec11ThreatMatch> threatMatches =
+                  jpaTm().loadAll(Spec11ThreatMatch.class);
+              assertThat(threatMatches)
+                  .comparingElementsUsing(immutableObjectCorrespondence("id", "domainRepoId"))
+                  .containsExactly(
+                      expectedThreatMatch("TheRegistrar", "a.com"),
+                      expectedThreatMatch("NewRegistrar", "b.com"),
+                      expectedThreatMatch("NewRegistrar", "c.com"));
+            });
+  }
+
+  private Spec11ThreatMatch expectedThreatMatch(String registrarId, String domainName) {
+    return new Spec11ThreatMatch.Builder()
+        .setDomainRepoId("ignored")
+        .setDomainName(domainName)
+        .setRegistrarId(registrarId)
+        .setCheckDate(CURRENT_DATE)
+        .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
+        .build();
+  }
+}

db/src/main/resources/sql/flyway.txt

@@ -78,3 +78,4 @@ V77__fixes_for_replay.sql
 V78__add_history_id_for_redemption_history_entry.sql
 V79__drop_foreign_keys_on_pollmessage.sql
 V80__defer_bill_event_key.sql
+V81__drop_spec11_fkeys.sql

db/src/main/resources/sql/flyway/V81__drop_spec11_fkeys.sql

@@ -0,0 +1,18 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Some objects haven't been fully populated in SQL yet, so don't depend on them.
+ALTER TABLE "Spec11ThreatMatch" DROP CONSTRAINT "fk_spec11_threat_match_domain_repo_id";
+ALTER TABLE "Spec11ThreatMatch" DROP CONSTRAINT "fk_spec11_threat_match_registrar_id";
+ALTER TABLE "Spec11ThreatMatch" DROP CONSTRAINT "fk_spec11_threat_match_tld";

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -2251,30 +2251,6 @@ ALTER TABLE ONLY public."RegistrarPoc"
     ADD CONSTRAINT fk_registrar_poc_registrar_id FOREIGN KEY (registrar_id) REFERENCES public."Registrar"(registrar_id);
 
 
---
--- Name: Spec11ThreatMatch fk_spec11_threat_match_domain_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public."Spec11ThreatMatch"
-    ADD CONSTRAINT fk_spec11_threat_match_domain_repo_id FOREIGN KEY (domain_repo_id) REFERENCES public."Domain"(repo_id);
-
-
---
--- Name: Spec11ThreatMatch fk_spec11_threat_match_registrar_id; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public."Spec11ThreatMatch"
-    ADD CONSTRAINT fk_spec11_threat_match_registrar_id FOREIGN KEY (registrar_id) REFERENCES public."Registrar"(registrar_id);
-
-
---
--- Name: Spec11ThreatMatch fk_spec11_threat_match_tld; Type: FK CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public."Spec11ThreatMatch"
-    ADD CONSTRAINT fk_spec11_threat_match_tld FOREIGN KEY (tld) REFERENCES public."Tld"(tld_name);
-
-
 --
 -- Name: DomainHistoryHost fka9woh3hu8gx5x0vly6bai327n; Type: FK CONSTRAINT; Schema: public; Owner: -
 --

java_common.gradle

@@ -74,10 +74,12 @@ test {
   useJUnitPlatform()
 }
 
-// Sets up integration test with a registry environment. The target environment is
-// passed by the 'test.gcp_integration.env' property. Test runner must have been
-// authorized to access the corresponding GCP project, e.g., by running 'gcloud auth'
-// or placing a credential file at a well known place.
+// Sets up integration test with a registry environment. The target environment
+// is passed by the 'test.gcp_integration.env' property. Test runner must have
+// been authorized to access the corresponding GCP project, e.g., by running
+// 'gcloud auth application-default login' or by downloading a credential file
+// and assign the path to it to the GOOGLE_APPLICATION_CREDENTIALS environment
+// variable.
 //
 // A typical use case is to run tests from desktop that accesses Cloud resources. See
 // core/src/test/java/google/registry/beam/initsql/BeamJpaModuleTest.java for an example.

processor/build.gradle

@@ -18,6 +18,12 @@ plugins {
 
 dependencies {
     def deps = rootProject.dependencyMap
+
+    // Custom-built objectify jar at commit ecd5165, included in Nomulus
+    // release.
+    compile files(
+        "${rootDir}/third_party/objectify/v4_1/objectify-4.1.3.jar")
+
     compile deps['com.google.code.findbugs:jsr305']
     compile deps['com.google.guava:guava']
     compile deps['com.squareup:javapoet']

processor/src/main/java/google/registry/processors/AbstractVKeyProcessor.java

@@ -27,11 +27,14 @@ import com.squareup.javapoet.TypeSpec;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import javax.annotation.processing.AbstractProcessor;
 import javax.annotation.processing.RoundEnvironment;
 import javax.lang.model.element.AnnotationMirror;
+import javax.lang.model.element.AnnotationValue;
 import javax.lang.model.element.Element;
+import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.element.Modifier;
 import javax.lang.model.element.TypeElement;
 import javax.lang.model.type.DeclaredType;
@@ -47,9 +50,12 @@ import javax.persistence.Converter;
 public abstract class AbstractVKeyProcessor extends AbstractProcessor {
 
   private static final String CONVERTER_CLASS_NAME_TEMP = "VKeyConverter_%s";
-  // The method with same name should be defined in StringVKey and LongVKey
+  // The method with same name should be defined in WithStringVKey and WithLongVKey
   private static final String CLASS_NAME_SUFFIX_KEY = "classNameSuffix";
 
+  // Method in WithStringVKey and WithLongVKey to indicate that this is a composite key.
+  private static final String COMPOSITE_KEY_KEY = "compositeKey";
+
   abstract Class<?> getSqlColumnType();
 
   abstract String getAnnotationSimpleName();
@@ -76,18 +82,18 @@ public abstract class AbstractVKeyProcessor extends AbstractProcessor {
                         actualAnnotation.size() == 1,
                         String.format(
                             "type can have only 1 %s annotation", getAnnotationSimpleName()));
-                    String converterClassNameSuffix =
-                        actualAnnotation.get(0).getElementValues().entrySet().stream()
-                            .filter(
-                                entry ->
-                                    entry
-                                        .getKey()
-                                        .getSimpleName()
-                                        .toString()
-                                        .equals(CLASS_NAME_SUFFIX_KEY))
-                            .map(entry -> ((String) entry.getValue().getValue()).trim())
-                            .findFirst()
-                            .orElse("");
+                    String converterClassNameSuffix = "";
+                    boolean hasCompositeOfyKey = false;
+                    for (Map.Entry<? extends ExecutableElement, ? extends AnnotationValue> entry :
+                        actualAnnotation.get(0).getElementValues().entrySet()) {
+                      String keyName = entry.getKey().getSimpleName().toString();
+                      Object value = entry.getValue().getValue();
+                      if (keyName.equals(CLASS_NAME_SUFFIX_KEY)) {
+                        converterClassNameSuffix = ((String) value).trim();
+                      } else if (keyName.equals(COMPOSITE_KEY_KEY)) {
+                        hasCompositeOfyKey = (Boolean) value;
+                      }
+                    }
                     if (converterClassNameSuffix.isEmpty()) {
                       converterClassNameSuffix =
                           getTypeUtils().asElement(entityType).getSimpleName().toString();
@@ -97,7 +103,8 @@ public abstract class AbstractVKeyProcessor extends AbstractProcessor {
                       createJavaFile(
                               getPackageName(annotatedTypeElement),
                               String.format(CONVERTER_CLASS_NAME_TEMP, converterClassNameSuffix),
-                              entityType)
+                              entityType,
+                              hasCompositeOfyKey)
                           .writeTo(processingEnv.getFiler());
                     } catch (IOException e) {
                       throw new UncheckedIOException(e);
@@ -108,7 +115,10 @@ public abstract class AbstractVKeyProcessor extends AbstractProcessor {
   }
 
   private JavaFile createJavaFile(
-      String packageName, String converterClassName, TypeMirror entityTypeMirror) {
+      String packageName,
+      String converterClassName,
+      TypeMirror entityTypeMirror,
+      boolean hasCompositeOfyKey) {
     TypeName entityType = ClassName.get(entityTypeMirror);
 
     ParameterizedTypeName attributeConverter =
@@ -127,7 +137,7 @@ public abstract class AbstractVKeyProcessor extends AbstractProcessor {
             .addStatement("return $T.class", entityType)
             .build();
 
-    TypeSpec vKeyConverter =
+    TypeSpec.Builder classBuilder =
         TypeSpec.classBuilder(converterClassName)
             .addAnnotation(
                 AnnotationSpec.builder(ClassName.get(Converter.class))
@@ -135,9 +145,24 @@ public abstract class AbstractVKeyProcessor extends AbstractProcessor {
                     .build())
             .addModifiers(Modifier.FINAL)
             .superclass(attributeConverter)
-            .addMethod(getAttributeClass)
-            .build();
+            .addMethod(getAttributeClass);
 
+    // If this is a converter for a composite vkey type, generate an override for the default
+    // {@link google.registry.persistence.VKeyConverter.hasCompositeOfyKey()} method, which returns
+    // false.
+    if (hasCompositeOfyKey) {
+      MethodSpec hasCompositeOfyKeyMethod =
+          MethodSpec.methodBuilder("hasCompositeOfyKey")
+              .addAnnotation(Override.class)
+              .addModifiers(Modifier.PROTECTED)
+              .returns(boolean.class)
+              .addStatement("return true", entityType)
+              .build();
+
+      classBuilder.addMethod(hasCompositeOfyKeyMethod);
+    }
+
+    TypeSpec vKeyConverter = classBuilder.build();
     return JavaFile.builder(packageName, vKeyConverter).build();
   }
 

proxy/src/main/java/google/registry/proxy/handler/EppServiceHandler.java

@@ -34,7 +34,9 @@ import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.ssl.SslHandshakeCompletionEvent;
 import io.netty.util.AttributeKey;
 import io.netty.util.concurrent.Promise;
+import java.security.cert.CertificateEncodingException;
 import java.security.cert.X509Certificate;
+import java.util.Base64;
 import java.util.function.Supplier;
 
 /** Handler that processes EPP protocol logic. */
@@ -52,18 +54,26 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
   /** Name of the HTTP header that stores the client certificate hash. */
   public static final String SSL_CLIENT_CERTIFICATE_HASH_FIELD = "X-SSL-Certificate";
 
+  /** Name of the HTTP header that stores the full client certificate. */
+  public static final String SSL_CLIENT_FULL_CERTIFICATE_FIELD = "X-SSL-Full-Certificate";
+
   /** Name of the HTTP header that stores the client IP address. */
   public static final String FORWARDED_FOR_FIELD = "X-Forwarded-For";
 
   /** Name of the HTTP header that indicates if the EPP session should be closed. */
   public static final String EPP_SESSION_FIELD = "Epp-Session";
 
+  /** Name of the HTTP header that indicates a successful login has occurred. */
+  public static final String EPP_LOGGED_IN_FIELD = "Logged-In";
+
   public static final String EPP_CONTENT_TYPE = "application/epp+xml";
 
   private final byte[] helloBytes;
 
   private String sslClientCertificateHash;
+  private X509Certificate sslClientCertificate;
   private String clientAddress;
+  private boolean isLoggedIn = false;
 
   public EppServiceHandler(
       String relayHost,
@@ -103,7 +113,8 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
             .addListener(
                 (Promise<X509Certificate> promise) -> {
                   if (promise.isSuccess()) {
-                    sslClientCertificateHash = getCertificateHash(promise.get());
+                    sslClientCertificate = promise.get();
+                    sslClientCertificateHash = getCertificateHash(sslClientCertificate);
                     // Set the client cert hash key attribute for both this channel,
                     // used for collecting metrics on specific clients.
                     ctx.channel().attr(CLIENT_CERTIFICATE_HASH_KEY).set(sslClientCertificateHash);
@@ -132,6 +143,17 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
         .set(FORWARDED_FOR_FIELD, clientAddress)
         .set(HttpHeaderNames.CONTENT_TYPE, EPP_CONTENT_TYPE)
         .set(HttpHeaderNames.ACCEPT, EPP_CONTENT_TYPE);
+    if (!isLoggedIn) {
+      try {
+        request
+            .headers()
+            .set(
+                SSL_CLIENT_FULL_CERTIFICATE_FIELD,
+                Base64.getEncoder().encodeToString(sslClientCertificate.getEncoded()));
+      } catch (CertificateEncodingException e) {
+        throw new RuntimeException("Cannot encode client certificate", e);
+      }
+    }
     return request;
   }
 
@@ -141,9 +163,13 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
     checkArgument(msg instanceof HttpResponse);
     HttpResponse response = (HttpResponse) msg;
     String sessionAliveValue = response.headers().get(EPP_SESSION_FIELD);
+    String loginValue = response.headers().get(EPP_LOGGED_IN_FIELD);
     if (sessionAliveValue != null && sessionAliveValue.equals("close")) {
       promise.addListener(ChannelFutureListener.CLOSE);
     }
+    if (loginValue != null && loginValue.equals("true")) {
+      isLoggedIn = true;
+    }
     super.write(ctx, msg, promise);
   }
 }

proxy/src/test/java/google/registry/proxy/EppProtocolModuleTest.java

@@ -16,6 +16,7 @@ package google.registry.proxy;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.networking.handler.SslServerInitializer.CLIENT_CERTIFICATE_PROMISE_KEY;
+import static google.registry.proxy.TestUtils.SAMPLE_CERT;
 import static google.registry.proxy.handler.ProxyProtocolHandler.REMOTE_ADDRESS_KEY;
 import static google.registry.util.ResourceUtils.readResourceBytes;
 import static google.registry.util.X509Utils.getCertificateHash;
@@ -25,7 +26,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.base.Throwables;
 import google.registry.proxy.handler.HttpsRelayServiceHandler.NonOkHttpResponseException;
 import google.registry.testing.FakeClock;
-import google.registry.util.SelfSignedCaCertificate;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.embedded.EmbeddedChannel;
@@ -36,6 +36,8 @@ import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.cookie.Cookie;
 import io.netty.handler.codec.http.cookie.DefaultCookie;
 import io.netty.util.concurrent.Promise;
+import java.io.ByteArrayInputStream;
+import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -96,8 +98,8 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
     return buffer;
   }
 
-  private FullHttpRequest makeEppHttpRequest(byte[] content, Cookie... cookies) {
-    return TestUtils.makeEppHttpRequest(
+  private FullHttpRequest makeEppHttpRequestWithCertificate(byte[] content, Cookie... cookies) {
+    return TestUtils.makeEppHttpRequestWithCertificate(
         new String(content, UTF_8),
         PROXY_CONFIG.epp.relayHost,
         PROXY_CONFIG.epp.relayPath,
@@ -120,7 +122,10 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
   @Override
   void beforeEach() throws Exception {
     testComponent = makeTestComponent(new FakeClock());
-    certificate = SelfSignedCaCertificate.create().cert();
+    CertificateFactory cf = CertificateFactory.getInstance("X.509");
+    certificate =
+        (X509Certificate)
+            cf.generateCertificate(new ByteArrayInputStream(SAMPLE_CERT.getBytes(UTF_8)));
     initializeChannel(
         ch -> {
           ch.attr(REMOTE_ADDRESS_KEY).set(CLIENT_ADDRESS);
@@ -134,13 +139,15 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
   @Test
   void testSuccess_singleFrameInboundMessage() throws Exception {
     // First inbound message is hello.
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(HELLO_BYTES));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(HELLO_BYTES));
 
     byte[] inputBytes = readResourceBytes(getClass(), "login.xml").read();
 
     // Verify inbound message is as expected.
     assertThat(channel.writeInbound(getByteBufFromContent(inputBytes))).isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes));
 
     // Nothing more to read.
     assertThat((Object) channel.readInbound()).isNull();
@@ -161,8 +168,10 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
                 Unpooled.wrappedBuffer(
                     getByteBufFromContent(inputBytes1), getByteBufFromContent(inputBytes2))))
         .isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes1));
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes2));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes1));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes2));
 
     // Nothing more to read.
     assertThat((Object) channel.readInbound()).isNull();
@@ -186,11 +195,13 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
 
     // The second frame contains the first message, and part of the second message.
     assertThat(channel.writeInbound(inputBuffer.readBytes(inputBytes2.length))).isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes1));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes1));
 
     // The third frame contains the rest of the second message.
     assertThat(channel.writeInbound(inputBuffer)).isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes2));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes2));
 
     // Nothing more to read.
     assertThat((Object) channel.readInbound()).isNull();
@@ -252,7 +263,7 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
     byte[] inputBytes1 = readResourceBytes(getClass(), "logout.xml").read();
     assertThat(channel.writeInbound(getByteBufFromContent(inputBytes1))).isTrue();
     assertThat((FullHttpRequest) channel.readInbound())
-        .isEqualTo(makeEppHttpRequest(inputBytes1, cookie1, cookie2));
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes1, cookie1, cookie2));
 
     // Second outbound message change cookies.
     byte[] outputBytes2 = readResourceBytes(getClass(), "logout_response.xml").read();
@@ -267,7 +278,7 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
     byte[] inputBytes2 = readResourceBytes(getClass(), "login.xml").read();
     assertThat(channel.writeInbound(getByteBufFromContent(inputBytes2))).isTrue();
     assertThat((FullHttpRequest) channel.readInbound())
-        .isEqualTo(makeEppHttpRequest(inputBytes2, cookie1, cookie2, cookie3));
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes2, cookie1, cookie2, cookie3));
 
     // Nothing more to write or read.
     assertThat((Object) channel.readOutbound()).isNull();

proxy/src/test/java/google/registry/proxy/TestUtils.java

@@ -36,6 +36,54 @@ import io.netty.handler.codec.http.cookie.ServerCookieEncoder;
 /** Utility class for various helper methods used in testing. */
 public class TestUtils {
 
+  public static final String SAMPLE_CERT =
+      "-----BEGIN CERTIFICATE-----\n"
+          + "MIIDvTCCAqWgAwIBAgIJAK/PgPT0jTwRMA0GCSqGSIb3DQEBCwUAMHUxCzAJBgNV\n"
+          + "BAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxDzAN\n"
+          + "BgNVBAoMBkdvb2dsZTEdMBsGA1UECwwUZG9tYWluLXJlZ2lzdHJ5LXRlc3QxEDAO\n"
+          + "BgNVBAMMB2NsaWVudDEwHhcNMTUwODI2MTkxODA4WhcNNDMwMTExMTkxODA4WjB1\n"
+          + "MQswCQYDVQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxETAPBgNVBAcMCE5ldyBZ\n"
+          + "b3JrMQ8wDQYDVQQKDAZHb29nbGUxHTAbBgNVBAsMFGRvbWFpbi1yZWdpc3RyeS10\n"
+          + "ZXN0MRAwDgYDVQQDDAdjbGllbnQxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB\n"
+          + "CgKCAQEAvoE/IoFJyzb0dU4NFhL8FYgy+B/GnUd5aA66CMx5xKRMbEAtIgxU8TTO\n"
+          + "W+9jdTsE00Grk3Ct4KdY73CYW+6IFXL4O0K/m5S+uajh+I2UMVZJV38RAIqNxue0\n"
+          + "Egv9M4haSsCVIPcX9b+6McywfYSF1bzPb2Gb2FAQO7Jb0BjlPhPMIROCrbG40qPg\n"
+          + "LWrl33dz+O52kO+DyZEzHqI55xH6au77sMITsJe+X23lzQcMFUUm8moiOw0EKrj/\n"
+          + "GaMTZLHP46BCRoJDAPTNx55seIwgAHbKA2VVtqrvmA2XYJQA6ipdhfKRoJFy8Z8H\n"
+          + "DYsorGtazQL2HhF/5uJD25z1m5eQHQIDAQABo1AwTjAdBgNVHQ4EFgQUParEmiSR\n"
+          + "U/Oqy8hr7k+MBKhZwVkwHwYDVR0jBBgwFoAUParEmiSRU/Oqy8hr7k+MBKhZwVkw\n"
+          + "DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAojsUhF6PtZrStnHBFWNR\n"
+          + "ryzvANB8krZlYeX9Hkqn8zIVfAkpbVmL8aZQ7yj17jSpw47PQh3x5gwA9yc/SS0G\n"
+          + "E1rGuxYH02UGbua8G0+vviSQfLtskPQzK7EIR63WNhHEo/Q9umLJkZ0LguWEBf3L\n"
+          + "q8CoXv2i/RNvqVPcTNp/zCKXJZAa8wAjNRJs834AZj4k5xwyYZ3F8D5PGz+YMOmV\n"
+          + "M9Qd+NdXSC/Qn7HQzFhE8p5elBV35P8oX5dXEfn0S7zOXDenp5JvvLoggOWOcKsq\n"
+          + "KiWDQrsT+TMKmHL94/h4t7FghtQLMzY5SGYJsYTv/LG8tewrz6KRb/Wj3JNojyEw\n"
+          + "Ug==\n"
+          + "-----END CERTIFICATE-----\n";
+
+  public static final String SAMPLE_CERT_ENCODED =
+      "MIIDvTCCAqWgAwIBAgIJAK/PgPT0jTwRMA0GCSqGSIb3DQEBCwUAMHUxCzAJBgNV"
+          + "BAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxDzAN"
+          + "BgNVBAoMBkdvb2dsZTEdMBsGA1UECwwUZG9tYWluLXJlZ2lzdHJ5LXRlc3QxEDAO"
+          + "BgNVBAMMB2NsaWVudDEwHhcNMTUwODI2MTkxODA4WhcNNDMwMTExMTkxODA4WjB1"
+          + "MQswCQYDVQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxETAPBgNVBAcMCE5ldyBZ"
+          + "b3JrMQ8wDQYDVQQKDAZHb29nbGUxHTAbBgNVBAsMFGRvbWFpbi1yZWdpc3RyeS10"
+          + "ZXN0MRAwDgYDVQQDDAdjbGllbnQxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB"
+          + "CgKCAQEAvoE/IoFJyzb0dU4NFhL8FYgy+B/GnUd5aA66CMx5xKRMbEAtIgxU8TTO"
+          + "W+9jdTsE00Grk3Ct4KdY73CYW+6IFXL4O0K/m5S+uajh+I2UMVZJV38RAIqNxue0"
+          + "Egv9M4haSsCVIPcX9b+6McywfYSF1bzPb2Gb2FAQO7Jb0BjlPhPMIROCrbG40qPg"
+          + "LWrl33dz+O52kO+DyZEzHqI55xH6au77sMITsJe+X23lzQcMFUUm8moiOw0EKrj/"
+          + "GaMTZLHP46BCRoJDAPTNx55seIwgAHbKA2VVtqrvmA2XYJQA6ipdhfKRoJFy8Z8H"
+          + "DYsorGtazQL2HhF/5uJD25z1m5eQHQIDAQABo1AwTjAdBgNVHQ4EFgQUParEmiSR"
+          + "U/Oqy8hr7k+MBKhZwVkwHwYDVR0jBBgwFoAUParEmiSRU/Oqy8hr7k+MBKhZwVkw"
+          + "DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAojsUhF6PtZrStnHBFWNR"
+          + "ryzvANB8krZlYeX9Hkqn8zIVfAkpbVmL8aZQ7yj17jSpw47PQh3x5gwA9yc/SS0G"
+          + "E1rGuxYH02UGbua8G0+vviSQfLtskPQzK7EIR63WNhHEo/Q9umLJkZ0LguWEBf3L"
+          + "q8CoXv2i/RNvqVPcTNp/zCKXJZAa8wAjNRJs834AZj4k5xwyYZ3F8D5PGz+YMOmV"
+          + "M9Qd+NdXSC/Qn7HQzFhE8p5elBV35P8oX5dXEfn0S7zOXDenp5JvvLoggOWOcKsq"
+          + "KiWDQrsT+TMKmHL94/h4t7FghtQLMzY5SGYJsYTv/LG8tewrz6KRb/Wj3JNojyEw"
+          + "Ug==";
+
   public static FullHttpRequest makeHttpPostRequest(String content, String host, String path) {
     ByteBuf buf = Unpooled.wrappedBuffer(content.getBytes(US_ASCII));
     FullHttpRequest request =
@@ -101,6 +149,21 @@ public class TestUtils {
     return request;
   }
 
+  public static FullHttpRequest makeEppHttpRequestWithCertificate(
+      String content,
+      String host,
+      String path,
+      String accessToken,
+      String sslClientCertificateHash,
+      String clientAddress,
+      Cookie... cookies) {
+    FullHttpRequest request =
+        makeEppHttpRequest(
+            content, host, path, accessToken, sslClientCertificateHash, clientAddress, cookies);
+    request.headers().set("X-SSL-Full-Certificate", SAMPLE_CERT_ENCODED);
+    return request;
+  }
+
   public static FullHttpResponse makeWhoisHttpResponse(String content, HttpResponseStatus status) {
     FullHttpResponse response = makeHttpResponse(content, status);
     response.headers().set("content-type", "text/plain");

proxy/src/test/java/google/registry/proxy/handler/EppServiceHandlerTest.java

@@ -16,6 +16,7 @@ package google.registry.proxy.handler;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.networking.handler.SslServerInitializer.CLIENT_CERTIFICATE_PROMISE_KEY;
+import static google.registry.proxy.TestUtils.SAMPLE_CERT;
 import static google.registry.proxy.TestUtils.assertHttpRequestEquivalent;
 import static google.registry.proxy.TestUtils.makeEppHttpResponse;
 import static google.registry.proxy.handler.ProxyProtocolHandler.REMOTE_ADDRESS_KEY;
@@ -43,6 +44,8 @@ import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.cookie.Cookie;
 import io.netty.handler.codec.http.cookie.DefaultCookie;
 import io.netty.util.concurrent.Promise;
+import java.io.ByteArrayInputStream;
+import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -109,9 +112,23 @@ class EppServiceHandlerTest {
         cookies);
   }
 
+  private FullHttpRequest makeEppHttpRequestWithCertificate(String content, Cookie... cookies) {
+    return TestUtils.makeEppHttpRequestWithCertificate(
+        content,
+        RELAY_HOST,
+        RELAY_PATH,
+        ACCESS_TOKEN,
+        getCertificateHash(clientCertificate),
+        CLIENT_ADDRESS,
+        cookies);
+  }
+
   @BeforeEach
   void beforeEach() throws Exception {
-    clientCertificate = SelfSignedCaCertificate.create().cert();
+    CertificateFactory cf = CertificateFactory.getInstance("X.509");
+    clientCertificate =
+        (X509Certificate)
+            cf.generateCertificate(new ByteArrayInputStream(SAMPLE_CERT.getBytes(UTF_8)));
     channel = setUpNewChannel(eppServiceHandler);
   }
 
@@ -194,7 +211,7 @@ class EppServiceHandlerTest {
     setHandshakeSuccess();
     // hello bytes should be passed to the next handler.
     FullHttpRequest helloRequest = channel.readInbound();
-    assertThat(helloRequest).isEqualTo(makeEppHttpRequest(HELLO));
+    assertThat(helloRequest).isEqualTo(makeEppHttpRequestWithCertificate(HELLO));
     // Nothing further to pass to the next handler.
     assertThat((Object) channel.readInbound()).isNull();
     assertThat(channel.isActive()).isTrue();
@@ -216,12 +233,34 @@ class EppServiceHandlerTest {
     String content = "<epp>stuff</epp>";
     channel.writeInbound(Unpooled.wrappedBuffer(content.getBytes(UTF_8)));
     FullHttpRequest request = channel.readInbound();
-    assertThat(request).isEqualTo(makeEppHttpRequest(content));
+    assertThat(request).isEqualTo(makeEppHttpRequestWithCertificate(content));
     // Nothing further to pass to the next handler.
     assertThat((Object) channel.readInbound()).isNull();
     assertThat(channel.isActive()).isTrue();
   }
 
+  @Test
+  void testSuccess_sendCertificateOnlyBeforeLogin() throws Exception {
+    setHandshakeSuccess();
+    // First inbound message is hello.
+    channel.readInbound();
+    String content = "<epp>stuff</epp>";
+    channel.writeInbound(Unpooled.wrappedBuffer(content.getBytes(UTF_8)));
+    FullHttpRequest request = channel.readInbound();
+    assertThat(request).isEqualTo(makeEppHttpRequestWithCertificate(content));
+    // Receive response indicating session is logged in
+    HttpResponse response = makeEppHttpResponse(content, HttpResponseStatus.OK);
+    response.headers().set("Logged-In", "true");
+    // Send another inbound message after login
+    channel.writeOutbound(response);
+    channel.writeInbound(Unpooled.wrappedBuffer(content.getBytes(UTF_8)));
+    request = channel.readInbound();
+    // Second request should not have full certificate
+    assertThat(request).isEqualTo(makeEppHttpRequest(content));
+    assertThat((Object) channel.readInbound()).isNull();
+    assertThat(channel.isActive()).isTrue();
+  }
+
   @Test
   void testSuccess_sendResponseToNextHandler() throws Exception {
     setHandshakeSuccess();
@@ -294,7 +333,8 @@ class EppServiceHandlerTest {
     String requestContent = "<epp>request</epp>";
     channel.writeInbound(Unpooled.wrappedBuffer(requestContent.getBytes(UTF_8)));
     FullHttpRequest request = channel.readInbound();
-    assertHttpRequestEquivalent(request, makeEppHttpRequest(requestContent, cookie1, cookie2));
+    assertHttpRequestEquivalent(
+        request, makeEppHttpRequestWithCertificate(requestContent, cookie1, cookie2));
     // Nothing further to pass to the next handler.
     assertThat((Object) channel.readInbound()).isNull();
     assertThat((Object) channel.readOutbound()).isNull();
@@ -317,13 +357,16 @@ class EppServiceHandlerTest {
     // First request written.
     channel.writeInbound(Unpooled.wrappedBuffer(requestContent1.getBytes(UTF_8)));
     FullHttpRequest request1 = channel.readInbound();
-    assertHttpRequestEquivalent(request1, makeEppHttpRequest(requestContent1, cookie1, cookie2));
+    assertHttpRequestEquivalent(
+        request1, makeEppHttpRequestWithCertificate(requestContent1, cookie1, cookie2));
     String responseContent2 = "<epp>response2</epp>";
     Cookie cookie3 = new DefaultCookie("name3", "value3");
     Cookie newCookie2 = new DefaultCookie("name2", "newValue");
     // Second response written.
-    channel.writeOutbound(
-        makeEppHttpResponse(responseContent2, HttpResponseStatus.OK, cookie3, newCookie2));
+    HttpResponse response =
+        makeEppHttpResponse(responseContent2, HttpResponseStatus.OK, cookie3, newCookie2);
+    response.headers().set("Logged-In", "true");
+    channel.writeOutbound(response);
     channel.readOutbound();
     String requestContent2 = "<epp>request2</epp>";
     // Second request written.