Add BillingVKey to restore symmetric VKey in GracePeriodBase (#902 )

* Use PollMessageVKey to replace VKey<PollMessage> in DomainBase * Revert changes to DomainContent * Use BillingVKey in GracePeriodBase to restore symmetric vkey * Rebase on HEAD
Validate SQL credentials in Secret Manager (#907 )
2026-07-10 18:13:02 +00:00 · 2020-12-17 14:13:47 -05:00 · 2020-12-16 10:57:03 -05:00 · 2020-12-16 10:03:32 -05:00 · 2020-12-15 17:34:38 -05:00 · 2020-12-15 16:36:39 -05:00
60 changed files with 5784 additions and 4693 deletions
@@ -65,7 +65,7 @@ class PresubmitCheck:
    for pattern in self.skipped_patterns:
      if pattern in file:
        return False
-    with open(file, "r") as f:
+    with open(file, "r", encoding='utf8') as f:
      file_content = f.read()
      matches = re.match(self.regex, file_content, re.DOTALL)
      if self.regex_type == FORBIDDEN:
@@ -241,7 +241,7 @@ def verify_flyway_index():

  # Remove the sequence numbers and compare against the index file contents.
  files = [filename[1] for filename in sorted(files)]
-  with open('db/src/main/resources/sql/flyway.txt') as index:
+  with open('db/src/main/resources/sql/flyway.txt', encoding='utf8') as index:
    indexed_files = index.read().splitlines()
  if files != indexed_files:
    unindexed = set(files) - set(indexed_files)
@@ -31,6 +31,7 @@ import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.JdbcJpaTm;
 import google.registry.persistence.PersistenceModule.SocketFactoryJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -168,6 +169,7 @@ public class BeamJpaModule {
        BeamJpaModule.class,
        KmsModule.class,
        PersistenceModule.class,
+        SecretManagerModule.class,
        UtilsModule.class
      })
  public interface JpaTransactionManagerComponent {
@@ -110,7 +110,6 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.persistence.VKey;
 import google.registry.tmch.LordnTaskUtils;
 import java.util.Optional;
@@ -372,7 +371,7 @@ public class DomainCreateFlow implements TransactionalFlow {
        && TokenType.SINGLE_USE.equals(allocationToken.get().getTokenType())) {
      entitiesToSave.add(
          allocationTokenFlowUtils.redeemToken(
-              allocationToken.get(), DomainHistoryVKey.create(Key.create(historyEntry))));
+              allocationToken.get(), HistoryEntry.createVKey(Key.create(historyEntry))));
    }
    enqueueTasks(newDomain, hasSignedMarks, hasClaimsNotice);

@@ -32,7 +32,8 @@ import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.registry.Registry;
-import google.registry.persistence.DomainHistoryVKey;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -107,7 +108,7 @@ public class AllocationTokenFlowUtils {

  /** Redeems a SINGLE_USE {@link AllocationToken}, returning the redeemed copy. */
  public AllocationToken redeemToken(
-      AllocationToken token, DomainHistoryVKey redemptionHistoryEntry) {
+      AllocationToken token, VKey<? extends HistoryEntry> redemptionHistoryEntry) {
    checkArgument(
        TokenType.SINGLE_USE.equals(token.getTokenType()),
        "Only SINGLE_USE tokens can be marked as redeemed");
@@ -0,0 +1,38 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model;
+
+import com.google.common.flogger.FluentLogger;
+import google.registry.config.RegistryEnvironment;
+
+/** Utility methods related to migrating dual-read/dual-write entities. */
+public class DatabaseMigrationUtils {
+
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  /** Throws exceptions only in unit tests, otherwise only logs exceptions. */
+  public static void suppressExceptionUnlessInTest(Runnable work, String message) {
+    try {
+      work.run();
+    } catch (Exception e) {
+      if (RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)) {
+        throw e;
+      }
+      logger.atWarning().withCause(e).log(message);
+    }
+  }
+
+  private DatabaseMigrationUtils() {}
+}
@@ -28,6 +28,11 @@ import org.joda.time.DateTime;
 */
 public class UpdateAutoTimestamp extends ImmutableObject {

+  // When set to true, database converters/translators should do tha auto update.  When set to
+  // false, auto update should be suspended (this exists to allow us to preserve the original value
+  // during a replay).
+  static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
+
  DateTime timestamp;

  /** Returns the timestamp, or {@code START_OF_TIME} if it's null. */
@@ -40,4 +45,30 @@ public class UpdateAutoTimestamp extends ImmutableObject {
    instance.timestamp = timestamp;
    return instance;
  }
+
+  // TODO(b/175610935): Remove the auto-update disabling code below after migration.
+
+  /** Class to allow us to safely disable auto-update in a try-with-resources block. */
+  public static class DisableAutoUpdateResource implements AutoCloseable {
+    DisableAutoUpdateResource() {
+      autoUpdateEnabled.set(false);
+    }
+
+    @Override
+    public void close() {
+      autoUpdateEnabled.set(true);
+    }
+  }
+
+  /**
+   * Resturns a resource that disables auto-updates on all {@link UpdateAutoTimestamp}s in the
+   * current thread, suitable for use with in a try-with-resources block.
+   */
+  public static DisableAutoUpdateResource disableAutoUpdate() {
+    return new DisableAutoUpdateResource();
+  }
+
+  public static boolean autoUpdateEnabled() {
+    return autoUpdateEnabled.get();
+  }
 }
@@ -23,6 +23,8 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.ofy.ObjectifyService;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import google.registry.schema.replay.DatastoreAndSqlEntity;
 import javax.annotation.Nullable;
@@ -82,10 +84,8 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
    instance.domainRepoId = checkArgumentNotNull(domainRepoId);
    instance.expirationTime = checkArgumentNotNull(expirationTime);
    instance.clientId = checkArgumentNotNull(clientId);
-    instance.billingEventOneTime = billingEventOneTime;
-    instance.billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
-    instance.billingEventRecurring = billingEventRecurring;
-    instance.billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+    instance.billingEventOneTime = BillingEventVKey.create(billingEventOneTime);
+    instance.billingEventRecurring = BillingRecurrenceVKey.create(billingEventRecurring);
    return instance;
  }

@@ -178,7 +178,6 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
  public GracePeriod cloneAfterOfyLoad(String domainRepoId) {
    GracePeriod clone = clone(this);
    clone.domainRepoId = checkArgumentNotNull(domainRepoId);
-    clone.restoreHistoryIds();
    return clone;
  }

@@ -190,7 +189,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
   */
  public GracePeriod cloneWithRecurringBillingEvent(VKey<BillingEvent.Recurring> recurring) {
    GracePeriod clone = clone(this);
-    clone.billingEventRecurring = recurring;
+    clone.billingEventRecurring = BillingRecurrenceVKey.create(recurring);
    return clone;
  }

@@ -232,9 +231,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
      instance.expirationTime = gracePeriod.expirationTime;
      instance.clientId = gracePeriod.clientId;
      instance.billingEventOneTime = gracePeriod.billingEventOneTime;
-      instance.billingEventOneTimeHistoryId = gracePeriod.billingEventOneTimeHistoryId;
      instance.billingEventRecurring = gracePeriod.billingEventRecurring;
-      instance.billingEventRecurringHistoryId = gracePeriod.billingEventRecurringHistoryId;
      return instance;
    }
  }
@@ -14,14 +14,14 @@

 package google.registry.model.domain;

-import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Embed;
 import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.ImmutableObject;
 import google.registry.model.ModelUtils;
 import google.registry.model.billing.BillingEvent;
-import google.registry.model.billing.BillingEvent.OneTime;
 import google.registry.model.domain.rgp.GracePeriodStatus;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
 import java.lang.reflect.Field;
 import java.util.LinkedHashMap;
@@ -68,24 +68,16 @@ public class GracePeriodBase extends ImmutableObject {
   * billingEventRecurring}) or for redemption grace periods (since deletes have no cost).
   */
  // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  @Column(name = "billing_event_id")
-  VKey<OneTime> billingEventOneTime = null;
-
-  @Ignore
-  @Column(name = "billing_event_history_id")
-  Long billingEventOneTimeHistoryId;
+  @Access(AccessType.FIELD)
+  BillingEventVKey billingEventOneTime = null;

  /**
   * The recurring billing event corresponding to the action that triggered this grace period, if
   * applicable - i.e. if the action was an autorenew - or null in all other cases.
   */
  // NB: Would @IgnoreSave(IfNull.class), but not allowed for @Embed collections.
-  @Column(name = "billing_recurrence_id")
-  VKey<BillingEvent.Recurring> billingEventRecurring = null;
-
-  @Ignore
-  @Column(name = "billing_recurrence_history_id")
-  Long billingEventRecurringHistoryId;
+  @Access(AccessType.FIELD)
+  BillingRecurrenceVKey billingEventRecurring = null;

  public long getGracePeriodId() {
    return gracePeriodId;
@@ -123,8 +115,7 @@ public class GracePeriodBase extends ImmutableObject {
   * period is not AUTO_RENEW.
   */
  public VKey<BillingEvent.OneTime> getOneTimeBillingEvent() {
-    restoreOfyKeys();
-    return billingEventOneTime;
+    return billingEventOneTime == null ? null : billingEventOneTime.createVKey();
  }

  /**
@@ -132,18 +123,7 @@ public class GracePeriodBase extends ImmutableObject {
   * period is AUTO_RENEW.
   */
  public VKey<BillingEvent.Recurring> getRecurringBillingEvent() {
-    restoreOfyKeys();
-    return billingEventRecurring;
-  }
-
-  /**
-   * Restores history ids for composite VKeys after a load from datastore.
-   *
-   * <p>For use by DomainContent.load() ONLY.
-   */
-  protected void restoreHistoryIds() {
-    billingEventOneTimeHistoryId = DomainBase.getHistoryId(billingEventOneTime);
-    billingEventRecurringHistoryId = DomainBase.getHistoryId(billingEventRecurring);
+    return billingEventRecurring == null ? null : billingEventRecurring.createVKey();
  }

  /**
@@ -152,7 +132,6 @@ public class GracePeriodBase extends ImmutableObject {
   */
  @Override
  protected Map<Field, Object> getSignificantFields() {
-    restoreOfyKeys();
    // Can't use streams or ImmutableMap because we can have null values.
    Map<Field, Object> result = new LinkedHashMap();
    for (Map.Entry<Field, Object> entry : ModelUtils.getFieldValues(this).entrySet()) {
@@ -162,33 +141,4 @@ public class GracePeriodBase extends ImmutableObject {
    }
    return result;
  }
-
-  /**
-   * Restores Ofy keys in the billing events.
-   *
-   * <p>This must be called by all methods that access the one time or recurring billing event keys.
-   * When the billing event keys are loaded from SQL, they are loaded as asymmetric keys because the
-   * database columns that we load them from do not contain all of the information necessary to
-   * reconsitute the Ofy side of the key. In other cases, we restore the Ofy key during the
-   * hibernate {@link javax.persistence.PostLoad} method from the other fields of the object, but we
-   * have been unable to make this work with hibernate's internal persistence model in this case
-   * because the {@link GracePeriod}'s hash code is evaluated prior to these calls, and would be
-   * invalidated by changing the fields.
-   */
-  private final synchronized void restoreOfyKeys() {
-    if (billingEventOneTime != null && !billingEventOneTime.maybeGetOfyKey().isPresent()) {
-      billingEventOneTime =
-          DomainBase.restoreOfyFrom(
-              Key.create(DomainBase.class, domainRepoId),
-              billingEventOneTime,
-              billingEventOneTimeHistoryId);
-    }
-    if (billingEventRecurring != null && !billingEventRecurring.maybeGetOfyKey().isPresent()) {
-      billingEventRecurring =
-          DomainBase.restoreOfyFrom(
-              Key.create(DomainBase.class, domainRepoId),
-              billingEventRecurring,
-              billingEventRecurringHistoryId);
-    }
-  }
 }
@@ -111,9 +111,9 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
  @Nullable
  @Index
  @AttributeOverrides({
-    @AttributeOverride(name = "domainRepoId", column = @Column(name = "redemption_domain_repo_id")),
+    @AttributeOverride(name = "repoId", column = @Column(name = "redemption_domain_repo_id")),
    @AttributeOverride(
-        name = "domainHistoryId",
+        name = "historyRevisionId",
        column = @Column(name = "redemption_domain_history_id"))
  })
  DomainHistoryVKey redemptionHistoryEntry;
@@ -192,8 +192,9 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
    return token;
  }

-  public Optional<VKey<HistoryEntry>> getRedemptionHistoryEntry() {
-    return Optional.ofNullable(redemptionHistoryEntry);
+  public Optional<VKey<? extends HistoryEntry>> getRedemptionHistoryEntry() {
+    return Optional.ofNullable(
+        redemptionHistoryEntry == null ? null : redemptionHistoryEntry.createDomainHistoryVKey());
  }

  public boolean isRedeemed() {
@@ -291,9 +292,10 @@ public class AllocationToken extends BackupGroupRoot implements Buildable, Datas
      return this;
    }

-    public Builder setRedemptionHistoryEntry(DomainHistoryVKey redemptionHistoryEntry) {
+    public Builder setRedemptionHistoryEntry(VKey<? extends HistoryEntry> redemptionHistoryEntry) {
+      checkArgumentNotNull(redemptionHistoryEntry, "Redemption history entry must not be null");
      getInstance().redemptionHistoryEntry =
-          checkArgumentNotNull(redemptionHistoryEntry, "Redemption history entry must not be null");
+          DomainHistoryVKey.create(redemptionHistoryEntry.getOfyKey());
      return this;
    }

@@ -42,8 +42,8 @@ import google.registry.model.translators.CidrAddressBlockTranslatorFactory;
 import google.registry.model.translators.CommitLogRevisionsTranslatorFactory;
 import google.registry.model.translators.CreateAutoTimestampTranslatorFactory;
 import google.registry.model.translators.CurrencyUnitTranslatorFactory;
-import google.registry.model.translators.DomainHistoryVKeyTranslatorFactory;
 import google.registry.model.translators.DurationTranslatorFactory;
+import google.registry.model.translators.EppHistoryVKeyTranslatorFactory;
 import google.registry.model.translators.InetAddressTranslatorFactory;
 import google.registry.model.translators.ReadableInstantUtcTranslatorFactory;
 import google.registry.model.translators.UpdateAutoTimestampTranslatorFactory;
@@ -128,7 +128,7 @@ public class ObjectifyService {
            new CreateAutoTimestampTranslatorFactory(),
            new CurrencyUnitTranslatorFactory(),
            new DurationTranslatorFactory(),
-            new DomainHistoryVKeyTranslatorFactory(),
+            new EppHistoryVKeyTranslatorFactory(),
            new InetAddressTranslatorFactory(),
            new MoneyStringTranslatorFactory(),
            new ReadableInstantUtcTranslatorFactory(),
@@ -21,10 +21,10 @@ import javax.persistence.TemporalType;
 import org.joda.time.LocalDate;

 /**
- * Data access object for {@link google.registry.model.reporting.Spec11ThreatMatch}.
+ * Data access object for {@link Spec11ThreatMatch}.
 *
- * <p>A JpaTransactionManager is passed into each static method because they are called from a BEAM
- * pipeline and we don't know where it's coming from.
+ * <p>The transaction manager is passed as a parameter because this could be called either from a
+ * BEAM pipeline or standard non-BEAM code.
 */
 public class Spec11ThreatMatchDao {

@@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.Iterables.isEmpty;
 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
 import static google.registry.model.ofy.ObjectifyService.allocateId;
 import static google.registry.model.ofy.ObjectifyService.ofy;
@@ -32,7 +33,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.EmbedMap;
 import com.googlecode.objectify.annotation.Entity;
@@ -82,8 +82,6 @@ import org.joda.time.DateTime;
@NotBackedUp(reason = Reason.EXTERNALLY_SOURCED)
 public class SignedMarkRevocationList extends ImmutableObject implements NonReplicatedEntity {

-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
  @VisibleForTesting static final int SHARD_SIZE = 10000;

  /** Common ancestor for queries. */
@@ -121,12 +119,11 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
      memoizeWithShortExpiration(
          () -> {
            SignedMarkRevocationList datastoreList = loadFromDatastore();
-            // Also load the list from Cloud SQL, compare the two lists, and log if different.
-            try {
-              loadAndCompareCloudSqlList(datastoreList);
-            } catch (Throwable t) {
-              logger.atSevere().withCause(t).log("Error comparing signed mark revocation lists.");
-            }
+            suppressExceptionUnlessInTest(
+                () -> {
+                  loadAndCompareCloudSqlList(datastoreList);
+                },
+                "Error comparing signed mark revocation lists.");
            return datastoreList;
          });

@@ -229,11 +226,12 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
          Maps.difference(datastoreList.revokes, cloudSqlList.revokes);
      if (!diff.areEqual()) {
        if (diff.entriesDiffering().size() > 10) {
-          logger.atWarning().log(
+          String message =
              String.format(
                  "Unequal SM revocation lists detected, Cloud SQL list with revision id %d has %d"
                      + " different records than the current Datastore list.",
-                  cloudSqlList.revisionId, diff.entriesDiffering().size()));
+                  cloudSqlList.revisionId, diff.entriesDiffering().size());
+          throw new RuntimeException(message);
        } else {
          StringBuilder diffMessage = new StringBuilder("Unequal SM revocation lists detected:\n");
          diff.entriesDiffering()
@@ -243,11 +241,13 @@ public class SignedMarkRevocationList extends ImmutableObject implements NonRepl
                          String.format(
                              "SMD %s has key %s in Datastore and key %s in Cloud SQL.\n",
                              label, valueDiff.leftValue(), valueDiff.rightValue())));
-          logger.atWarning().log(diffMessage.toString());
+          throw new RuntimeException(diffMessage.toString());
        }
      }
    } else {
-      logger.atWarning().log("Signed mark revocation list in Cloud SQL is empty.");
+      if (datastoreList.size() != 0) {
+        throw new RuntimeException("Signed mark revocation list in Cloud SQL is empty.");
+      }
    }
  }

@@ -15,6 +15,7 @@
 package google.registry.model.smd;

 import static google.registry.model.CacheUtils.memoizeWithShortExpiration;
+import static google.registry.model.DatabaseMigrationUtils.suppressExceptionUnlessInTest;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;

 import com.google.common.base.Supplier;
@@ -60,14 +61,14 @@ public class SignedMarkRevocationListDao {
   * the authoritative database.
   */
  static void trySave(SignedMarkRevocationList signedMarkRevocationList) {
-    try {
-      SignedMarkRevocationListDao.save(signedMarkRevocationList);
-      logger.atInfo().log(
-          "Inserted %,d signed mark revocations into Cloud SQL",
-          signedMarkRevocationList.revokes.size());
-    } catch (Throwable e) {
-      logger.atSevere().withCause(e).log("Error inserting signed mark revocations into Cloud SQL");
-    }
+    suppressExceptionUnlessInTest(
+        () -> {
+          SignedMarkRevocationListDao.save(signedMarkRevocationList);
+          logger.atInfo().log(
+              "Inserted %,d signed mark revocations into Cloud SQL.",
+              signedMarkRevocationList.revokes.size());
+        },
+        "Error inserting signed mark revocations into Cloud SQL.");
  }

  private static void save(SignedMarkRevocationList signedMarkRevocationList) {
@@ -1,48 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.model.translators;
-
-import com.google.appengine.api.datastore.Key;
-import google.registry.persistence.DomainHistoryVKey;
-import javax.annotation.Nullable;
-
-/** Translator factory for {@link DomainHistoryVKey}. */
-public class DomainHistoryVKeyTranslatorFactory
-    extends AbstractSimpleTranslatorFactory<DomainHistoryVKey, Key> {
-
-  public DomainHistoryVKeyTranslatorFactory() {
-    super(DomainHistoryVKey.class);
-  }
-
-  @Override
-  SimpleTranslator<DomainHistoryVKey, Key> createTranslator() {
-    return new SimpleTranslator<DomainHistoryVKey, Key>() {
-
-      @Nullable
-      @Override
-      public DomainHistoryVKey loadValue(@Nullable Key datastoreValue) {
-        return datastoreValue == null
-            ? null
-            : DomainHistoryVKey.create(com.googlecode.objectify.Key.create(datastoreValue));
-      }
-
-      @Nullable
-      @Override
-      public Key saveValue(@Nullable DomainHistoryVKey pojoValue) {
-        return pojoValue == null ? null : pojoValue.getOfyKey().getRaw();
-      }
-    };
-  }
-}
@@ -0,0 +1,110 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.translators;
+
+import static com.google.common.collect.ImmutableMap.toImmutableMap;
+import static java.util.function.Function.identity;
+
+import com.google.appengine.api.datastore.Key;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
+import google.registry.persistence.DomainHistoryVKey;
+import google.registry.persistence.EppHistoryVKey;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import javax.annotation.Nullable;
+
+/** Translator factory for {@link EppHistoryVKey}. */
+public class EppHistoryVKeyTranslatorFactory
+    extends AbstractSimpleTranslatorFactory<EppHistoryVKey, Key> {
+
+  public EppHistoryVKeyTranslatorFactory() {
+    super(EppHistoryVKey.class);
+  }
+
+  // This map is used when we need to convert the raw Datastore key to its VKey instance. We have
+  // one dedicated VKey class, e.g. DomainHistoryVKey, for each such kind of entity, and we need
+  // a way to map the raw Datastore key to its VKey class. So, we use the kind path as the key of
+  // the map, and the kind path is created by concatenating all the kind strings in a raw Datastore
+  // key, e.g. the map key for ContactPollMessageVKey is "ContactResource/HistoryEntry/PollMessage".
+  @VisibleForTesting
+  static final ImmutableMap<String, Class<? extends EppHistoryVKey>> kindPathToVKeyClass =
+      ImmutableSet.of(DomainHistoryVKey.class, BillingEventVKey.class, BillingRecurrenceVKey.class)
+          .stream()
+          .collect(toImmutableMap(EppHistoryVKeyTranslatorFactory::getKindPath, identity()));
+
+  /**
+   * Gets the kind path string for the given {@link Class}.
+   *
+   * <p>This method calls the getKindPath method on an instance of the given {@link Class} to get
+   * the kind path string.
+   */
+  private static String getKindPath(Class<? extends EppHistoryVKey> clazz) {
+    try {
+      Constructor<?> constructor = clazz.getDeclaredConstructor();
+      constructor.setAccessible(true);
+      Object instance = constructor.newInstance();
+      Method getKindPathMethod = EppHistoryVKey.class.getDeclaredMethod("getKindPath");
+      getKindPathMethod.setAccessible(true);
+      return (String) getKindPathMethod.invoke(instance);
+    } catch (Throwable t) {
+      throw new IllegalStateException(t);
+    }
+  }
+
+  @Override
+  SimpleTranslator<EppHistoryVKey, Key> createTranslator() {
+    return new SimpleTranslator<EppHistoryVKey, Key>() {
+
+      @Nullable
+      @Override
+      public EppHistoryVKey loadValue(@Nullable Key datastoreValue) {
+        if (datastoreValue == null) {
+          return null;
+        } else {
+          com.googlecode.objectify.Key<?> ofyKey =
+              com.googlecode.objectify.Key.create(datastoreValue);
+          String kindPath = EppHistoryVKey.createKindPath(ofyKey);
+          if (kindPathToVKeyClass.containsKey(kindPath)) {
+            Class<? extends EppHistoryVKey> vKeyClass = kindPathToVKeyClass.get(kindPath);
+            try {
+              Method createVKeyMethod =
+                  vKeyClass.getDeclaredMethod("create", com.googlecode.objectify.Key.class);
+              return (EppHistoryVKey) createVKeyMethod.invoke(null, ofyKey);
+            } catch (NoSuchMethodException e) {
+              throw new IllegalStateException(
+                  "Missing static method create(com.googlecode.objectify.Key) on " + vKeyClass);
+            } catch (IllegalAccessException | InvocationTargetException e) {
+              throw new IllegalStateException("Error invoking createVKey on " + vKeyClass, e);
+            }
+          } else {
+            throw new IllegalStateException(
+                "Missing EppHistoryVKey implementation for kind path: " + kindPath);
+          }
+        }
+      }
+
+      @Nullable
+      @Override
+      public Key saveValue(@Nullable EppHistoryVKey pojoValue) {
+        return pojoValue == null ? null : pojoValue.createOfyKey().getRaw();
+      }
+    };
+  }
+}
@@ -46,7 +46,10 @@ public class UpdateAutoTimestampTranslatorFactory
      /** Save a timestamp, setting it to the current time. */
      @Override
      public Date saveValue(UpdateAutoTimestamp pojoValue) {
-        return tm().getTransactionTime().toDate();
-      }};
+        return UpdateAutoTimestamp.autoUpdateEnabled()
+            ? tm().getTransactionTime().toDate()
+            : pojoValue.getTimestamp().toDate();
+      }
+    };
  }
 }
@@ -0,0 +1,134 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+
+import com.googlecode.objectify.Key;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.billing.BillingEvent.OneTime;
+import google.registry.model.billing.BillingEvent.Recurring;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.HistoryEntry;
+import javax.annotation.Nullable;
+import javax.persistence.AttributeOverride;
+import javax.persistence.AttributeOverrides;
+import javax.persistence.Column;
+import javax.persistence.Embeddable;
+import javax.persistence.MappedSuperclass;
+
+/** Base class for {@link BillingEvent}'s {@link VKey}. */
+@MappedSuperclass
+public abstract class BillingVKey<K> extends EppHistoryVKey<K, DomainBase> {
+  Long billingId;
+
+  // Hibernate requires a default constructor.
+  BillingVKey() {}
+
+  BillingVKey(String repoId, long historyRevisionId, long billingId) {
+    super(repoId, historyRevisionId);
+    this.billingId = billingId;
+  }
+
+  Key<HistoryEntry> createHistoryEntryKey() {
+    return Key.create(Key.create(DomainBase.class, repoId), HistoryEntry.class, historyRevisionId);
+  }
+
+  @Override
+  public Object createSqlKey() {
+    return billingId;
+  }
+
+  /** VKey class for {@link BillingEvent.OneTime} that belongs to a {@link DomainBase} entity. */
+  @Embeddable
+  @AttributeOverrides({
+    @AttributeOverride(name = "repoId", column = @Column(name = "billing_event_domain_repo_id")),
+    @AttributeOverride(
+        name = "historyRevisionId",
+        column = @Column(name = "billing_event_history_id")),
+    @AttributeOverride(name = "billingId", column = @Column(name = "billing_event_id"))
+  })
+  public static class BillingEventVKey extends BillingVKey<OneTime> {
+
+    // Hibernate requires this default constructor
+    private BillingEventVKey() {}
+
+    private BillingEventVKey(String repoId, long historyRevisionId, long billingEventId) {
+      super(repoId, historyRevisionId, billingEventId);
+    }
+
+    @Override
+    public Key<OneTime> createOfyKey() {
+      return Key.create(createHistoryEntryKey(), BillingEvent.OneTime.class, billingId);
+    }
+
+    /** Creates a {@link BillingEventVKey} instance from the given {@link Key} instance. */
+    public static BillingEventVKey create(@Nullable Key<BillingEvent.OneTime> ofyKey) {
+      if (ofyKey == null) {
+        return null;
+      }
+      long billingEventId = ofyKey.getId();
+      long historyRevisionId = ofyKey.getParent().getId();
+      String repoId = ofyKey.getParent().getParent().getName();
+      return new BillingEventVKey(repoId, historyRevisionId, billingEventId);
+    }
+
+    /** Creates a {@link BillingEventVKey} instance from the given {@link VKey} instance. */
+    public static BillingEventVKey create(@Nullable VKey<BillingEvent.OneTime> vKey) {
+      return vKey == null ? null : create(vKey.getOfyKey());
+    }
+  }
+
+  /** VKey class for {@link BillingEvent.Recurring} that belongs to a {@link DomainBase} entity. */
+  @Embeddable
+  @AttributeOverrides({
+    @AttributeOverride(
+        name = "repoId",
+        column = @Column(name = "billing_recurrence_domain_repo_id")),
+    @AttributeOverride(
+        name = "historyRevisionId",
+        column = @Column(name = "billing_recurrence_history_id")),
+    @AttributeOverride(name = "billingId", column = @Column(name = "billing_recurrence_id"))
+  })
+  public static class BillingRecurrenceVKey extends BillingVKey<Recurring> {
+
+    // Hibernate requires this default constructor
+    private BillingRecurrenceVKey() {}
+
+    private BillingRecurrenceVKey(String repoId, long historyRevisionId, long billingEventId) {
+      super(repoId, historyRevisionId, billingEventId);
+    }
+
+    @Override
+    public Key<Recurring> createOfyKey() {
+      return Key.create(createHistoryEntryKey(), BillingEvent.Recurring.class, billingId);
+    }
+
+    /** Creates a {@link BillingRecurrenceVKey} instance from the given {@link Key} instance. */
+    public static BillingRecurrenceVKey create(@Nullable Key<BillingEvent.Recurring> ofyKey) {
+      if (ofyKey == null) {
+        return null;
+      }
+      long billingEventId = ofyKey.getId();
+      long historyRevisionId = ofyKey.getParent().getId();
+      String repoId = ofyKey.getParent().getParent().getName();
+      return new BillingRecurrenceVKey(repoId, historyRevisionId, billingEventId);
+    }
+
+    /** Creates a {@link BillingRecurrenceVKey} instance from the given {@link VKey} instance. */
+    public static BillingRecurrenceVKey create(@Nullable VKey<BillingEvent.Recurring> vKey) {
+      return vKey == null ? null : create(vKey.getOfyKey());
+    }
+  }
+}
@@ -18,45 +18,44 @@ import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;

 import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.DomainHistory.DomainHistoryId;
 import google.registry.model.reporting.HistoryEntry;
 import javax.persistence.Embeddable;
-import javax.persistence.PostLoad;

 /** {@link VKey} for {@link HistoryEntry} which parent is {@link DomainBase}. */
@Embeddable
-public class DomainHistoryVKey extends VKey<HistoryEntry> {
-
-  private String domainRepoId;
-
-  private Long domainHistoryId;
+public class DomainHistoryVKey extends EppHistoryVKey<HistoryEntry, DomainBase> {

  // Hibernate requires a default constructor
  private DomainHistoryVKey() {}

-  private DomainHistoryVKey(String domainRepoId, long domainHistoryId) {
-    initWith(domainRepoId, domainHistoryId);
+  private DomainHistoryVKey(String repoId, long historyRevisionId) {
+    super(repoId, historyRevisionId);
  }

-  @PostLoad
-  void postLoad() {
-    initWith(domainRepoId, domainHistoryId);
+  @Override
+  public Object createSqlKey() {
+    return new DomainHistoryId(repoId, historyRevisionId);
  }

-  private void initWith(String domainRepoId, long domainHistoryId) {
-    this.kind = HistoryEntry.class;
-    this.ofyKey =
-        Key.create(Key.create(DomainBase.class, domainRepoId), HistoryEntry.class, domainHistoryId);
-    this.sqlKey = new DomainHistoryId(domainRepoId, domainHistoryId);
-    this.domainRepoId = domainRepoId;
-    this.domainHistoryId = domainHistoryId;
+  @Override
+  public Key<HistoryEntry> createOfyKey() {
+    return Key.create(Key.create(DomainBase.class, repoId), HistoryEntry.class, historyRevisionId);
  }

  /** Creates {@link DomainHistoryVKey} from the given {@link Key} instance. */
-  public static DomainHistoryVKey create(Key<HistoryEntry> ofyKey) {
+  public static DomainHistoryVKey create(Key<? extends HistoryEntry> ofyKey) {
    checkArgumentNotNull(ofyKey, "ofyKey must be specified");
-    String domainRepoId = ofyKey.getParent().getName();
-    long domainHistoryId = ofyKey.getId();
-    return new DomainHistoryVKey(domainRepoId, domainHistoryId);
+    String repoId = ofyKey.getParent().getName();
+    long historyRevisionId = ofyKey.getId();
+    return new DomainHistoryVKey(repoId, historyRevisionId);
+  }
+
+  public VKey<? extends HistoryEntry> createDomainHistoryVKey() {
+    return VKey.create(
+        DomainHistory.class,
+        createSqlKey(),
+        Key.create(Key.create(DomainBase.class, repoId), DomainHistory.class, historyRevisionId));
  }
 }
@@ -0,0 +1,107 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import com.google.common.base.Joiner;
+import com.googlecode.objectify.Key;
+import google.registry.model.EppResource;
+import google.registry.model.ImmutableObject;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.util.TypeUtils.TypeInstantiator;
+import java.io.Serializable;
+import javax.annotation.Nullable;
+import javax.persistence.Access;
+import javax.persistence.AccessType;
+import javax.persistence.MappedSuperclass;
+
+/**
+ * Base class for {@link VKey} which ofyKey has a {@link HistoryEntry} key as its parent and a key
+ * for EPP resource as its grandparent.
+ *
+ * <p>For such a {@link VKey}, we need to provide two type parameters to indicate the type of {@link
+ * VKey} itself and the type of EPP resource respectively.
+ *
+ * @param <K> type of the {@link VKey}
+ * @param <E> type of the EPP resource that the key belongs to
+ */
+@MappedSuperclass
+@Access(AccessType.FIELD)
+public abstract class EppHistoryVKey<K, E extends EppResource> extends ImmutableObject
+    implements Serializable {
+
+  private static final long serialVersionUID = -3906580677709539818L;
+
+  String repoId;
+
+  Long historyRevisionId;
+
+  // Hibernate requires a default constructor.
+  EppHistoryVKey() {}
+
+  EppHistoryVKey(String repoId, long historyRevisionId) {
+    this.repoId = repoId;
+    this.historyRevisionId = historyRevisionId;
+  }
+
+  /**
+   * Returns the kind path for the ofyKey in this instance.
+   *
+   * <p>This method is only used reflectively by {@link EppHistoryVKeyTranslatorFactory} to get the
+   * kind path for a given {@link EppHistoryVKey} instance so it is marked as a private method.
+   *
+   * @see #createKindPath(Key)
+   */
+  @SuppressWarnings("unused")
+  private String getKindPath() {
+    String eppKind = Key.getKind(new TypeInstantiator<E>(getClass()) {}.getExactType());
+    String keyKind = Key.getKind(new TypeInstantiator<K>(getClass()) {}.getExactType());
+    if (keyKind.equals(Key.getKind(HistoryEntry.class))) {
+      return createKindPath(eppKind, keyKind);
+    } else {
+      return createKindPath(eppKind, Key.getKind(HistoryEntry.class), keyKind);
+    }
+  }
+
+  /**
+   * Creates the kind path for the given ofyKey}.
+   *
+   * <p>The kind path is a string including all kind names(delimited by slash) of a hierarchical
+   * {@link Key}, e.g., the kind path for BillingEvent.OneTime is "DomainBase/HistoryEntry/OneTime".
+   */
+  @Nullable
+  public static String createKindPath(@Nullable Key<?> ofyKey) {
+    if (ofyKey == null) {
+      return null;
+    } else if (ofyKey.getParent() == null) {
+      return ofyKey.getKind();
+    } else {
+      return createKindPath(createKindPath(ofyKey.getParent()), ofyKey.getKind());
+    }
+  }
+
+  private static String createKindPath(String... kinds) {
+    return Joiner.on("/").join(kinds);
+  }
+
+  /** Creates a {@link VKey} from this instance. */
+  public VKey<K> createVKey() {
+    Class<K> vKeyType = new TypeInstantiator<K>(getClass()) {}.getExactType();
+    return VKey.create(vKeyType, createSqlKey(), createOfyKey());
+  }
+
+  public abstract Object createSqlKey();
+
+  public abstract Key<K> createOfyKey();
+}
@@ -20,6 +20,7 @@ import google.registry.config.RegistryConfig.ConfigModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule.AppEngineJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
+import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.util.UtilsModule;
 import javax.inject.Singleton;
 import javax.persistence.EntityManagerFactory;
@@ -32,6 +33,7 @@ import javax.persistence.EntityManagerFactory;
      CredentialModule.class,
      KmsModule.class,
      PersistenceModule.class,
+      SecretManagerModule.class,
      UtilsModule.class
    })
 public interface PersistenceComponent {
@@ -26,6 +26,7 @@ import com.google.api.client.auth.oauth2.Credential;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
+import com.google.common.flogger.FluentLogger;
 import dagger.Module;
 import dagger.Provides;
 import google.registry.config.RegistryConfig.Config;
@@ -33,6 +34,11 @@ import google.registry.keyring.kms.KmsKeyring;
 import google.registry.persistence.transaction.CloudSqlCredentialSupplier;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
+import google.registry.privileges.secretmanager.SqlCredential;
+import google.registry.privileges.secretmanager.SqlCredentialStore;
+import google.registry.privileges.secretmanager.SqlUser;
+import google.registry.privileges.secretmanager.SqlUser.RobotId;
+import google.registry.privileges.secretmanager.SqlUser.RobotUser;
 import google.registry.tools.AuthModule.CloudSqlClientCredential;
 import google.registry.util.Clock;
 import java.lang.annotation.Documented;
@@ -47,6 +53,8 @@ import org.hibernate.cfg.Environment;
 /** Dagger module class for the persistence layer. */
@Module
 public class PersistenceModule {
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
  // This name must be the same as the one defined in persistence.xml.
  public static final String PERSISTENCE_UNIT_NAME = "nomulus";
  public static final String HIKARI_CONNECTION_TIMEOUT = "hibernate.hikari.connectionTimeout";
@@ -122,11 +130,17 @@ public class PersistenceModule {
  static JpaTransactionManager provideAppEngineJpaTm(
      @Config("cloudSqlUsername") String username,
      KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
      Clock clock) {
    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
    overrides.put(Environment.USER, username);
    overrides.put(Environment.PASS, kmsKeyring.getCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
    return new JpaTransactionManagerImpl(create(overrides), clock);
  }

@@ -136,6 +150,7 @@ public class PersistenceModule {
  static JpaTransactionManager provideNomulusToolJpaTm(
      @Config("toolsCloudSqlUsername") String username,
      KmsKeyring kmsKeyring,
+      SqlCredentialStore credentialStore,
      @PartialCloudSqlConfigs ImmutableMap<String, String> cloudSqlConfigs,
      @CloudSqlClientCredential Credential credential,
      Clock clock) {
@@ -143,6 +158,11 @@ public class PersistenceModule {
    HashMap<String, String> overrides = Maps.newHashMap(cloudSqlConfigs);
    overrides.put(Environment.USER, username);
    overrides.put(Environment.PASS, kmsKeyring.getToolsCloudSqlPassword());
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.TOOL),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
    return new JpaTransactionManagerImpl(create(overrides), clock);
  }

@@ -150,6 +170,7 @@ public class PersistenceModule {
  @Singleton
  @SocketFactoryJpaTm
  static JpaTransactionManager provideSocketFactoryJpaTm(
+      SqlCredentialStore credentialStore,
      @Config("beamCloudSqlUsername") String username,
      @Config("beamCloudSqlPassword") String password,
      @Config("beamHibernateHikariMaximumPoolSize") int hikariMaximumPoolSize,
@@ -159,6 +180,12 @@ public class PersistenceModule {
    overrides.put(Environment.USER, username);
    overrides.put(Environment.PASS, password);
    overrides.put(HIKARI_MAXIMUM_POOL_SIZE, String.valueOf(hikariMaximumPoolSize));
+    // TODO(b/175700623): consider assigning different logins to pipelines
+    validateCredentialStore(
+        credentialStore,
+        new RobotUser(RobotId.NOMULUS),
+        overrides.get(Environment.USER),
+        overrides.get(Environment.PASS));
    return new JpaTransactionManagerImpl(create(overrides), clock);
  }

@@ -203,6 +230,30 @@ public class PersistenceModule {
    return emf;
  }

+  /** Verifies that the credential from the Secret Manager matches the one currently in use.
+   *
+   * <p>This is a helper for the transition to the Secret Manager, and will be removed once data
+   * and permissions are properly set up for all projects.
+   **/
+  private static void validateCredentialStore(
+      SqlCredentialStore credentialStore, SqlUser sqlUser, String login, String password) {
+    try {
+      SqlCredential credential = credentialStore.getCredential(sqlUser);
+      if (!credential.login().equals(login)) {
+        logger.atWarning().log(
+            "Wrong login for %s. Expecting %s, found %s.",
+            sqlUser.geUserName(), login, credential.login());
+        return;
+      }
+      if (!credential.password().equals(password)) {
+        logger.atWarning().log("Wrong password for %s.", sqlUser.geUserName());
+      }
+      logger.atWarning().log("Credentials in the kerying and the secret manager match.");
+    } catch (Throwable e) {
+      logger.atWarning().log(e.getMessage());
+    }
+  }
+
  /** Dagger qualifier for {@link JpaTransactionManager} used for App Engine application. */
  @Qualifier
  @Documented
@@ -31,7 +31,14 @@ public class UpdateAutoTimestampConverter

  @Override
  public Timestamp convertToDatabaseColumn(UpdateAutoTimestamp entity) {
-    return Timestamp.from(DateTimeUtils.toZonedDateTime(jpaTm().getTransactionTime()).toInstant());
+    return Timestamp.from(
+        DateTimeUtils.toZonedDateTime(
+                UpdateAutoTimestamp.autoUpdateEnabled()
+                        || entity == null
+                        || entity.getTimestamp() == null
+                    ? jpaTm().getTransactionTime()
+                    : entity.getTimestamp())
+            .toInstant());
  }

  @Override
@@ -16,13 +16,13 @@ package google.registry.privileges.secretmanager;


 import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
-import dagger.Component;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.config.CredentialModule.DefaultCredential;
 import google.registry.config.RegistryConfig.Config;
-import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.util.GoogleCredentialsBundle;
 import google.registry.util.Retrier;
-import google.registry.util.UtilsModule;
 import java.io.IOException;
 import javax.inject.Singleton;

@@ -32,20 +32,29 @@ public abstract class SecretManagerModule {

  @Provides
  @Singleton
-  static SecretManagerClient provideSecretManagerClient(
-      @Config("projectId") String project, Retrier retrier) {
+  static SecretManagerServiceSettings provideSecretManagerSetting(
+      @DefaultCredential GoogleCredentialsBundle credentialsBundle) {
    try {
-      SecretManagerServiceClient stub = SecretManagerServiceClient.create();
+      return SecretManagerServiceSettings.newBuilder()
+          .setCredentialsProvider(() -> credentialsBundle.getGoogleCredentials())
+          .build();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Provides
+  @Singleton
+  static SecretManagerClient provideSecretManagerClient(
+      SecretManagerServiceSettings serviceSettings,
+      @Config("projectId") String project,
+      Retrier retrier) {
+    try {
+      SecretManagerServiceClient stub = SecretManagerServiceClient.create(serviceSettings);
      Runtime.getRuntime().addShutdownHook(new Thread(stub::close));
      return new SecretManagerClientImpl(project, stub, retrier);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
  }
-
-  @Singleton
-  @Component(modules = {ConfigModule.class, SecretManagerModule.class, UtilsModule.class})
-  public interface SecretManagerComponent {
-    SecretManagerClient secretManagerClient();
-  }
 }
@@ -48,7 +48,12 @@ public abstract class SqlUser {

  /** Enumerates the {@link RobotUser RobotUsers} in the system. */
  public enum RobotId {
-    NOMULUS;
+    NOMULUS,
+    /**
+     * Credential for RegistryTool. This is temporary, and will be removed when tool users are
+     * assigned their personal credentials.
+     */
+    TOOL;
  }

  /** Information of a RobotUser for privilege management purposes. */
@@ -14,9 +14,11 @@

 package google.registry.reporting.spec11;

+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.nio.charset.StandardCharsets.UTF_8;

 import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.flogger.FluentLogger;
@@ -59,6 +61,25 @@ public class Spec11RegistrarThreatMatchesParser {
    return getFromFile(getGcsFilename(date));
  }

+  /** Returns registrar:set-of-threat-match pairings from the file, or empty if it doesn't exist. */
+  public ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
+      throws IOException {
+    if (!gcsUtils.existsAndNotEmpty(spec11ReportFilename)) {
+      return ImmutableSet.of();
+    }
+    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
+    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename);
+        InputStreamReader isr = new InputStreamReader(in, UTF_8)) {
+      // Skip the header at line 0
+      return Splitter.on("\n")
+          .omitEmptyStrings()
+          .splitToStream(CharStreams.toString(isr))
+          .skip(1)
+          .map(this::parseRegistrarThreatMatch)
+          .collect(toImmutableSet());
+    }
+  }
+
  public Optional<LocalDate> getPreviousDateWithMatches(LocalDate date) {
    LocalDate yesterday = date.minusDays(1);
    GcsFilename gcsFilename = getGcsFilename(yesterday);
@@ -82,20 +103,6 @@ public class Spec11RegistrarThreatMatchesParser {
    return new GcsFilename(reportingBucket, Spec11Pipeline.getSpec11ReportFilePath(localDate));
  }

-  private ImmutableSet<RegistrarThreatMatches> getFromFile(GcsFilename spec11ReportFilename)
-      throws IOException, JSONException {
-    ImmutableSet.Builder<RegistrarThreatMatches> builder = ImmutableSet.builder();
-    try (InputStream in = gcsUtils.openInputStream(spec11ReportFilename)) {
-      ImmutableList<String> reportLines =
-          ImmutableList.copyOf(CharStreams.toString(new InputStreamReader(in, UTF_8)).split("\n"));
-      // Iterate from 1 to size() to skip the header at line 0.
-      for (int i = 1; i < reportLines.size(); i++) {
-        builder.add(parseRegistrarThreatMatch(reportLines.get(i)));
-      }
-      return builder.build();
-    }
-  }
-
  private RegistrarThreatMatches parseRegistrarThreatMatch(String line) throws JSONException {
    JSONObject reportJSON = new JSONObject(line);
    String clientId = reportJSON.getString(Spec11Pipeline.REGISTRAR_CLIENT_ID_FIELD);
@@ -16,6 +16,7 @@ package google.registry.tools;

 import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
+import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;

@@ -32,6 +33,7 @@ public final class RegistryTool {
      new ImmutableMap.Builder<String, Class<? extends Command>>()
          .put("ack_poll_messages", AckPollMessagesCommand.class)
          .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
+          .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
          .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
          .put("check_domain", CheckDomainCommand.class)
          .put("check_domain_claims", CheckDomainClaimsCommand.class)
@@ -0,0 +1,216 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.beam.spec11.ThreatMatch;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.model.reporting.Spec11ThreatMatchDao;
+import google.registry.reporting.spec11.RegistrarThreatMatches;
+import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import google.registry.util.Clock;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+import javax.inject.Inject;
+import org.joda.time.LocalDate;
+
+/**
+ * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
+ *
+ * <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
+ * rough estimate of when we started using this format) and convert those RegistrarThreatMatches
+ * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
+ *
+ * <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
+ * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
+ * or if the domain objects were not active at the time of the scan.
+ */
+@Parameters(
+    commandDescription =
+        "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
+            + "Cloud SQL database.")
+public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
+    implements CommandWithRemoteApi {
+
+  private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
+
+  @Parameter(
+      names = {"-o", "--overwrite_existing_dates"},
+      description =
+          "Whether the command will overwrite data that already exists for dates that exist in the "
+              + "GCS bucket. Defaults to false.")
+  private boolean overrideExistingDates;
+
+  @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
+  // Inject the clock for testing purposes
+  @Inject Clock clock;
+
+  @Override
+  protected String prompt() {
+    return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
+  }
+
+  @Override
+  protected String execute() {
+    ImmutableList<LocalDate> dates = getDatesToBackfill();
+    ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
+        new ImmutableListMultimap.Builder<>();
+    for (LocalDate date : dates) {
+      try {
+        // It's OK if the file doesn't exist for a particular date; the result will be empty.
+        threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
+      } catch (IOException e) {
+        throw new RuntimeException(
+            String.format("Error parsing through file with date %s.", date), e);
+      }
+    }
+    ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
+        threatMatchesBuilder.build();
+    // Look up all possible DomainBases for these domain names, any of which can be in the past
+    ImmutableListMultimap<String, DomainBase> domainsByDomainName =
+        getDomainsByDomainName(threatMatches);
+
+    // For each date, convert all threat matches with the proper domain repo ID
+    int totalNumThreats = 0;
+    for (LocalDate date : threatMatches.keySet()) {
+      ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
+      for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
+        rtm.threatMatches().stream()
+            .map(
+                threatMatch ->
+                    threatMatchToCloudSqlObject(
+                        threatMatch, date, rtm.clientId(), domainsByDomainName))
+            .forEach(spec11ThreatsBuilder::add);
+      }
+      ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
+      jpaTm()
+          .transact(
+              () -> {
+                Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
+                jpaTm().putAll(spec11Threats);
+              });
+      totalNumThreats += spec11Threats.size();
+    }
+    return String.format(
+        "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
+  }
+
+  /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
+  private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
+      ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
+    return threatMatchesByDate.values().stream()
+        .map(RegistrarThreatMatches::threatMatches)
+        .flatMap(ImmutableList::stream)
+        .map(ThreatMatch::fullyQualifiedDomainName)
+        .distinct()
+        .collect(
+            flatteningToImmutableListMultimap(
+                Function.identity(),
+                (domainName) -> {
+                  List<DomainBase> domains =
+                      ofy()
+                          .load()
+                          .type(DomainBase.class)
+                          .filter("fullyQualifiedDomainName", domainName)
+                          .list();
+                  domains.sort(Comparator.comparing(DomainBase::getCreationTime).reversed());
+                  checkState(
+                      !domains.isEmpty(),
+                      "Domain name %s had no associated DomainBase objects.",
+                      domainName);
+                  return domains.stream();
+                }));
+  }
+
+  /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
+  private Spec11ThreatMatch threatMatchToCloudSqlObject(
+      ThreatMatch threatMatch,
+      LocalDate date,
+      String registrarId,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    DomainBase domain =
+        findDomainAsOfDateOrThrow(
+            threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
+    return new Spec11ThreatMatch.Builder()
+        .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
+        .setCheckDate(date)
+        .setRegistrarId(registrarId)
+        .setDomainName(threatMatch.fullyQualifiedDomainName())
+        .setDomainRepoId(domain.getRepoId())
+        .build();
+  }
+
+  /** Returns the DomainBase object as of the particular date, which is likely in the past. */
+  private DomainBase findDomainAsOfDateOrThrow(
+      String domainName,
+      LocalDate date,
+      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
+    ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
+    for (DomainBase domain : domains) {
+      // We only know the date (not datetime) of the threat scan, so we approximate
+      LocalDate creationDate = domain.getCreationTime().toLocalDate();
+      LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
+      if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
+        return domain;
+      }
+    }
+    throw new IllegalStateException(
+        String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
+  }
+
+  /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
+  private ImmutableList<LocalDate> getDatesToBackfill() {
+    ImmutableSet<LocalDate> datesToSkip =
+        overrideExistingDates ? ImmutableSet.of() : getExistingDates();
+    ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
+    LocalDate endDate = clock.nowUtc().toLocalDate();
+    for (LocalDate currentDate = START_DATE;
+        !currentDate.isAfter(endDate);
+        currentDate = currentDate.plusDays(1)) {
+      if (!datesToSkip.contains(currentDate)) {
+        result.add(currentDate);
+      }
+    }
+    return result.build();
+  }
+
+  private ImmutableSet<LocalDate> getExistingDates() {
+    return jpaTm()
+        .transact(
+            () ->
+                jpaTm()
+                    .getEntityManager()
+                    .createQuery(
+                        "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
+                    .getResultStream()
+                    .collect(toImmutableSet()));
+  }
+}
@@ -71,7 +71,6 @@ import google.registry.model.registry.Registry;
 import google.registry.model.registry.Registry.TldState;
 import google.registry.model.registry.label.ReservedList;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -169,7 +168,7 @@ class DomainCheckFlowTest extends ResourceCheckFlowTestCase<DomainCheckFlow, Dom
        new AllocationToken.Builder()
            .setToken("abc123")
            .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
            .build());
    doCheckTest(
        create(false, "example1.tld", "In use"),
@@ -162,7 +162,6 @@ import google.registry.model.reporting.DomainTransactionRecord;
 import google.registry.model.reporting.DomainTransactionRecord.TransactionReportField;
 import google.registry.model.reporting.HistoryEntry;
 import google.registry.monitoring.whitebox.EppMetric;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import java.math.BigDecimal;
@@ -505,7 +504,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
        new AllocationToken.Builder()
            .setToken("abc123")
            .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
            .build());
    clock.advanceOneMilli();
    EppException thrown =
@@ -528,7 +527,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
    HistoryEntry historyEntry =
        ofy().load().type(HistoryEntry.class).ancestor(reloadResourceByForeignKey()).first().now();
    assertThat(ofy().load().entity(token).now().getRedemptionHistoryEntry())
-        .hasValue(DomainHistoryVKey.create(Key.create(historyEntry)));
+        .hasValue(HistoryEntry.createVKey(Key.create(historyEntry)));
  }

  @Test
@@ -1275,7 +1274,7 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
    assertThat(reloadedToken.isRedeemed()).isTrue();
    assertThat(reloadedToken.getRedemptionHistoryEntry())
        .hasValue(
-            DomainHistoryVKey.create(
+            HistoryEntry.createVKey(
                Key.create(getHistoryEntries(reloadResourceByForeignKey()).get(0))));
  }

@@ -49,7 +49,6 @@ import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.registry.Registry;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.testing.AppEngineExtension;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
@@ -198,7 +197,7 @@ class AllocationTokenFlowUtilsTest {
        new AllocationToken.Builder()
            .setToken("tokeN")
            .setTokenType(SINGLE_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
            .build());
    assertThat(
            flowUtils
@@ -28,6 +28,7 @@ import com.googlecode.objectify.annotation.Parent;
 import com.googlecode.objectify.annotation.Serialize;
 import com.googlecode.objectify.cmd.Query;
 import google.registry.model.ofy.Ofy;
+import google.registry.persistence.EppHistoryVKey;
 import google.registry.persistence.VKey;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.FakeClock;
@@ -173,7 +174,8 @@ public abstract class EntityTestCase {
          }
          // Descend into persisted ImmutableObject classes, but not anything else.
          if (ImmutableObject.class.isAssignableFrom(fieldClass)
-              && !VKey.class.isAssignableFrom(fieldClass)) {
+              && !VKey.class.isAssignableFrom(fieldClass)
+              && !EppHistoryVKey.class.isAssignableFrom(fieldClass)) {
            getAllPotentiallyIndexedFieldPaths(fieldClass).stream()
                .map(subfield -> field.getName() + "." + subfield)
                .distinct()
@@ -15,64 +15,106 @@
 package google.registry.model;

 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static org.joda.time.DateTimeZone.UTC;

+import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Ignore;
 import google.registry.model.common.CrossTldSingleton;
+import google.registry.persistence.VKey;
 import google.registry.schema.replay.EntityTest.EntityForTesting;
 import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.FakeClock;
+import google.registry.testing.TestOfyAndSql;
 import org.joda.time.DateTime;
-import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;

 /** Unit tests for {@link UpdateAutoTimestamp}. */
+@DualDatabaseTest
 public class UpdateAutoTimestampTest {

+  FakeClock clock = new FakeClock();
+
  @RegisterExtension
  public final AppEngineExtension appEngine =
      AppEngineExtension.builder()
          .withDatastoreAndCloudSql()
+          .withJpaUnitTestEntities(UpdateAutoTimestampTestObject.class)
          .withOfyTestEntities(UpdateAutoTimestampTestObject.class)
+          .withClock(clock)
          .build();

  /** Timestamped class. */
  @Entity(name = "UatTestEntity")
+  @javax.persistence.Entity
  @EntityForTesting
  public static class UpdateAutoTimestampTestObject extends CrossTldSingleton {
+    @Ignore @javax.persistence.Id long id = SINGLETON_ID;
    UpdateAutoTimestamp updateTime = UpdateAutoTimestamp.create(null);
  }

  private UpdateAutoTimestampTestObject reload() {
-    return ofy().load().entity(new UpdateAutoTimestampTestObject()).now();
+    return tm().transact(
+            () ->
+                tm().load(
+                        VKey.create(
+                            UpdateAutoTimestampTestObject.class,
+                            1L,
+                            Key.create(new UpdateAutoTimestampTestObject()))));
  }

-  @Test
+  @TestOfyAndSql
  void testSaveSetsTime() {
    DateTime transactionTime =
        tm().transact(
                () -> {
                  UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                  assertThat(object.updateTime.timestamp).isNull();
-                  ofy().save().entity(object);
+                  tm().insert(object);
                  return tm().getTransactionTime();
                });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
    assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
  }

-  @Test
+  @TestOfyAndSql
+  void testDisabledUpdates() throws Exception {
+    DateTime initialTime =
+        tm().transact(
+                () -> {
+                  tm().insert(new UpdateAutoTimestampTestObject());
+                  return tm().getTransactionTime();
+                });
+
+    UpdateAutoTimestampTestObject object = reload();
+    clock.advanceOneMilli();
+
+    try (UpdateAutoTimestamp.DisableAutoUpdateResource disabler =
+        new UpdateAutoTimestamp.DisableAutoUpdateResource()) {
+      DateTime secondTransactionTime =
+          tm().transact(
+                  () -> {
+                    tm().put(object);
+                    return tm().getTransactionTime();
+                  });
+      assertThat(secondTransactionTime).isGreaterThan(initialTime);
+    }
+    assertThat(reload().updateTime.timestamp).isEqualTo(initialTime);
+  }
+
+  @TestOfyAndSql
  void testResavingOverwritesOriginalTime() {
    DateTime transactionTime =
        tm().transact(
                () -> {
                  UpdateAutoTimestampTestObject object = new UpdateAutoTimestampTestObject();
                  object.updateTime = UpdateAutoTimestamp.create(DateTime.now(UTC).minusDays(1));
-                  ofy().save().entity(object);
+                  tm().insert(object);
                  return tm().getTransactionTime();
                });
-    ofy().clearSessionCache();
+    tm().clearSessionCache();
    assertThat(reload().updateTime.timestamp).isEqualTo(transactionTime);
  }
 }
@@ -633,14 +633,13 @@ public class DomainBaseSqlTest {
                          "4-COM",
                          END_OF_TIME,
                          "registrar1",
-                          createLegacyVKey(
-                              BillingEvent.OneTime.class, oneTimeBillingEvent.getId())),
+                          oneTimeBillingEvent.createVKey()),
                      GracePeriod.createForRecurring(
                          GracePeriodStatus.AUTO_RENEW,
                          "4-COM",
                          END_OF_TIME,
                          "registrar1",
-                          createLegacyVKey(BillingEvent.Recurring.class, billEvent.getId())));
+                          billEvent.createVKey()));

              jpaTm().insert(contact);
              jpaTm().insert(contact2);
@@ -875,16 +875,13 @@ public class DomainBaseTest extends EntityTestCase {
    ImmutableSet<BillEventInfo> historyIds =
        domain.getGracePeriods().stream()
            .map(
-                gp ->
-                    new BillEventInfo(
-                        gp.getRecurringBillingEvent(), gp.billingEventRecurringHistoryId,
-                        gp.getOneTimeBillingEvent(), gp.billingEventOneTimeHistoryId))
+                gp -> new BillEventInfo(gp.getRecurringBillingEvent(), gp.getOneTimeBillingEvent()))
            .collect(toImmutableSet());
    assertThat(historyIds)
        .isEqualTo(
            ImmutableSet.of(
-                new BillEventInfo(null, null, oneTimeBillKey, historyEntryKey.getId()),
-                new BillEventInfo(recurringBillKey, historyEntryKey.getId(), null, null)));
+                new BillEventInfo(null, oneTimeBillKey),
+                new BillEventInfo(recurringBillKey, null)));
  }

  static class BillEventInfo extends ImmutableObject {
@@ -895,13 +892,9 @@ public class DomainBaseTest extends EntityTestCase {

    BillEventInfo(
        VKey<BillingEvent.Recurring> billingEventRecurring,
-        Long billingEventRecurringHistoryId,
-        VKey<BillingEvent.OneTime> billingEventOneTime,
-        Long billingEventOneTimeHistoryId) {
+        VKey<BillingEvent.OneTime> billingEventOneTime) {
      this.billingEventRecurring = billingEventRecurring;
-      this.billingEventRecurringHistoryId = billingEventRecurringHistoryId;
      this.billingEventOneTime = billingEventOneTime;
-      this.billingEventOneTimeHistoryId = billingEventOneTimeHistoryId;
    }
  }
 }
@@ -63,11 +63,11 @@ public class GracePeriodTest {
    recurringKey =
        VKey.create(
            Recurring.class,
-            12345,
+            12345L,
            Key.create(
                Key.create(Key.create(DomainBase.class, "1-TEST"), HistoryEntry.class, 343L),
                Recurring.class,
-                12345));
+                12345L));
  }

  @Test
@@ -80,8 +80,6 @@ public class GracePeriodTest {
    assertThat(gracePeriod.getClientId()).isEqualTo("TheRegistrar");
    assertThat(gracePeriod.getExpirationTime()).isEqualTo(now.plusDays(1));
    assertThat(gracePeriod.hasBillingEvent()).isTrue();
-    assertThat(gracePeriod.billingEventOneTimeHistoryId).isEqualTo(12345L);
-    assertThat(gracePeriod.billingEventRecurringHistoryId).isNull();
  }

  @Test
@@ -96,8 +94,6 @@ public class GracePeriodTest {
    assertThat(gracePeriod.getClientId()).isEqualTo("TheRegistrar");
    assertThat(gracePeriod.getExpirationTime()).isEqualTo(now.plusDays(1));
    assertThat(gracePeriod.hasBillingEvent()).isTrue();
-    assertThat(gracePeriod.billingEventOneTimeHistoryId).isNull();
-    assertThat(gracePeriod.billingEventRecurringHistoryId).isEqualTo(343L);
  }

  @Test
@@ -40,7 +40,6 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import google.registry.persistence.VKey;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
@@ -86,7 +85,7 @@ public class AllocationTokenTest extends EntityTestCase {
        persistResource(
            new AllocationToken.Builder()
                .setToken("abc123Single")
-                .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+                .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
                .setDomainName("example.foo")
                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
                .setTokenType(SINGLE_USE)
@@ -125,7 +124,7 @@ public class AllocationTokenTest extends EntityTestCase {
            new AllocationToken.Builder()
                .setToken("abc123")
                .setTokenType(SINGLE_USE)
-                .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey))
+                .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey))
                .setDomainName("blahdomain.foo")
                .setCreationTimeForTest(DateTime.parse("2010-11-12T05:00:00Z"))
                .build()),
@@ -231,7 +230,7 @@ public class AllocationTokenTest extends EntityTestCase {
        new AllocationToken.Builder()
            .setToken("foobar")
            .setTokenType(TokenType.UNLIMITED_USE)
-            .setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey));
+            .setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey));
    IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, builder::build);
    assertThat(thrown)
        .hasMessageThat()
@@ -50,21 +50,6 @@ public class SignedMarkRevocationListDaoTest {
    assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
  }

-  @Test
-  void trySave_failureIsSwallowed() {
-    SignedMarkRevocationList list =
-        SignedMarkRevocationList.create(
-            fakeClock.nowUtc(), ImmutableMap.of("mark", fakeClock.nowUtc().minusHours(1)));
-    SignedMarkRevocationListDao.trySave(list);
-    SignedMarkRevocationList fromDb = SignedMarkRevocationListDao.getLatestRevision().get();
-    assertAboutImmutableObjects().that(fromDb).isEqualExceptFields(list);
-
-    // This should throw an exception, which is swallowed and nothing changed
-    SignedMarkRevocationListDao.trySave(list);
-    SignedMarkRevocationList secondFromDb = SignedMarkRevocationListDao.getLatestRevision().get();
-    assertAboutImmutableObjects().that(secondFromDb).isEqualExceptFields(fromDb);
-  }
-
  @Test
  void testRetrieval_notPresent() {
    assertThat(SignedMarkRevocationListDao.getLatestRevision().isPresent()).isFalse();
@@ -18,6 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
 import static google.registry.model.ofy.ObjectifyService.ofy;
 import static google.registry.model.smd.SignedMarkRevocationList.SHARD_SIZE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static org.joda.time.Duration.standardDays;
@@ -138,6 +139,33 @@ public class SignedMarkRevocationListTest {
        .isEqualTo(DateTime.parse("2000-01-01T00:00:00Z"));
  }

+  @Test
+  void test_getCreationTime_missingInCloudSQL() {
+    clock.setTo(DateTime.parse("2000-01-01T00:00:00Z"));
+    createSaveGetHelper(1);
+    jpaTm().transact(() -> jpaTm().delete(SignedMarkRevocationListDao.getLatestRevision().get()));
+    RuntimeException thrown =
+        assertThrows(RuntimeException.class, () -> SignedMarkRevocationList.get());
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("Signed mark revocation list in Cloud SQL is empty.");
+  }
+
+  @Test
+  void test_getCreationTime_unequalListsInDatabases() {
+    clock.setTo(DateTime.parse("2000-01-01T00:00:00Z"));
+    createSaveGetHelper(1);
+    ImmutableMap.Builder<String, DateTime> revokes = new ImmutableMap.Builder<>();
+    for (int i = 0; i < 3; i++) {
+      revokes.put(Integer.toString(i), clock.nowUtc());
+    }
+    SignedMarkRevocationListDao.trySave(
+        SignedMarkRevocationList.create(clock.nowUtc(), revokes.build()));
+    RuntimeException thrown =
+        assertThrows(RuntimeException.class, () -> SignedMarkRevocationList.get());
+    assertThat(thrown).hasMessageThat().contains("Unequal SM revocation lists detected:");
+  }
+
  @Test
  void test_isSmdRevoked_present() {
    final int rows = SHARD_SIZE + 1;
@@ -0,0 +1,36 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.model.translators;
+
+import static google.registry.model.translators.EppHistoryVKeyTranslatorFactory.kindPathToVKeyClass;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import org.junit.jupiter.api.Test;
+
+/** Unit test for {@link EppHistoryVKeyTranslatorFactory}. */
+class EppHistoryVKeyTranslatorFactoryTest {
+
+  @Test
+  void assertAllVKeyClassesHavingCreateFromOfyKeyMethod() {
+    kindPathToVKeyClass.forEach(
+        (kindPath, vKeyClass) -> {
+          try {
+            vKeyClass.getDeclaredMethod("create", com.googlecode.objectify.Key.class);
+          } catch (NoSuchMethodException e) {
+            fail("Missing static method create(com.googlecode.objectify.Key) on " + vKeyClass, e);
+          }
+        });
+  }
+}
@@ -0,0 +1,116 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.common.EntityGroupRoot.getCrossTldKey;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Parent;
+import google.registry.model.ImmutableObject;
+import google.registry.model.billing.BillingEvent;
+import google.registry.model.common.EntityGroupRoot;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.BillingVKey.BillingEventVKey;
+import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
+import google.registry.schema.replay.EntityTest.EntityForTesting;
+import google.registry.testing.AppEngineExtension;
+import google.registry.testing.DualDatabaseTest;
+import google.registry.testing.TestOfyAndSql;
+import javax.persistence.Transient;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit test for {@link BillingVKey}. */
+@DualDatabaseTest
+class BillingVKeyTest {
+  @RegisterExtension
+  final AppEngineExtension appEngine =
+      AppEngineExtension.builder()
+          .withDatastoreAndCloudSql()
+          .withOfyTestEntities(BillingVKeyTestEntity.class)
+          .withJpaUnitTestEntities(BillingVKeyTestEntity.class)
+          .build();
+
+  @TestOfyAndSql
+  void testRestoreSymmetricVKey() {
+    Key<HistoryEntry> domainHistoryKey =
+        Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L);
+
+    Key<BillingEvent.OneTime> oneTimeOfyKey =
+        Key.create(domainHistoryKey, BillingEvent.OneTime.class, 100L);
+    VKey<BillingEvent.OneTime> oneTimeVKey =
+        VKey.create(BillingEvent.OneTime.class, 100L, oneTimeOfyKey);
+
+    Key<BillingEvent.Recurring> recurringOfyKey =
+        Key.create(domainHistoryKey, BillingEvent.Recurring.class, 200L);
+    VKey<BillingEvent.Recurring> recurringVKey =
+        VKey.create(BillingEvent.Recurring.class, 200L, recurringOfyKey);
+
+    BillingVKeyTestEntity original = new BillingVKeyTestEntity(oneTimeVKey, recurringVKey);
+    tm().transact(() -> tm().insert(original));
+    BillingVKeyTestEntity persisted = tm().transact(() -> tm().load(original.createVKey()));
+
+    assertThat(persisted).isEqualTo(original);
+    assertThat(persisted.getBillingEventVKey()).isEqualTo(oneTimeVKey);
+    assertThat(persisted.getBillingRecurrenceVKey()).isEqualTo(recurringVKey);
+  }
+
+  @TestOfyAndSql
+  void testHandleNullVKeyCorrectly() {
+    BillingVKeyTestEntity original = new BillingVKeyTestEntity(null, null);
+    tm().transact(() -> tm().insert(original));
+    BillingVKeyTestEntity persisted = tm().transact(() -> tm().load(original.createVKey()));
+
+    assertThat(persisted).isEqualTo(original);
+  }
+
+  @EntityForTesting
+  @Entity
+  @javax.persistence.Entity
+  private static class BillingVKeyTestEntity extends ImmutableObject {
+    @Transient @Parent Key<EntityGroupRoot> parent = getCrossTldKey();
+
+    @Id @javax.persistence.Id String id = "id";
+
+    BillingEventVKey billingEventVKey;
+
+    BillingRecurrenceVKey billingRecurrenceVKey;
+
+    BillingVKeyTestEntity() {}
+
+    BillingVKeyTestEntity(
+        VKey<BillingEvent.OneTime> onetime, VKey<BillingEvent.Recurring> recurring) {
+      this.billingEventVKey = BillingEventVKey.create(onetime);
+      this.billingRecurrenceVKey = BillingRecurrenceVKey.create(recurring);
+    }
+
+    VKey<BillingEvent.OneTime> getBillingEventVKey() {
+      return billingEventVKey.createVKey();
+    }
+
+    VKey<BillingEvent.Recurring> getBillingRecurrenceVKey() {
+      return billingRecurrenceVKey.createVKey();
+    }
+
+    VKey<BillingVKeyTestEntity> createVKey() {
+      return VKey.create(
+          BillingVKeyTestEntity.class, id, Key.create(parent, BillingVKeyTestEntity.class, id));
+    }
+  }
+}
@@ -56,12 +56,17 @@ class DomainHistoryVKeyTest {
    TestEntity persisted = tm().transact(() -> tm().load(original.createVKey()));
    assertThat(persisted).isEqualTo(original);
    // Double check that the persisted.domainHistoryVKey is a symmetric VKey
-    assertThat(persisted.domainHistoryVKey.getKind()).isEqualTo(HistoryEntry.class);
-    assertThat(persisted.domainHistoryVKey.getOfyKey())
+    assertThat(persisted.domainHistoryVKey.createOfyKey())
        .isEqualTo(
            Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L));
-    assertThat(persisted.domainHistoryVKey.getSqlKey())
+    assertThat(persisted.domainHistoryVKey.createSqlKey())
        .isEqualTo(new DomainHistoryId("domainRepoId", 10L));
+    assertThat(persisted.domainHistoryVKey.createVKey())
+        .isEqualTo(
+            VKey.create(
+                HistoryEntry.class,
+                new DomainHistoryId("domainRepoId", 10L),
+                Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L)));
  }

  @TestOfyAndSql
@@ -69,9 +74,12 @@ class DomainHistoryVKeyTest {
    Key<HistoryEntry> ofyKey =
        Key.create(Key.create(DomainBase.class, "domainRepoId"), HistoryEntry.class, 10L);
    DomainHistoryVKey domainHistoryVKey = DomainHistoryVKey.create(ofyKey);
-    assertThat(domainHistoryVKey.getKind()).isEqualTo(HistoryEntry.class);
-    assertThat(domainHistoryVKey.getOfyKey()).isEqualTo(ofyKey);
-    assertThat(domainHistoryVKey.getSqlKey()).isEqualTo(new DomainHistoryId("domainRepoId", 10L));
+    assertThat(domainHistoryVKey.createOfyKey()).isEqualTo(ofyKey);
+    assertThat(domainHistoryVKey.createSqlKey())
+        .isEqualTo(new DomainHistoryId("domainRepoId", 10L));
+    assertThat(domainHistoryVKey.createVKey())
+        .isEqualTo(
+            VKey.create(HistoryEntry.class, new DomainHistoryId("domainRepoId", 10L), ofyKey));
  }

  @EntityForTesting
@@ -17,6 +17,8 @@ package google.registry.privileges.secretmanager;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;

+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings;
 import com.google.cloud.secretmanager.v1.SecretVersion.State;
 import google.registry.privileges.secretmanager.SecretManagerClient.NoSuchSecretResourceException;
 import google.registry.privileges.secretmanager.SecretManagerClient.SecretAlreadyExistsException;
@@ -54,11 +56,14 @@ public class SecretManagerClientTest {
  private String secretId;

  @BeforeAll
-  static void beforeAll() {
+  static void beforeAll() throws IOException {
    String environmentName = System.getProperty("test.gcp_integration.env");
    if (environmentName != null) {
      secretManagerClient =
          SecretManagerModule.provideSecretManagerClient(
+              SecretManagerServiceSettings.newBuilder()
+                  .setCredentialsProvider(() -> GoogleCredentials.getApplicationDefault())
+                  .build(),
              String.format("domain-registry-%s", environmentName),
              new Retrier(new SystemSleeper(), 1));
      isUnitTest = false;
@@ -34,7 +34,7 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;

 /** Unit tests for {@link Spec11RegistrarThreatMatchesParser}. */
-class Spec11RegistrarThreatMatchesParserTest {
+public class Spec11RegistrarThreatMatchesParserTest {

  private static final String TODAY = "2018-07-21";
  private static final String YESTERDAY = "2018-07-20";
@@ -66,6 +66,12 @@ class Spec11RegistrarThreatMatchesParserTest {
    assertThat(parser.getPreviousDateWithMatches(LocalDate.parse(TODAY))).isEmpty();
  }

+  @Test
+  void testNonexistent_returnsEmpty() throws Exception {
+    assertThat(parser.getRegistrarThreatMatches(LocalDate.parse(YESTERDAY).minusYears(1)))
+        .isEmpty();
+  }
+
  @Test
  void testFindPrevious_olderThanYesterdayFound() throws Exception {
    setupFile("spec11_fake_report_previous_day", "2018-07-14");
@@ -95,7 +101,7 @@ class Spec11RegistrarThreatMatchesParserTest {
  }

  /** The expected contents of the sample spec11 report file */
-  static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
+  public static ImmutableSet<RegistrarThreatMatches> sampleThreatMatches() throws Exception {
    return ImmutableSet.of(getMatchA(), getMatchB());
  }

@@ -27,7 +27,6 @@ import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.model.reporting.HistoryEntry;
-import google.registry.persistence.DomainHistoryVKey;
 import java.util.Collection;
 import javax.annotation.Nullable;
 import org.junit.jupiter.api.BeforeEach;
@@ -173,7 +172,7 @@ class DeleteAllocationTokensCommandTest extends CommandTestCase<DeleteAllocation
      String domainToPersist = domainName != null ? domainName : "example.foo";
      DomainBase domain = persistActiveDomain(domainToPersist);
      Key<HistoryEntry> historyEntryKey = Key.create(Key.create(domain), HistoryEntry.class, 1051L);
-      builder.setRedemptionHistoryEntry(DomainHistoryVKey.create(historyEntryKey));
+      builder.setRedemptionHistoryEntry(HistoryEntry.createVKey(historyEntryKey));
    }
    return persistResource(builder.build());
  }
@@ -39,7 +39,8 @@ import com.google.common.collect.Iterables;
 import com.google.common.io.Files;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
-import google.registry.persistence.DomainHistoryVKey;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.VKey;
 import google.registry.testing.DeterministicStringGenerator;
 import google.registry.testing.DeterministicStringGenerator.Rule;
 import google.registry.testing.FakeClock;
@@ -313,7 +314,7 @@ class GenerateAllocationTokensCommandTest extends CommandTestCase<GenerateAlloca

  private AllocationToken createToken(
      String token,
-      @Nullable DomainHistoryVKey redemptionHistoryEntry,
+      @Nullable VKey<? extends HistoryEntry> redemptionHistoryEntry,
      @Nullable String domainName) {
    AllocationToken.Builder builder =
        new AllocationToken.Builder().setToken(token).setTokenType(SINGLE_USE);
@@ -28,7 +28,7 @@ import com.google.common.collect.ImmutableList;
 import com.googlecode.objectify.Key;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.token.AllocationToken;
-import google.registry.persistence.DomainHistoryVKey;
+import google.registry.model.reporting.HistoryEntry;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.Test;

@@ -85,7 +85,7 @@ class GetAllocationTokenCommandTest extends CommandTestCase<GetAllocationTokenCo
                .setTokenType(SINGLE_USE)
                .setDomainName("fqqdn.tld")
                .setRedemptionHistoryEntry(
-                    DomainHistoryVKey.create(Key.create(createHistoryEntryForEppResource(domain))))
+                    HistoryEntry.createVKey(Key.create(createHistoryEntryForEppResource(domain))))
                .build());
    runCommand("foo");
    assertInStdout(
@@ -0,0 +1,271 @@
+// Copyright 2020 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
+import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.deleteResource;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistActiveDomain;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.reporting.Spec11ThreatMatch;
+import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
+import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
+import google.registry.tools.CommandTestCase;
+import java.io.IOException;
+import org.joda.time.DateTime;
+import org.joda.time.LocalDate;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Tests for {@link BackfillSpec11ThreatMatchesCommand}. */
+public class BackfillSpec11ThreatMatchesCommandTest
+    extends CommandTestCase<BackfillSpec11ThreatMatchesCommand> {
+
+  private static final LocalDate CURRENT_DATE = DateTime.parse("2020-11-22").toLocalDate();
+  private final Spec11RegistrarThreatMatchesParser threatMatchesParser =
+      mock(Spec11RegistrarThreatMatchesParser.class);
+
+  private DomainBase domainA;
+
+  @BeforeEach
+  void beforeEach() throws Exception {
+    createTld("com");
+    domainA = persistActiveDomain("a.com");
+    persistActiveDomain("b.com");
+    persistActiveDomain("c.com");
+    fakeClock.setTo(CURRENT_DATE.toDateTimeAtStartOfDay());
+    command.threatMatchesParser = threatMatchesParser;
+    command.clock = fakeClock;
+    when(threatMatchesParser.getRegistrarThreatMatches(any(LocalDate.class)))
+        .thenReturn(ImmutableSet.of());
+  }
+
+  @Test
+  void testSuccess_singleFile() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    assertInStdout("Backfill Spec11 results from 692 files?");
+    assertInStdout("Successfully parsed through 692 files with 3 threats.");
+    verifyExactlyThreeEntriesInDbFromLastDay();
+  }
+
+  @Test
+  void testSuccess_sameDomain_multipleDays() throws Exception {
+    // If the same domains show up on multiple days, there should be multiple entries for them
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    when(threatMatchesParser.getRegistrarThreatMatches(LocalDate.parse("2019-01-01")))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    assertInStdout("Backfill Spec11 results from 692 files?");
+    assertInStdout("Successfully parsed through 692 files with 6 threats.");
+    jpaTm()
+        .transact(
+            () -> {
+              ImmutableList<Spec11ThreatMatch> threatMatches =
+                  jpaTm().loadAll(Spec11ThreatMatch.class);
+              assertThat(threatMatches).hasSize(6);
+              assertThat(
+                      threatMatches.stream()
+                          .map(Spec11ThreatMatch::getDomainName)
+                          .collect(toImmutableSet()))
+                  .containsExactly("a.com", "b.com", "c.com");
+              assertThat(
+                      threatMatches.stream()
+                          .map(Spec11ThreatMatch::getCheckDate)
+                          .collect(toImmutableSet()))
+                  .containsExactly(CURRENT_DATE, LocalDate.parse("2019-01-01"));
+            });
+  }
+
+  @Test
+  void testSuccess_empty() throws Exception {
+    runCommandForced();
+    assertInStdout("Backfill Spec11 results from 692 files?");
+    assertInStdout("Successfully parsed through 692 files with 0 threats.");
+  }
+
+  @Test
+  void testSuccess_sameDayTwice() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    runCommandForced();
+    verifyExactlyThreeEntriesInDbFromLastDay();
+  }
+
+  @Test
+  void testSuccess_threeDomainsForDomainName() throws Exception {
+    // We should use the repo ID from the proper DomainBase object at the scan's point in time.
+    // First, domain was created at START_OF_TIME and deleted one year ago
+    DateTime now = fakeClock.nowUtc();
+    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
+
+    // Next, domain was created six months ago and deleted two months ago
+    DomainBase secondSave =
+        persistResource(
+            newDomainBase("a.com")
+                .asBuilder()
+                .setCreationTimeForTest(now.minusMonths(6))
+                .setDeletionTime(now.minusMonths(2))
+                .build());
+
+    // Lastly, domain was created one month ago and is still valid
+    DomainBase thirdSave =
+        persistResource(
+            newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
+
+    // If the scan result was from three months ago, we should use the second save
+    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
+        .thenReturn(sampleThreatMatches());
+    runCommandForced();
+    String threatMatchRepoId =
+        jpaTm()
+            .transact(
+                () ->
+                    jpaTm().loadAll(Spec11ThreatMatch.class).stream()
+                        .filter((match) -> match.getDomainName().equals("a.com"))
+                        .findFirst()
+                        .get()
+                        .getDomainRepoId());
+    assertThat(threatMatchRepoId).isNotEqualTo(domainA.getRepoId());
+    assertThat(threatMatchRepoId).isEqualTo(secondSave.getRepoId());
+    assertThat(threatMatchRepoId).isNotEqualTo(thirdSave.getRepoId());
+  }
+
+  @Test
+  void testSuccess_skipsExistingDatesWithoutOverwrite() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    Spec11ThreatMatch previous =
+        new Spec11ThreatMatch.Builder()
+            .setCheckDate(CURRENT_DATE)
+            .setDomainName("previous.tld")
+            .setDomainRepoId("1-DOMAIN")
+            .setRegistrarId("TheRegistrar")
+            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(previous));
+
+    runCommandForced();
+    ImmutableList<Spec11ThreatMatch> threatMatches =
+        jpaTm().transact(() -> jpaTm().loadAll(Spec11ThreatMatch.class));
+    assertAboutImmutableObjects()
+        .that(Iterables.getOnlyElement(threatMatches))
+        .isEqualExceptFields(previous, "id");
+  }
+
+  @Test
+  void testSuccess_overwritesExistingDatesWhenSpecified() throws Exception {
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    Spec11ThreatMatch previous =
+        new Spec11ThreatMatch.Builder()
+            .setCheckDate(CURRENT_DATE)
+            .setDomainName("previous.tld")
+            .setDomainRepoId("1-DOMAIN")
+            .setRegistrarId("TheRegistrar")
+            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
+            .build();
+    jpaTm().transact(() -> jpaTm().put(previous));
+
+    runCommandForced("--overwrite_existing_dates");
+    verifyExactlyThreeEntriesInDbFromLastDay();
+  }
+
+  @Test
+  void testFailure_oneFileFails() throws Exception {
+    // If there are any exceptions, we should fail loud and fast
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE.minusDays(1)))
+        .thenThrow(new IOException("hi"));
+    RuntimeException runtimeException =
+        assertThrows(RuntimeException.class, this::runCommandForced);
+    assertThat(runtimeException.getCause().getClass()).isEqualTo(IOException.class);
+    assertThat(runtimeException).hasCauseThat().hasMessageThat().isEqualTo("hi");
+    jpaTm().transact(() -> assertThat(jpaTm().loadAll(Spec11ThreatMatch.class)).isEmpty());
+  }
+
+  @Test
+  void testFailure_noDomainForDomainName() throws Exception {
+    deleteResource(domainA);
+    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
+        .thenReturn(sampleThreatMatches());
+    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
+        .hasMessageThat()
+        .isEqualTo("Domain name a.com had no associated DomainBase objects.");
+  }
+
+  @Test
+  void testFailure_noDomainAtTimeOfScan() throws Exception {
+    // If the domain existed at some point(s) in time but not the time of the scan, fail.
+    // First, domain was created at START_OF_TIME and deleted one year ago
+    DateTime now = fakeClock.nowUtc();
+    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
+
+    // Second, domain was created one month ago and is still valid
+    persistResource(
+        newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
+
+    // If we have a result for this domain from 3 months ago when it didn't exist, fail.
+    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
+        .thenReturn(sampleThreatMatches());
+    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
+        .hasMessageThat()
+        .isEqualTo("Could not find a DomainBase valid for a.com on day 2020-08-22.");
+  }
+
+  private void verifyExactlyThreeEntriesInDbFromLastDay() {
+    jpaTm()
+        .transact(
+            () -> {
+              ImmutableList<Spec11ThreatMatch> threatMatches =
+                  jpaTm().loadAll(Spec11ThreatMatch.class);
+              assertThat(threatMatches)
+                  .comparingElementsUsing(immutableObjectCorrespondence("id", "domainRepoId"))
+                  .containsExactly(
+                      expectedThreatMatch("TheRegistrar", "a.com"),
+                      expectedThreatMatch("NewRegistrar", "b.com"),
+                      expectedThreatMatch("NewRegistrar", "c.com"));
+            });
+  }
+
+  private Spec11ThreatMatch expectedThreatMatch(String registrarId, String domainName) {
+    return new Spec11ThreatMatch.Builder()
+        .setDomainRepoId("ignored")
+        .setDomainName(domainName)
+        .setRegistrarId(registrarId)
+        .setCheckDate(CURRENT_DATE)
+        .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
+        .build();
+  }
+}
@@ -278,8 +278,8 @@ class google.registry.model.domain.DomainHistory {
 }
 class google.registry.model.domain.GracePeriod {
  google.registry.model.domain.rgp.GracePeriodStatus type;
-  google.registry.persistence.VKey<google.registry.model.billing.BillingEvent$OneTime> billingEventOneTime;
-  google.registry.persistence.VKey<google.registry.model.billing.BillingEvent$Recurring> billingEventRecurring;
+  google.registry.persistence.BillingVKey$BillingEventVKey billingEventOneTime;
+  google.registry.persistence.BillingVKey$BillingRecurrenceVKey billingEventRecurring;
  java.lang.Long gracePeriodId;
  java.lang.String clientId;
  org.joda.time.DateTime expirationTime;
@@ -888,10 +888,17 @@ enum google.registry.model.transfer.TransferStatus {
  SERVER_APPROVED;
  SERVER_CANCELLED;
 }
-class google.registry.persistence.DomainHistoryVKey {
-  com.googlecode.objectify.Key<T> ofyKey;
-  java.lang.Class<? extends T> kind;
-  java.lang.Long domainHistoryId;
-  java.lang.Object sqlKey;
-  java.lang.String domainRepoId;
+class google.registry.persistence.BillingVKey$BillingEventVKey {
+  java.lang.Long billingId;
+  java.lang.Long historyRevisionId;
+  java.lang.String repoId;
+}
+class google.registry.persistence.BillingVKey$BillingRecurrenceVKey {
+  java.lang.Long billingId;
+  java.lang.Long historyRevisionId;
+  java.lang.String repoId;
+}
+class google.registry.persistence.DomainHistoryVKey {
+  java.lang.Long historyRevisionId;
+  java.lang.String repoId;
 }
@@ -78,3 +78,5 @@ V77__fixes_for_replay.sql
 V78__add_history_id_for_redemption_history_entry.sql
 V79__drop_foreign_keys_on_pollmessage.sql
 V80__defer_bill_event_key.sql
+V81__drop_spec11_fkeys.sql
+V82__add_columns_to_restore_symmetric_billing_vkey.sql
@@ -0,0 +1,18 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+-- Some objects haven't been fully populated in SQL yet, so don't depend on them.
+ALTER TABLE "Spec11ThreatMatch" DROP CONSTRAINT "fk_spec11_threat_match_domain_repo_id";
+ALTER TABLE "Spec11ThreatMatch" DROP CONSTRAINT "fk_spec11_threat_match_registrar_id";
+ALTER TABLE "Spec11ThreatMatch" DROP CONSTRAINT "fk_spec11_threat_match_tld";
@@ -0,0 +1,22 @@
+-- Copyright 2020 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "GracePeriod"
+    add column if not exists "billing_event_domain_repo_id" text;
+alter table "GracePeriod"
+    add column if not exists "billing_recurrence_domain_repo_id" text;
+alter table "GracePeriodHistory"
+    add column if not exists "billing_event_domain_repo_id" text;
+alter table "GracePeriodHistory"
+    add column if not exists "billing_recurrence_domain_repo_id" text;
@@ -398,8 +398,10 @@
       grace_period_id int8 not null,
        billing_event_id int8,
        billing_event_history_id int8,
+        billing_event_domain_repo_id text,
        billing_recurrence_id int8,
        billing_recurrence_history_id int8,
+        billing_recurrence_domain_repo_id text,
        registrar_id text not null,
        domain_repo_id text not null,
        expiration_time timestamptz not null,
@@ -411,8 +413,10 @@
       grace_period_history_revision_id int8 not null,
        billing_event_id int8,
        billing_event_history_id int8,
+        billing_event_domain_repo_id text,
        billing_recurrence_id int8,
        billing_recurrence_history_id int8,
+        billing_recurrence_domain_repo_id text,
        registrar_id text not null,
        domain_repo_id text not null,
        expiration_time timestamptz not null,
@@ -533,7 +533,9 @@ CREATE TABLE public."GracePeriod" (
    expiration_time timestamp with time zone NOT NULL,
    type text NOT NULL,
    billing_event_history_id bigint,
-    billing_recurrence_history_id bigint
+    billing_recurrence_history_id bigint,
+    billing_event_domain_repo_id text,
+    billing_recurrence_domain_repo_id text
 );


@@ -552,7 +554,9 @@ CREATE TABLE public."GracePeriodHistory" (
    expiration_time timestamp with time zone NOT NULL,
    type text NOT NULL,
    domain_history_revision_id bigint,
-    grace_period_id bigint NOT NULL
+    grace_period_id bigint NOT NULL,
+    billing_event_domain_repo_id text,
+    billing_recurrence_domain_repo_id text
 );


@@ -2251,30 +2255,6 @@ ALTER TABLE ONLY public."RegistrarPoc"
    ADD CONSTRAINT fk_registrar_poc_registrar_id FOREIGN KEY (registrar_id) REFERENCES public."Registrar"(registrar_id);


--
-- Name: Spec11ThreatMatch fk_spec11_threat_match_domain_repo_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public."Spec11ThreatMatch"
-    ADD CONSTRAINT fk_spec11_threat_match_domain_repo_id FOREIGN KEY (domain_repo_id) REFERENCES public."Domain"(repo_id);
-
-
--
-- Name: Spec11ThreatMatch fk_spec11_threat_match_registrar_id; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public."Spec11ThreatMatch"
-    ADD CONSTRAINT fk_spec11_threat_match_registrar_id FOREIGN KEY (registrar_id) REFERENCES public."Registrar"(registrar_id);
-
-
--
-- Name: Spec11ThreatMatch fk_spec11_threat_match_tld; Type: FK CONSTRAINT; Schema: public; Owner: -
--
-
-ALTER TABLE ONLY public."Spec11ThreatMatch"
-    ADD CONSTRAINT fk_spec11_threat_match_tld FOREIGN KEY (tld) REFERENCES public."Tld"(tld_name);
-
-
 --
 -- Name: DomainHistoryHost fka9woh3hu8gx5x0vly6bai327n; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -74,10 +74,12 @@ test {
  useJUnitPlatform()
 }

-// Sets up integration test with a registry environment. The target environment is
-// passed by the 'test.gcp_integration.env' property. Test runner must have been
-// authorized to access the corresponding GCP project, e.g., by running 'gcloud auth'
-// or placing a credential file at a well known place.
+// Sets up integration test with a registry environment. The target environment
+// is passed by the 'test.gcp_integration.env' property. Test runner must have
+// been authorized to access the corresponding GCP project, e.g., by running
+// 'gcloud auth application-default login' or by downloading a credential file
+// and assign the path to it to the GOOGLE_APPLICATION_CREDENTIALS environment
+// variable.
 //
 // A typical use case is to run tests from desktop that accesses Cloud resources. See
 // core/src/test/java/google/registry/beam/initsql/BeamJpaModuleTest.java for an example.
@@ -34,7 +34,9 @@ import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.ssl.SslHandshakeCompletionEvent;
 import io.netty.util.AttributeKey;
 import io.netty.util.concurrent.Promise;
+import java.security.cert.CertificateEncodingException;
 import java.security.cert.X509Certificate;
+import java.util.Base64;
 import java.util.function.Supplier;

 /** Handler that processes EPP protocol logic. */
@@ -52,18 +54,26 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
  /** Name of the HTTP header that stores the client certificate hash. */
  public static final String SSL_CLIENT_CERTIFICATE_HASH_FIELD = "X-SSL-Certificate";

+  /** Name of the HTTP header that stores the full client certificate. */
+  public static final String SSL_CLIENT_FULL_CERTIFICATE_FIELD = "X-SSL-Full-Certificate";
+
  /** Name of the HTTP header that stores the client IP address. */
  public static final String FORWARDED_FOR_FIELD = "X-Forwarded-For";

  /** Name of the HTTP header that indicates if the EPP session should be closed. */
  public static final String EPP_SESSION_FIELD = "Epp-Session";

+  /** Name of the HTTP header that indicates a successful login has occurred. */
+  public static final String EPP_LOGGED_IN_FIELD = "Logged-In";
+
  public static final String EPP_CONTENT_TYPE = "application/epp+xml";

  private final byte[] helloBytes;

  private String sslClientCertificateHash;
+  private X509Certificate sslClientCertificate;
  private String clientAddress;
+  private boolean isLoggedIn = false;

  public EppServiceHandler(
      String relayHost,
@@ -103,7 +113,8 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
            .addListener(
                (Promise<X509Certificate> promise) -> {
                  if (promise.isSuccess()) {
-                    sslClientCertificateHash = getCertificateHash(promise.get());
+                    sslClientCertificate = promise.get();
+                    sslClientCertificateHash = getCertificateHash(sslClientCertificate);
                    // Set the client cert hash key attribute for both this channel,
                    // used for collecting metrics on specific clients.
                    ctx.channel().attr(CLIENT_CERTIFICATE_HASH_KEY).set(sslClientCertificateHash);
@@ -132,6 +143,17 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
        .set(FORWARDED_FOR_FIELD, clientAddress)
        .set(HttpHeaderNames.CONTENT_TYPE, EPP_CONTENT_TYPE)
        .set(HttpHeaderNames.ACCEPT, EPP_CONTENT_TYPE);
+    if (!isLoggedIn) {
+      try {
+        request
+            .headers()
+            .set(
+                SSL_CLIENT_FULL_CERTIFICATE_FIELD,
+                Base64.getEncoder().encodeToString(sslClientCertificate.getEncoded()));
+      } catch (CertificateEncodingException e) {
+        throw new RuntimeException("Cannot encode client certificate", e);
+      }
+    }
    return request;
  }

@@ -141,9 +163,13 @@ public class EppServiceHandler extends HttpsRelayServiceHandler {
    checkArgument(msg instanceof HttpResponse);
    HttpResponse response = (HttpResponse) msg;
    String sessionAliveValue = response.headers().get(EPP_SESSION_FIELD);
+    String loginValue = response.headers().get(EPP_LOGGED_IN_FIELD);
    if (sessionAliveValue != null && sessionAliveValue.equals("close")) {
      promise.addListener(ChannelFutureListener.CLOSE);
    }
+    if (loginValue != null && loginValue.equals("true")) {
+      isLoggedIn = true;
+    }
    super.write(ctx, msg, promise);
  }
 }
@@ -16,6 +16,7 @@ package google.registry.proxy;

 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.networking.handler.SslServerInitializer.CLIENT_CERTIFICATE_PROMISE_KEY;
+import static google.registry.proxy.TestUtils.SAMPLE_CERT;
 import static google.registry.proxy.handler.ProxyProtocolHandler.REMOTE_ADDRESS_KEY;
 import static google.registry.util.ResourceUtils.readResourceBytes;
 import static google.registry.util.X509Utils.getCertificateHash;
@@ -25,7 +26,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.base.Throwables;
 import google.registry.proxy.handler.HttpsRelayServiceHandler.NonOkHttpResponseException;
 import google.registry.testing.FakeClock;
-import google.registry.util.SelfSignedCaCertificate;
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.channel.embedded.EmbeddedChannel;
@@ -36,6 +36,8 @@ import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.cookie.Cookie;
 import io.netty.handler.codec.http.cookie.DefaultCookie;
 import io.netty.util.concurrent.Promise;
+import java.io.ByteArrayInputStream;
+import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -96,8 +98,8 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
    return buffer;
  }

-  private FullHttpRequest makeEppHttpRequest(byte[] content, Cookie... cookies) {
-    return TestUtils.makeEppHttpRequest(
+  private FullHttpRequest makeEppHttpRequestWithCertificate(byte[] content, Cookie... cookies) {
+    return TestUtils.makeEppHttpRequestWithCertificate(
        new String(content, UTF_8),
        PROXY_CONFIG.epp.relayHost,
        PROXY_CONFIG.epp.relayPath,
@@ -120,7 +122,10 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
  @Override
  void beforeEach() throws Exception {
    testComponent = makeTestComponent(new FakeClock());
-    certificate = SelfSignedCaCertificate.create().cert();
+    CertificateFactory cf = CertificateFactory.getInstance("X.509");
+    certificate =
+        (X509Certificate)
+            cf.generateCertificate(new ByteArrayInputStream(SAMPLE_CERT.getBytes(UTF_8)));
    initializeChannel(
        ch -> {
          ch.attr(REMOTE_ADDRESS_KEY).set(CLIENT_ADDRESS);
@@ -134,13 +139,15 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
  @Test
  void testSuccess_singleFrameInboundMessage() throws Exception {
    // First inbound message is hello.
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(HELLO_BYTES));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(HELLO_BYTES));

    byte[] inputBytes = readResourceBytes(getClass(), "login.xml").read();

    // Verify inbound message is as expected.
    assertThat(channel.writeInbound(getByteBufFromContent(inputBytes))).isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes));

    // Nothing more to read.
    assertThat((Object) channel.readInbound()).isNull();
@@ -161,8 +168,10 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
                Unpooled.wrappedBuffer(
                    getByteBufFromContent(inputBytes1), getByteBufFromContent(inputBytes2))))
        .isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes1));
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes2));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes1));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes2));

    // Nothing more to read.
    assertThat((Object) channel.readInbound()).isNull();
@@ -186,11 +195,13 @@ class EppProtocolModuleTest extends ProtocolModuleTest {

    // The second frame contains the first message, and part of the second message.
    assertThat(channel.writeInbound(inputBuffer.readBytes(inputBytes2.length))).isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes1));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes1));

    // The third frame contains the rest of the second message.
    assertThat(channel.writeInbound(inputBuffer)).isTrue();
-    assertThat((FullHttpRequest) channel.readInbound()).isEqualTo(makeEppHttpRequest(inputBytes2));
+    assertThat((FullHttpRequest) channel.readInbound())
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes2));

    // Nothing more to read.
    assertThat((Object) channel.readInbound()).isNull();
@@ -252,7 +263,7 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
    byte[] inputBytes1 = readResourceBytes(getClass(), "logout.xml").read();
    assertThat(channel.writeInbound(getByteBufFromContent(inputBytes1))).isTrue();
    assertThat((FullHttpRequest) channel.readInbound())
-        .isEqualTo(makeEppHttpRequest(inputBytes1, cookie1, cookie2));
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes1, cookie1, cookie2));

    // Second outbound message change cookies.
    byte[] outputBytes2 = readResourceBytes(getClass(), "logout_response.xml").read();
@@ -267,7 +278,7 @@ class EppProtocolModuleTest extends ProtocolModuleTest {
    byte[] inputBytes2 = readResourceBytes(getClass(), "login.xml").read();
    assertThat(channel.writeInbound(getByteBufFromContent(inputBytes2))).isTrue();
    assertThat((FullHttpRequest) channel.readInbound())
-        .isEqualTo(makeEppHttpRequest(inputBytes2, cookie1, cookie2, cookie3));
+        .isEqualTo(makeEppHttpRequestWithCertificate(inputBytes2, cookie1, cookie2, cookie3));

    // Nothing more to write or read.
    assertThat((Object) channel.readOutbound()).isNull();
@@ -36,6 +36,54 @@ import io.netty.handler.codec.http.cookie.ServerCookieEncoder;
 /** Utility class for various helper methods used in testing. */
 public class TestUtils {

+  public static final String SAMPLE_CERT =
+      "-----BEGIN CERTIFICATE-----\n"
+          + "MIIDvTCCAqWgAwIBAgIJAK/PgPT0jTwRMA0GCSqGSIb3DQEBCwUAMHUxCzAJBgNV\n"
+          + "BAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxDzAN\n"
+          + "BgNVBAoMBkdvb2dsZTEdMBsGA1UECwwUZG9tYWluLXJlZ2lzdHJ5LXRlc3QxEDAO\n"
+          + "BgNVBAMMB2NsaWVudDEwHhcNMTUwODI2MTkxODA4WhcNNDMwMTExMTkxODA4WjB1\n"
+          + "MQswCQYDVQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxETAPBgNVBAcMCE5ldyBZ\n"
+          + "b3JrMQ8wDQYDVQQKDAZHb29nbGUxHTAbBgNVBAsMFGRvbWFpbi1yZWdpc3RyeS10\n"
+          + "ZXN0MRAwDgYDVQQDDAdjbGllbnQxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB\n"
+          + "CgKCAQEAvoE/IoFJyzb0dU4NFhL8FYgy+B/GnUd5aA66CMx5xKRMbEAtIgxU8TTO\n"
+          + "W+9jdTsE00Grk3Ct4KdY73CYW+6IFXL4O0K/m5S+uajh+I2UMVZJV38RAIqNxue0\n"
+          + "Egv9M4haSsCVIPcX9b+6McywfYSF1bzPb2Gb2FAQO7Jb0BjlPhPMIROCrbG40qPg\n"
+          + "LWrl33dz+O52kO+DyZEzHqI55xH6au77sMITsJe+X23lzQcMFUUm8moiOw0EKrj/\n"
+          + "GaMTZLHP46BCRoJDAPTNx55seIwgAHbKA2VVtqrvmA2XYJQA6ipdhfKRoJFy8Z8H\n"
+          + "DYsorGtazQL2HhF/5uJD25z1m5eQHQIDAQABo1AwTjAdBgNVHQ4EFgQUParEmiSR\n"
+          + "U/Oqy8hr7k+MBKhZwVkwHwYDVR0jBBgwFoAUParEmiSRU/Oqy8hr7k+MBKhZwVkw\n"
+          + "DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAojsUhF6PtZrStnHBFWNR\n"
+          + "ryzvANB8krZlYeX9Hkqn8zIVfAkpbVmL8aZQ7yj17jSpw47PQh3x5gwA9yc/SS0G\n"
+          + "E1rGuxYH02UGbua8G0+vviSQfLtskPQzK7EIR63WNhHEo/Q9umLJkZ0LguWEBf3L\n"
+          + "q8CoXv2i/RNvqVPcTNp/zCKXJZAa8wAjNRJs834AZj4k5xwyYZ3F8D5PGz+YMOmV\n"
+          + "M9Qd+NdXSC/Qn7HQzFhE8p5elBV35P8oX5dXEfn0S7zOXDenp5JvvLoggOWOcKsq\n"
+          + "KiWDQrsT+TMKmHL94/h4t7FghtQLMzY5SGYJsYTv/LG8tewrz6KRb/Wj3JNojyEw\n"
+          + "Ug==\n"
+          + "-----END CERTIFICATE-----\n";
+
+  public static final String SAMPLE_CERT_ENCODED =
+      "MIIDvTCCAqWgAwIBAgIJAK/PgPT0jTwRMA0GCSqGSIb3DQEBCwUAMHUxCzAJBgNV"
+          + "BAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazERMA8GA1UEBwwITmV3IFlvcmsxDzAN"
+          + "BgNVBAoMBkdvb2dsZTEdMBsGA1UECwwUZG9tYWluLXJlZ2lzdHJ5LXRlc3QxEDAO"
+          + "BgNVBAMMB2NsaWVudDEwHhcNMTUwODI2MTkxODA4WhcNNDMwMTExMTkxODA4WjB1"
+          + "MQswCQYDVQQGEwJVUzERMA8GA1UECAwITmV3IFlvcmsxETAPBgNVBAcMCE5ldyBZ"
+          + "b3JrMQ8wDQYDVQQKDAZHb29nbGUxHTAbBgNVBAsMFGRvbWFpbi1yZWdpc3RyeS10"
+          + "ZXN0MRAwDgYDVQQDDAdjbGllbnQxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB"
+          + "CgKCAQEAvoE/IoFJyzb0dU4NFhL8FYgy+B/GnUd5aA66CMx5xKRMbEAtIgxU8TTO"
+          + "W+9jdTsE00Grk3Ct4KdY73CYW+6IFXL4O0K/m5S+uajh+I2UMVZJV38RAIqNxue0"
+          + "Egv9M4haSsCVIPcX9b+6McywfYSF1bzPb2Gb2FAQO7Jb0BjlPhPMIROCrbG40qPg"
+          + "LWrl33dz+O52kO+DyZEzHqI55xH6au77sMITsJe+X23lzQcMFUUm8moiOw0EKrj/"
+          + "GaMTZLHP46BCRoJDAPTNx55seIwgAHbKA2VVtqrvmA2XYJQA6ipdhfKRoJFy8Z8H"
+          + "DYsorGtazQL2HhF/5uJD25z1m5eQHQIDAQABo1AwTjAdBgNVHQ4EFgQUParEmiSR"
+          + "U/Oqy8hr7k+MBKhZwVkwHwYDVR0jBBgwFoAUParEmiSRU/Oqy8hr7k+MBKhZwVkw"
+          + "DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAojsUhF6PtZrStnHBFWNR"
+          + "ryzvANB8krZlYeX9Hkqn8zIVfAkpbVmL8aZQ7yj17jSpw47PQh3x5gwA9yc/SS0G"
+          + "E1rGuxYH02UGbua8G0+vviSQfLtskPQzK7EIR63WNhHEo/Q9umLJkZ0LguWEBf3L"
+          + "q8CoXv2i/RNvqVPcTNp/zCKXJZAa8wAjNRJs834AZj4k5xwyYZ3F8D5PGz+YMOmV"
+          + "M9Qd+NdXSC/Qn7HQzFhE8p5elBV35P8oX5dXEfn0S7zOXDenp5JvvLoggOWOcKsq"
+          + "KiWDQrsT+TMKmHL94/h4t7FghtQLMzY5SGYJsYTv/LG8tewrz6KRb/Wj3JNojyEw"
+          + "Ug==";
+
  public static FullHttpRequest makeHttpPostRequest(String content, String host, String path) {
    ByteBuf buf = Unpooled.wrappedBuffer(content.getBytes(US_ASCII));
    FullHttpRequest request =
@@ -101,6 +149,21 @@ public class TestUtils {
    return request;
  }

+  public static FullHttpRequest makeEppHttpRequestWithCertificate(
+      String content,
+      String host,
+      String path,
+      String accessToken,
+      String sslClientCertificateHash,
+      String clientAddress,
+      Cookie... cookies) {
+    FullHttpRequest request =
+        makeEppHttpRequest(
+            content, host, path, accessToken, sslClientCertificateHash, clientAddress, cookies);
+    request.headers().set("X-SSL-Full-Certificate", SAMPLE_CERT_ENCODED);
+    return request;
+  }
+
  public static FullHttpResponse makeWhoisHttpResponse(String content, HttpResponseStatus status) {
    FullHttpResponse response = makeHttpResponse(content, status);
    response.headers().set("content-type", "text/plain");
@@ -16,6 +16,7 @@ package google.registry.proxy.handler;

 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.networking.handler.SslServerInitializer.CLIENT_CERTIFICATE_PROMISE_KEY;
+import static google.registry.proxy.TestUtils.SAMPLE_CERT;
 import static google.registry.proxy.TestUtils.assertHttpRequestEquivalent;
 import static google.registry.proxy.TestUtils.makeEppHttpResponse;
 import static google.registry.proxy.handler.ProxyProtocolHandler.REMOTE_ADDRESS_KEY;
@@ -43,6 +44,8 @@ import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.cookie.Cookie;
 import io.netty.handler.codec.http.cookie.DefaultCookie;
 import io.netty.util.concurrent.Promise;
+import java.io.ByteArrayInputStream;
+import java.security.cert.CertificateFactory;
 import java.security.cert.X509Certificate;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -109,9 +112,23 @@ class EppServiceHandlerTest {
        cookies);
  }

+  private FullHttpRequest makeEppHttpRequestWithCertificate(String content, Cookie... cookies) {
+    return TestUtils.makeEppHttpRequestWithCertificate(
+        content,
+        RELAY_HOST,
+        RELAY_PATH,
+        ACCESS_TOKEN,
+        getCertificateHash(clientCertificate),
+        CLIENT_ADDRESS,
+        cookies);
+  }
+
  @BeforeEach
  void beforeEach() throws Exception {
-    clientCertificate = SelfSignedCaCertificate.create().cert();
+    CertificateFactory cf = CertificateFactory.getInstance("X.509");
+    clientCertificate =
+        (X509Certificate)
+            cf.generateCertificate(new ByteArrayInputStream(SAMPLE_CERT.getBytes(UTF_8)));
    channel = setUpNewChannel(eppServiceHandler);
  }

@@ -194,7 +211,7 @@ class EppServiceHandlerTest {
    setHandshakeSuccess();
    // hello bytes should be passed to the next handler.
    FullHttpRequest helloRequest = channel.readInbound();
-    assertThat(helloRequest).isEqualTo(makeEppHttpRequest(HELLO));
+    assertThat(helloRequest).isEqualTo(makeEppHttpRequestWithCertificate(HELLO));
    // Nothing further to pass to the next handler.
    assertThat((Object) channel.readInbound()).isNull();
    assertThat(channel.isActive()).isTrue();
@@ -216,12 +233,34 @@ class EppServiceHandlerTest {
    String content = "<epp>stuff</epp>";
    channel.writeInbound(Unpooled.wrappedBuffer(content.getBytes(UTF_8)));
    FullHttpRequest request = channel.readInbound();
-    assertThat(request).isEqualTo(makeEppHttpRequest(content));
+    assertThat(request).isEqualTo(makeEppHttpRequestWithCertificate(content));
    // Nothing further to pass to the next handler.
    assertThat((Object) channel.readInbound()).isNull();
    assertThat(channel.isActive()).isTrue();
  }

+  @Test
+  void testSuccess_sendCertificateOnlyBeforeLogin() throws Exception {
+    setHandshakeSuccess();
+    // First inbound message is hello.
+    channel.readInbound();
+    String content = "<epp>stuff</epp>";
+    channel.writeInbound(Unpooled.wrappedBuffer(content.getBytes(UTF_8)));
+    FullHttpRequest request = channel.readInbound();
+    assertThat(request).isEqualTo(makeEppHttpRequestWithCertificate(content));
+    // Receive response indicating session is logged in
+    HttpResponse response = makeEppHttpResponse(content, HttpResponseStatus.OK);
+    response.headers().set("Logged-In", "true");
+    // Send another inbound message after login
+    channel.writeOutbound(response);
+    channel.writeInbound(Unpooled.wrappedBuffer(content.getBytes(UTF_8)));
+    request = channel.readInbound();
+    // Second request should not have full certificate
+    assertThat(request).isEqualTo(makeEppHttpRequest(content));
+    assertThat((Object) channel.readInbound()).isNull();
+    assertThat(channel.isActive()).isTrue();
+  }
+
  @Test
  void testSuccess_sendResponseToNextHandler() throws Exception {
    setHandshakeSuccess();
@@ -294,7 +333,8 @@ class EppServiceHandlerTest {
    String requestContent = "<epp>request</epp>";
    channel.writeInbound(Unpooled.wrappedBuffer(requestContent.getBytes(UTF_8)));
    FullHttpRequest request = channel.readInbound();
-    assertHttpRequestEquivalent(request, makeEppHttpRequest(requestContent, cookie1, cookie2));
+    assertHttpRequestEquivalent(
+        request, makeEppHttpRequestWithCertificate(requestContent, cookie1, cookie2));
    // Nothing further to pass to the next handler.
    assertThat((Object) channel.readInbound()).isNull();
    assertThat((Object) channel.readOutbound()).isNull();
@@ -317,13 +357,16 @@ class EppServiceHandlerTest {
    // First request written.
    channel.writeInbound(Unpooled.wrappedBuffer(requestContent1.getBytes(UTF_8)));
    FullHttpRequest request1 = channel.readInbound();
-    assertHttpRequestEquivalent(request1, makeEppHttpRequest(requestContent1, cookie1, cookie2));
+    assertHttpRequestEquivalent(
+        request1, makeEppHttpRequestWithCertificate(requestContent1, cookie1, cookie2));
    String responseContent2 = "<epp>response2</epp>";
    Cookie cookie3 = new DefaultCookie("name3", "value3");
    Cookie newCookie2 = new DefaultCookie("name2", "newValue");
    // Second response written.
-    channel.writeOutbound(
-        makeEppHttpResponse(responseContent2, HttpResponseStatus.OK, cookie3, newCookie2));
+    HttpResponse response =
+        makeEppHttpResponse(responseContent2, HttpResponseStatus.OK, cookie3, newCookie2);
+    response.headers().set("Logged-In", "true");
+    channel.writeOutbound(response);
    channel.readOutbound();
    String requestContent2 = "<epp>request2</epp>";
    // Second request written.
Author	SHA1	Message	Date
Shicong Huang	6e2bbd1a7e	Add BillingVKey to restore symmetric VKey in GracePeriodBase (#902 ) * Use PollMessageVKey to replace VKey<PollMessage> in DomainBase * Revert changes to DomainContent * Use BillingVKey in GracePeriodBase to restore symmetric vkey * Rebase on HEAD	2020-12-17 14:13:47 -05:00
Weimin Yu	495d7176d8	Validate SQL credentials in Secret Manager (#907 ) * Validate SQL credentials in Secret Manager Load SQL credentials from the SecretManager and compare them with the ones currently in use in Nomulus server, beam pipeline, and the registry tool. Normal operations are not affected by failures related to the SecretManager, be it IOException, insufficient permission , or wrong or missing credential. The appengine and compute engine default service accounts must be granted the permission to access the secret data. In the short term, we will grant the secretmanager.secretAccessor role to these accounts. In the long term, with the proposed privilege service, access will be granted on per-secret basis.	2020-12-16 10:57:03 -05:00
Michael Muller	d7aab524e5	Make config/presubmits.py use explicit encodings (#908 ) For some reason, our docker build image has started using a non-utf8 default encoding. Specify the encoding explicitly on python "open()" to override. Note that this might not entirely fix the build: it's possible that this problem may affect other portions of the build.	2020-12-16 10:03:32 -05:00
sarahcaseybot	c5bfe31b73	Modify SignedMarkRevocationList to throw Cloud SQL failures in unit tests (#898 ) * Modify SignedMarkRevocationList to not swallow CloudSQL failures in unittests * restore package-lock.json * Added suppressExceptionUnlessInTest() * Add a DatabaseMigrationUtils class * small changes	2020-12-15 17:34:38 -05:00
sarahcaseybot	9975bc2195	Modify proxy to pass full certificate before login (#896 ) * Modify proxy to pass full certificate until partner is logged in * refactor tests * revert package-lock.json * add sample cert string to tests	2020-12-15 16:36:39 -05:00
gbrodman	cb16a7649f	Add a scrap command to backfill Spec11 threats (#897 ) This parses through all pre-existing Spec11 files in GCS (starting at 2019-01-01 which is basically when the new format started) and maps them to the new Spec11ThreatMatch objects. Because the old format stored domain names only and the new format stores names + repo IDs, we need to retrieve the DomainBase objects from the point in time of the scan (failing if they don't exist). Because the same domains appear multiple times (we estimate a total of 100k+ entries but only 1-2k unique domains) we cache the DomainBase objects that we retrieve from Datastore.	2020-12-15 16:18:27 -05:00
Michael Muller	d7e2b24468	Allow disabling UpdateAutoTimestamp updates (#906 ) * Allow disabling UpdateAutoTimestamp updates Allow us to disable timestamp updates within a try-with-resources block for a given thread. This functionality will be needed for transaction replays both to and from datastore. As part of this, also upgrade the UpdateAutoTimestampTest to a DualDatabaseTest so we can verify that the functionality works both on Datastore and Cloud SQL.	2020-12-15 10:34:52 -05:00