Use the replica jpaTm in FKI and EppResource cache methods (#1503)

The cached methods are only used in situations where we don't really
care about being 100% synchronously up to date (e.g. whois), and they're
not used frequently anyway, so it's safe to use the replica in these
locations.

core/build.gradle

@@ -796,6 +796,16 @@ if (environment == 'alpha') {
               mainClass: 'google.registry.beam.rde.RdePipeline',
               metaData : 'google/registry/beam/rde_pipeline_metadata.json'
           ],
+      validateDatastore  :
+          [
+              mainClass: 'google.registry.beam.comparedb.ValidateDatastorePipeline',
+              metaData: 'google/registry/beam/validate_datastore_pipeline_metadata.json'
+          ],
+      validateSql  :
+          [
+              mainClass: 'google.registry.beam.comparedb.ValidateSqlPipeline',
+              metaData: 'google/registry/beam/validate_sql_pipeline_metadata.json'
+          ],
   ]
   project.tasks.create("stageBeamPipelines") {
     doLast {

core/src/main/java/google/registry/model/EppResource.java

@@ -21,6 +21,7 @@ import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
@@ -380,13 +381,13 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
         @Override
         public EppResource load(VKey<? extends EppResource> key) {
-          return tm().doTransactionless(() -> tm().loadByKey(key));
+          return replicaTm().doTransactionless(() -> replicaTm().loadByKey(key));
         }
 
         @Override
         public Map<VKey<? extends EppResource>, EppResource> loadAll(
             Iterable<? extends VKey<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> tm().loadByKeys(keys));
+          return replicaTm().doTransactionless(() -> replicaTm().loadByKeys(keys));
         }
       };
 

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -21,6 +21,7 @@ import static google.registry.config.RegistryConfig.getEppResourceCachingDuratio
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 import static google.registry.util.TypeUtils.instantiate;
@@ -51,6 +52,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.replay.DatastoreOnlyEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.util.NonFinalForTesting;
 import java.util.Collection;
 import java.util.Comparator;
@@ -198,7 +200,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
       Class<E> clazz, Collection<String> foreignKeys, final DateTime now) {
-    return loadIndexesFromStore(clazz, foreignKeys, true).entrySet().stream()
+    return loadIndexesFromStore(clazz, foreignKeys, true, false).entrySet().stream()
         .filter(e -> now.isBefore(e.getValue().getDeletionTime()))
         .collect(entriesToImmutableMap());
   }
@@ -217,7 +219,10 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    */
   private static <E extends EppResource>
       ImmutableMap<String, ForeignKeyIndex<E>> loadIndexesFromStore(
-          Class<E> clazz, Collection<String> foreignKeys, boolean inTransaction) {
+          Class<E> clazz,
+          Collection<String> foreignKeys,
+          boolean inTransaction,
+          boolean useReplicaJpaTm) {
     if (tm().isOfy()) {
       Class<ForeignKeyIndex<E>> fkiClass = mapToFkiClass(clazz);
       return ImmutableMap.copyOf(
@@ -226,17 +231,18 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               : tm().doTransactionless(() -> auditedOfy().load().type(fkiClass).ids(foreignKeys)));
     } else {
       String property = RESOURCE_CLASS_TO_FKI_PROPERTY.get(clazz);
+      JpaTransactionManager jpaTmToUse = useReplicaJpaTm ? replicaJpaTm() : jpaTm();
       ImmutableList<ForeignKeyIndex<E>> indexes =
-          tm().transact(
-                  () ->
-                      jpaTm()
-                          .criteriaQuery(
-                              CriteriaQueryBuilder.create(clazz)
-                                  .whereFieldIsIn(property, foreignKeys)
-                                  .build())
-                          .getResultStream()
-                          .map(e -> ForeignKeyIndex.create(e, e.getDeletionTime()))
-                          .collect(toImmutableList()));
+          jpaTmToUse.transact(
+              () ->
+                  jpaTmToUse
+                      .criteriaQuery(
+                          CriteriaQueryBuilder.create(clazz)
+                              .whereFieldIsIn(property, foreignKeys)
+                              .build())
+                      .getResultStream()
+                      .map(e -> ForeignKeyIndex.create(e, e.getDeletionTime()))
+                      .collect(toImmutableList()));
       // We need to find and return the entities with the maximum deletionTime for each foreign key.
       return Multimaps.index(indexes, ForeignKeyIndex::getForeignKey).asMap().entrySet().stream()
           .map(
@@ -260,7 +266,8 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               loadIndexesFromStore(
                       RESOURCE_CLASS_TO_FKI_CLASS.inverse().get(key.getKind()),
                       ImmutableSet.of(foreignKey),
-                      false)
+                      false,
+                      true)
                   .get(foreignKey));
         }
 
@@ -276,7 +283,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               Streams.stream(keys).map(v -> v.getSqlKey().toString()).collect(toImmutableSet());
           ImmutableSet<VKey<ForeignKeyIndex<?>>> typedKeys = ImmutableSet.copyOf(keys);
           ImmutableMap<String, ? extends ForeignKeyIndex<? extends EppResource>> existingFkis =
-              loadIndexesFromStore(resourceClass, foreignKeys, false);
+              loadIndexesFromStore(resourceClass, foreignKeys, false, true);
           // ofy omits keys that don't have values in Datastore, so re-add them in
           // here with Optional.empty() values.
           return Maps.asMap(
@@ -336,7 +343,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     // Safe to cast VKey<FKI<E>> to VKey<FKI<?>>
     @SuppressWarnings("unchecked")
     ImmutableList<VKey<ForeignKeyIndex<?>>> fkiVKeys =
-        Streams.stream(foreignKeys)
+        foreignKeys.stream()
             .map(fk -> (VKey<ForeignKeyIndex<?>>) VKey.create(fkiClass, fk))
             .collect(toImmutableList());
     try {

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -14,8 +14,8 @@
 
 package google.registry.persistence.transaction;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.appengine.api.utils.SystemProperty;
@@ -47,6 +47,10 @@ public final class TransactionManagerFactory {
   private static Supplier<JpaTransactionManager> jpaTm =
       Suppliers.memoize(TransactionManagerFactory::createJpaTransactionManager);
 
+  @NonFinalForTesting
+  private static Supplier<JpaTransactionManager> replicaJpaTm =
+      Suppliers.memoize(TransactionManagerFactory::createReplicaJpaTransactionManager);
+
   private static boolean onBeam = false;
 
   private TransactionManagerFactory() {}
@@ -61,6 +65,14 @@ public final class TransactionManagerFactory {
     }
   }
 
+  private static JpaTransactionManager createReplicaJpaTransactionManager() {
+    if (isInAppEngine()) {
+      return DaggerPersistenceComponent.create().readOnlyReplicaJpaTransactionManager();
+    } else {
+      return DummyJpaTransactionManager.create();
+    }
+  }
+
   private static DatastoreTransactionManager createTransactionManager() {
     return new DatastoreTransactionManager(null);
   }
@@ -108,6 +120,21 @@ public final class TransactionManagerFactory {
     return jpaTm.get();
   }
 
+  /** Returns a read-only {@link JpaTransactionManager} instance if configured. */
+  public static JpaTransactionManager replicaJpaTm() {
+    return replicaJpaTm.get();
+  }
+
+  /**
+   * Returns a {@link TransactionManager} that uses a replica database if one exists.
+   *
+   * <p>In Datastore mode, this is unchanged from the regular transaction manager. In SQL mode,
+   * however, this will be a reference to the read-only replica database if one is configured.
+   */
+  public static TransactionManager replicaTm() {
+    return tm().isOfy() ? tm() : replicaJpaTm();
+  }
+
   /** Returns {@link DatastoreTransactionManager} instance. */
   @VisibleForTesting
   public static DatastoreTransactionManager ofyTm() {
@@ -116,7 +143,7 @@ public final class TransactionManagerFactory {
 
   /** Sets the return of {@link #jpaTm()} to the given instance of {@link JpaTransactionManager}. */
   public static void setJpaTm(Supplier<JpaTransactionManager> jpaTmSupplier) {
-    checkNotNull(jpaTmSupplier, "jpaTmSupplier");
+    checkArgumentNotNull(jpaTmSupplier, "jpaTmSupplier");
     checkState(
         RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
             || RegistryToolEnvironment.get() != null,
@@ -124,13 +151,23 @@ public final class TransactionManagerFactory {
     jpaTm = Suppliers.memoize(jpaTmSupplier::get);
   }
 
+  /** Sets the value of {@link #replicaJpaTm()} to the given {@link JpaTransactionManager}. */
+  public static void setReplicaJpaTm(Supplier<JpaTransactionManager> replicaJpaTmSupplier) {
+    checkArgumentNotNull(replicaJpaTmSupplier, "replicaJpaTmSupplier");
+    checkState(
+        RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
+            || RegistryToolEnvironment.get() != null,
+        "setReplicaJpaTm() should only be called by tools and tests.");
+    replicaJpaTm = Suppliers.memoize(replicaJpaTmSupplier::get);
+  }
+
   /**
    * Makes {@link #jpaTm()} return the {@link JpaTransactionManager} instance provided by {@code
    * jpaTmSupplier} from now on. This method should only be called by an implementor of {@link
    * org.apache.beam.sdk.harness.JvmInitializer}.
    */
   public static void setJpaTmOnBeamWorker(Supplier<JpaTransactionManager> jpaTmSupplier) {
-    checkNotNull(jpaTmSupplier, "jpaTmSupplier");
+    checkArgumentNotNull(jpaTmSupplier, "jpaTmSupplier");
     jpaTm = Suppliers.memoize(jpaTmSupplier::get);
     onBeam = true;
   }

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -15,14 +15,8 @@
 package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
-import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
-import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
-import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.tools.javascrap.HardDeleteHostCommand;
-import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
-import google.registry.tools.javascrap.RemoveIpAddressCommand;
-import google.registry.tools.javascrap.ResaveAllTldsCommand;
 
 /** Container class to create and run remote commands against a Datastore instance. */
 public final class RegistryTool {
@@ -36,8 +30,6 @@ public final class RegistryTool {
   public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
       new ImmutableMap.Builder<String, Class<? extends Command>>()
           .put("ack_poll_messages", AckPollMessagesCommand.class)
-          .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
-          .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
@@ -57,7 +49,6 @@ public final class RegistryTool {
           .put("curl", CurlCommand.class)
           .put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class)
           .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class)
-          .put("delete_contact_by_roid", DeleteContactByRoidCommand.class)
           .put("delete_domain", DeleteDomainCommand.class)
           .put("delete_host", DeleteHostCommand.class)
           .put("delete_premium_list", DeletePremiumListCommand.class)
@@ -107,12 +98,9 @@ public final class RegistryTool {
           .put("login", LoginCommand.class)
           .put("logout", LogoutCommand.class)
           .put("pending_escrow", PendingEscrowCommand.class)
-          .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)
-          .put("remove_ip_address", RemoveIpAddressCommand.class)
           .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class)
           .put("renew_domain", RenewDomainCommand.class)
-          .put("resave_all_tlds", ResaveAllTldsCommand.class)
           .put("resave_entities", ResaveEntitiesCommand.class)
           .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
           .put("resave_epp_resource", ResaveEppResourceCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -42,9 +42,7 @@ import google.registry.request.Modules.URLFetchServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
-import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
-import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.tools.javascrap.HardDeleteHostCommand;
 import google.registry.util.UtilsModule;
 import google.registry.whois.NonCachingWhoisModule;
@@ -90,8 +88,6 @@ import javax.inject.Singleton;
 interface RegistryToolComponent {
   void inject(AckPollMessagesCommand command);
 
-  void inject(BackfillRegistryLocksCommand command);
-
   void inject(CheckDomainClaimsCommand command);
 
   void inject(CheckDomainCommand command);
@@ -112,8 +108,6 @@ interface RegistryToolComponent {
 
   void inject(CreateTldCommand command);
 
-  void inject(DeleteContactByRoidCommand command);
-
   void inject(EncryptEscrowDepositCommand command);
 
   void inject(EnqueuePollMessageCommand command);

core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java

@@ -1,157 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
-import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.RegistryLock;
-import google.registry.model.reporting.HistoryEntry;
-import google.registry.model.reporting.HistoryEntryDao;
-import google.registry.model.tld.RegistryLockDao;
-import google.registry.persistence.VKey;
-import google.registry.tools.CommandWithRemoteApi;
-import google.registry.tools.ConfirmingCommand;
-import google.registry.util.Clock;
-import google.registry.util.StringGenerator;
-import java.util.Comparator;
-import java.util.List;
-import javax.inject.Inject;
-import javax.inject.Named;
-import org.joda.time.DateTime;
-
-/**
- * Scrap tool to backfill {@link RegistryLock}s for domains previously locked.
- *
- * <p>This will save new objects for all existing domains that are locked but don't have any
- * corresponding lock objects already in the database.
- */
-@Parameters(
-    separators = " =",
-    commandDescription =
-        "Backfills RegistryLock objects for specified domain resource IDs that are locked but don't"
-            + " already have a corresponding RegistryLock object.")
-public class BackfillRegistryLocksCommand extends ConfirmingCommand
-    implements CommandWithRemoteApi {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final int VERIFICATION_CODE_LENGTH = 32;
-
-  @Parameter(
-      names = {"--domain_roids"},
-      description = "Comma-separated list of domain roids to check")
-  protected List<String> roids;
-
-  // Inject here so that we can create the command automatically for tests
-  @Inject Clock clock;
-
-  @Inject
-  @Config("registryAdminClientId")
-  String registryAdminClientId;
-
-  @Inject
-  @Named("base58StringGenerator")
-  StringGenerator stringGenerator;
-
-  private ImmutableList<DomainBase> lockedDomains;
-
-  @Override
-  protected String prompt() {
-    checkArgument(
-        roids != null && !roids.isEmpty(), "Must provide non-empty domain_roids argument");
-    lockedDomains =
-        jpaTm().transact(() -> getLockedDomainsWithoutLocks(jpaTm().getTransactionTime()));
-    ImmutableList<String> lockedDomainNames =
-        lockedDomains.stream().map(DomainBase::getDomainName).collect(toImmutableList());
-    return String.format(
-        "Locked domains for which there does not exist a RegistryLock object: %s",
-        lockedDomainNames);
-  }
-
-  @Override
-  protected String execute() {
-    ImmutableSet.Builder<String> failedDomainsBuilder = new ImmutableSet.Builder<>();
-    jpaTm()
-        .transact(
-            () -> {
-              for (DomainBase domainBase : lockedDomains) {
-                try {
-                  RegistryLockDao.save(
-                      new RegistryLock.Builder()
-                          .isSuperuser(true)
-                          .setRegistrarId(registryAdminClientId)
-                          .setRepoId(domainBase.getRepoId())
-                          .setDomainName(domainBase.getDomainName())
-                          .setLockCompletionTime(
-                              getLockCompletionTimestamp(domainBase, jpaTm().getTransactionTime()))
-                          .setVerificationCode(
-                              stringGenerator.createString(VERIFICATION_CODE_LENGTH))
-                          .build());
-                } catch (Throwable t) {
-                  logger.atSevere().withCause(t).log(
-                      "Error when creating lock object for domain '%s'.",
-                      domainBase.getDomainName());
-                  failedDomainsBuilder.add(domainBase.getDomainName());
-                }
-              }
-            });
-    ImmutableSet<String> failedDomains = failedDomainsBuilder.build();
-    if (failedDomains.isEmpty()) {
-      return String.format(
-          "Successfully created lock objects for %d domains.", lockedDomains.size());
-    } else {
-      return String.format(
-          "Successfully created lock objects for %d domains. We failed to create locks "
-              + "for the following domains: %s",
-          lockedDomains.size() - failedDomains.size(), failedDomains);
-    }
-  }
-
-  private DateTime getLockCompletionTimestamp(DomainBase domainBase, DateTime now) {
-    // Best-effort, if a domain was URS-locked we should use that time
-    // If we can't find that, return now.
-    return HistoryEntryDao.loadHistoryObjectsForResource(domainBase.createVKey()).stream()
-        // sort by modification time descending so we get the most recent one if it was locked twice
-        .sorted(Comparator.comparing(HistoryEntry::getModificationTime).reversed())
-        .filter(entry -> "Uniform Rapid Suspension".equals(entry.getReason()))
-        .findFirst()
-        .map(HistoryEntry::getModificationTime)
-        .orElse(now);
-  }
-
-  private ImmutableList<DomainBase> getLockedDomainsWithoutLocks(DateTime now) {
-    ImmutableList<VKey<DomainBase>> domainKeys =
-        roids.stream().map(roid -> VKey.create(DomainBase.class, roid)).collect(toImmutableList());
-    ImmutableCollection<DomainBase> domains =
-        transactIfJpaTm(() -> tm().loadByKeys(domainKeys)).values();
-    return domains.stream()
-        .filter(d -> d.getDeletionTime().isAfter(now))
-        .filter(d -> d.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES))
-        .filter(d -> !RegistryLockDao.getMostRecentByRepoId(d.getRepoId()).isPresent())
-        .collect(toImmutableList());
-  }
-}

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -1,223 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.ImmutableSet;
-import google.registry.beam.spec11.ThreatMatch;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.reporting.Spec11ThreatMatch;
-import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
-import google.registry.model.reporting.Spec11ThreatMatchDao;
-import google.registry.persistence.transaction.QueryComposer;
-import google.registry.reporting.spec11.RegistrarThreatMatches;
-import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
-import google.registry.tools.CommandWithRemoteApi;
-import google.registry.tools.ConfirmingCommand;
-import google.registry.util.Clock;
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.function.Function;
-import javax.inject.Inject;
-import org.joda.time.LocalDate;
-
-/**
- * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
- *
- * <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
- * rough estimate of when we started using this format) and convert those RegistrarThreatMatches
- * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
- *
- * <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
- * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
- * or if the domain objects were not active at the time of the scan.
- */
-@Parameters(
-    commandDescription =
-        "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
-            + "Cloud SQL database.")
-public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
-    implements CommandWithRemoteApi {
-
-  private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
-
-  @Parameter(
-      names = {"-o", "--overwrite_existing_dates"},
-      description =
-          "Whether the command will overwrite data that already exists for dates that exist in the "
-              + "GCS bucket. Defaults to false.")
-  private boolean overrideExistingDates;
-
-  @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
-  // Inject the clock for testing purposes
-  @Inject Clock clock;
-
-  @Override
-  protected String prompt() {
-    return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
-  }
-
-  @Override
-  protected String execute() {
-    ImmutableList<LocalDate> dates = getDatesToBackfill();
-    ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
-        new ImmutableListMultimap.Builder<>();
-    for (LocalDate date : dates) {
-      try {
-        // It's OK if the file doesn't exist for a particular date; the result will be empty.
-        threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
-      } catch (IOException e) {
-        throw new RuntimeException(
-            String.format("Error parsing through file with date %s.", date), e);
-      }
-    }
-    ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
-        threatMatchesBuilder.build();
-    // Look up all possible DomainBases for these domain names, any of which can be in the past
-    ImmutableListMultimap<String, DomainBase> domainsByDomainName =
-        getDomainsByDomainName(threatMatches);
-
-    // For each date, convert all threat matches with the proper domain repo ID
-    int totalNumThreats = 0;
-    for (LocalDate date : threatMatches.keySet()) {
-      ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
-      for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
-        rtm.threatMatches().stream()
-            .map(
-                threatMatch ->
-                    threatMatchToCloudSqlObject(
-                        threatMatch, date, rtm.clientId(), domainsByDomainName))
-            .forEach(spec11ThreatsBuilder::add);
-      }
-      ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
-      jpaTm()
-          .transact(
-              () -> {
-                Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
-                jpaTm().putAll(spec11Threats);
-              });
-      totalNumThreats += spec11Threats.size();
-    }
-    return String.format(
-        "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
-  }
-
-  /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
-  private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
-      ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
-    return threatMatchesByDate.values().stream()
-        .map(RegistrarThreatMatches::threatMatches)
-        .flatMap(ImmutableList::stream)
-        .map(ThreatMatch::fullyQualifiedDomainName)
-        .distinct()
-        .collect(
-            flatteningToImmutableListMultimap(
-                Function.identity(),
-                (domainName) -> {
-                  ImmutableList<DomainBase> domains = loadDomainsForFqdn(domainName);
-                  checkState(
-                      !domains.isEmpty(),
-                      "Domain name %s had no associated DomainBase objects.",
-                      domainName);
-                  return domains.stream()
-                      .sorted(Comparator.comparing(DomainBase::getCreationTime).reversed());
-                }));
-  }
-
-  /** Loads in all {@link DomainBase} objects for a given FQDN. */
-  private ImmutableList<DomainBase> loadDomainsForFqdn(String fullyQualifiedDomainName) {
-    return transactIfJpaTm(
-        () ->
-            tm().createQueryComposer(DomainBase.class)
-                .where(
-                    "fullyQualifiedDomainName",
-                    QueryComposer.Comparator.EQ,
-                    fullyQualifiedDomainName)
-                .list());
-  }
-
-  /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
-  private Spec11ThreatMatch threatMatchToCloudSqlObject(
-      ThreatMatch threatMatch,
-      LocalDate date,
-      String registrarId,
-      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
-    DomainBase domain =
-        findDomainAsOfDateOrThrow(
-            threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
-    return new Spec11ThreatMatch.Builder()
-        .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
-        .setCheckDate(date)
-        .setRegistrarId(registrarId)
-        .setDomainName(threatMatch.fullyQualifiedDomainName())
-        .setDomainRepoId(domain.getRepoId())
-        .build();
-  }
-
-  /** Returns the DomainBase object as of the particular date, which is likely in the past. */
-  private DomainBase findDomainAsOfDateOrThrow(
-      String domainName,
-      LocalDate date,
-      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
-    ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
-    for (DomainBase domain : domains) {
-      // We only know the date (not datetime) of the threat scan, so we approximate
-      LocalDate creationDate = domain.getCreationTime().toLocalDate();
-      LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
-      if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
-        return domain;
-      }
-    }
-    throw new IllegalStateException(
-        String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
-  }
-
-  /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
-  private ImmutableList<LocalDate> getDatesToBackfill() {
-    ImmutableSet<LocalDate> datesToSkip =
-        overrideExistingDates ? ImmutableSet.of() : getExistingDates();
-    ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
-    LocalDate endDate = clock.nowUtc().toLocalDate();
-    for (LocalDate currentDate = START_DATE;
-        !currentDate.isAfter(endDate);
-        currentDate = currentDate.plusDays(1)) {
-      if (!datesToSkip.contains(currentDate)) {
-        result.add(currentDate);
-      }
-    }
-    return result.build();
-  }
-
-  private ImmutableSet<LocalDate> getExistingDates() {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .query(
-                        "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
-                    .getResultStream()
-                    .collect(toImmutableSet()));
-  }
-}

core/src/main/java/google/registry/tools/javascrap/DeleteContactByRoidCommand.java

@@ -1,115 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.Verify.verify;
-import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
-import google.registry.model.contact.ContactResource;
-import google.registry.model.eppcommon.StatusValue;
-import google.registry.model.index.EppResourceIndex;
-import google.registry.model.index.ForeignKeyIndex;
-import google.registry.tools.CommandWithRemoteApi;
-import google.registry.tools.ConfirmingCommand;
-import google.registry.util.SystemClock;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * Deletes a {@link google.registry.model.contact.ContactResource} by its ROID.
- *
- * <p>This is a short-term tool for race condition clean up while the bug is being fixed.
- */
-@Parameters(separators = " =", commandDescription = "Delete a contact by its ROID.")
-public class DeleteContactByRoidCommand extends ConfirmingCommand implements CommandWithRemoteApi {
-
-  @Parameter(names = "--roid", description = "The roid of the contact to be deleted.")
-  String roid;
-
-  @Parameter(
-      names = "--contact_id",
-      description = "The user provided contactId, for verification purpose.")
-  String contactId;
-
-  ImmutableList<Key<?>> toDelete;
-
-  @Override
-  protected void init() {
-    System.out.printf("Deleting %s, which refers to %s.\n", roid, contactId);
-    tm().transact(
-            () -> {
-              Key<ContactResource> targetKey = Key.create(ContactResource.class, roid);
-              ContactResource targetContact = auditedOfy().load().key(targetKey).now();
-              verify(
-                  Objects.equals(targetContact.getContactId(), contactId),
-                  "contactId does not match.");
-              verify(
-                  Objects.equals(targetContact.getStatusValues(), ImmutableSet.of(StatusValue.OK)));
-              System.out.println("Target contact has the expected contactId");
-              String canonicalResource =
-                  ForeignKeyIndex.load(ContactResource.class, contactId, new SystemClock().nowUtc())
-                      .getResourceKey()
-                      .getOfyKey()
-                      .getName();
-              verify(!Objects.equals(canonicalResource, roid), "Contact still in ForeignKeyIndex.");
-              System.out.printf(
-                  "It is safe to delete %s, since the contactId is mapped to a different entry in"
-                      + " the Foreign key index (%s).\n\n",
-                  roid, canonicalResource);
-
-              List<Object> ancestors =
-                  auditedOfy().load().ancestor(Key.create(ContactResource.class, roid)).list();
-
-              System.out.println("Ancestor query returns: ");
-              for (Object entity : ancestors) {
-                System.out.println(Key.create(entity));
-              }
-
-              ImmutableSet<String> deletetableKinds =
-                  ImmutableSet.of("HistoryEntry", "ContactResource");
-              toDelete =
-                  ancestors.stream()
-                      .map(Key::create)
-                      .filter(key -> deletetableKinds.contains(key.getKind()))
-                      .collect(ImmutableList.toImmutableList());
-
-              EppResourceIndex eppResourceIndex =
-                  auditedOfy().load().entity(EppResourceIndex.create(targetKey)).now();
-              verify(eppResourceIndex.getKey().equals(targetKey), "Wrong EppResource Index loaded");
-              System.out.printf("\n\nEppResourceIndex found (%s).\n", Key.create(eppResourceIndex));
-
-              toDelete =
-                  new ImmutableList.Builder<Key<?>>()
-                      .addAll(toDelete)
-                      .add(Key.create(eppResourceIndex))
-                      .build();
-
-              System.out.printf("\n\nAbout to delete %s entities:\n", toDelete.size());
-              toDelete.forEach(System.out::println);
-            });
-  }
-
-  @Override
-  protected String execute() {
-    tm().transact(() -> auditedOfy().delete().keys(toDelete).now());
-    return "Done";
-  }
-}

core/src/main/java/google/registry/tools/javascrap/PopulateNullRegistrarFieldsCommand.java

@@ -1,70 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.MoreObjects.firstNonNull;
-
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableList;
-import google.registry.model.registrar.Registrar;
-import google.registry.model.registrar.RegistrarAddress;
-import google.registry.tools.MutatingCommand;
-import java.util.Objects;
-
-/**
- * Scrap tool to update Registrars with null registrarName or localizedAddress fields.
- *
- * <p>This sets a null registrarName to the key name, and null localizedAddress fields to fake data.
- */
-@Parameters(
-  separators = " =",
-  commandDescription = "Populate previously null required registrar fields."
-)
-public class PopulateNullRegistrarFieldsCommand extends MutatingCommand {
-
-  @Override
-  protected void init() {
-    for (Registrar registrar : Registrar.loadAll()) {
-      Registrar.Builder changeBuilder = registrar.asBuilder();
-      changeBuilder.setRegistrarName(
-          firstNonNull(registrar.getRegistrarName(), registrar.getRegistrarId()));
-
-      RegistrarAddress address = registrar.getLocalizedAddress();
-      if (address == null) {
-        changeBuilder.setLocalizedAddress(
-            new RegistrarAddress.Builder()
-                .setCity("Fakington")
-                .setCountryCode("US")
-                .setState("FL")
-                .setZip("12345")
-                .setStreet(ImmutableList.of("123 Fake Street"))
-                .build());
-      } else {
-        changeBuilder.setLocalizedAddress(
-            new RegistrarAddress.Builder()
-                .setCity(firstNonNull(address.getCity(), "Fakington"))
-                .setCountryCode(firstNonNull(address.getCountryCode(), "US"))
-                .setState(firstNonNull(address.getState(), "FL"))
-                .setZip(firstNonNull(address.getZip(), "12345"))
-                .setStreet(firstNonNull(address.getStreet(), ImmutableList.of("123 Fake Street")))
-                .build());
-      }
-      Registrar changedRegistrar = changeBuilder.build();
-      if (!Objects.equals(registrar, changedRegistrar)) {
-        stageEntityChange(registrar, changedRegistrar);
-      }
-    }
-  }
-}

core/src/main/java/google/registry/tools/javascrap/RemoveIpAddressCommand.java

@@ -1,88 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.template.soy.data.SoyMapData;
-import google.registry.model.host.HostResource;
-import google.registry.persistence.VKey;
-import google.registry.tools.MutatingEppToolCommand;
-import google.registry.tools.params.PathParameter;
-import google.registry.tools.soy.RemoveIpAddressSoyInfo;
-import java.net.Inet6Address;
-import java.net.InetAddress;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-/**
- * Command to remove external IP Addresses from HostResources identified by text file listing
- * resource ids, one per line.
- *
- * <p>Written for b/23757755 so we can clean up records with IP addresses that should always be
- * resolved by hostname.
- *
- * <p>The JSON file should contain a list of objects each of which has a "roid" attribute.
- */
-@Parameters(separators = " =", commandDescription = "Remove all IP Addresses.")
-public class RemoveIpAddressCommand extends MutatingEppToolCommand {
-  public static String registrarId = "CharlestonRoad";
-
-  @Parameter(names = "--roids_file",
-             description = "Text file containing a list of HostResource roids to remove",
-             required = true,
-             validateWith = PathParameter.InputFile.class)
-  private Path roidsFilePath;
-
-  @Override
-  protected void initMutatingEppToolCommand() throws Exception {
-    List<String> roids = Files.readAllLines(roidsFilePath, UTF_8);
-
-    for (String roid : roids) {
-      // Look up the HostResource from its roid.
-      Optional<HostResource> host =
-          transactIfJpaTm(() -> tm().loadByKeyIfPresent(VKey.create(HostResource.class, roid)));
-      if (!host.isPresent()) {
-        System.err.printf("Record for %s not found.\n", roid);
-        continue;
-      }
-
-      ArrayList<SoyMapData> ipAddresses = new ArrayList<>();
-      for (InetAddress address : host.get().getInetAddresses()) {
-        SoyMapData dataMap = new SoyMapData(
-            "address", address.getHostAddress(),
-            "version", address instanceof Inet6Address ? "v6" : "v4");
-        ipAddresses.add(dataMap);
-      }
-
-      // Build and execute the EPP command.
-      setSoyTemplate(
-          RemoveIpAddressSoyInfo.getInstance(), RemoveIpAddressSoyInfo.REMOVE_IP_ADDRESS);
-      addSoyRecord(
-          registrarId,
-          new SoyMapData(
-              "name", host.get().getHostName(),
-              "ipAddresses", ipAddresses,
-              "requestedByRegistrar", registrarId));
-    }
-  }
-}

core/src/main/java/google/registry/tools/javascrap/ResaveAllTldsCommand.java

@@ -1,30 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-
-import com.beust.jcommander.Parameters;
-import google.registry.model.tld.Registry;
-import google.registry.tools.CommandWithRemoteApi;
-
-/** Scrap command to resave all Registry entities. */
-@Parameters(commandDescription = "Resave all TLDs")
-public class ResaveAllTldsCommand implements CommandWithRemoteApi {
-  @Override
-  public void run() throws Exception {
-    tm().transact(() -> tm().putAll(tm().loadAllOf(Registry.class)));
-  }
-}

core/src/main/resources/google/registry/beam/validate_datastore_pipeline_metadata.json

@@ -0,0 +1,42 @@
+{
+  "name": "Validate Datastore with Cloud SQL",
+  "description": "An Apache Beam batch pipeline that compares Datastore with the primary Cloud SQL database.",
+  "parameters": [
+    {
+      "name": "registryEnvironment",
+      "label": "The Registry environment.",
+      "helpText": "The Registry environment.",
+      "is_optional": false,
+      "regexes": [
+        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
+      ]
+    },
+    {
+      "name": "isolationOverride",
+      "label": "The desired SQL transaction isolation level.",
+      "helpText": "The desired SQL transaction isolation level.",
+      "is_optional": true,
+      "regexes": [
+        "^[0-9A-Z_]+$"
+      ]
+    },
+    {
+      "name": "sqlSnapshotId",
+      "label": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
+      "helpText": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
+      "is_optional": true
+    },
+    {
+      "name": "latestCommitLogTimestamp",
+      "label": "Nomulus CommitLog start time",
+      "helpText": "The latest entity update time allowed for inclusion in validation, in ISO8601 format.",
+      "is_optional": false
+    },
+    {
+      "name": "comparisonStartTimestamp",
+      "label": "Only entities updated at or after this time are included for validation.",
+      "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
+      "is_optional": true
+    }
+  ]
+}

core/src/main/resources/google/registry/beam/validate_sql_pipeline_metadata.json

@@ -0,0 +1,21 @@
+{
+  "name": "Validate Cloud SQL with Datastore being primary",
+  "description": "An Apache Beam batch pipeline that compares Cloud SQL with the primary Datastore.",
+  "parameters": [
+    {
+      "name": "registryEnvironment",
+      "label": "The Registry environment.",
+      "helpText": "The Registry environment.",
+      "is_optional": false,
+      "regexes": [
+        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
+      ]
+    },
+    {
+      "name": "comparisonStartTimestamp",
+      "label": "Only entities updated at or after this time are included for validation.",
+      "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
+      "is_optional": true
+    }
+  ]
+}

core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java

@@ -217,11 +217,14 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft
     JpaTransactionManagerImpl txnManager = new JpaTransactionManagerImpl(emf, clock);
     cachedTm = TransactionManagerFactory.jpaTm();
     TransactionManagerFactory.setJpaTm(Suppliers.ofInstance(txnManager));
+    TransactionManagerFactory.setReplicaJpaTm(
+        Suppliers.ofInstance(new ReplicaSimulatingJpaTransactionManager(txnManager)));
   }
 
   @Override
   public void afterEach(ExtensionContext context) {
     TransactionManagerFactory.setJpaTm(Suppliers.ofInstance(cachedTm));
+    TransactionManagerFactory.setReplicaJpaTm(Suppliers.ofInstance(cachedTm));
     // Even though we didn't set this, reset it to make sure no other tests are affected
     JpaTransactionManagerImpl.removeReplaySqlToDsOverrideForTest();
     cachedTm = null;

core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java

@@ -91,9 +91,15 @@ public class ReplicaSimulatingJpaTransactionManager implements JpaTransactionMan
 
   @Override
   public <T> T transact(Supplier<T> work) {
+    if (delegate.inTransaction()) {
+      return work.get();
+    }
     return delegate.transact(
         () -> {
-          delegate.getEntityManager().createQuery("SET TRANSACTION READ ONLY").executeUpdate();
+          delegate
+              .getEntityManager()
+              .createNativeQuery("SET TRANSACTION READ ONLY")
+              .executeUpdate();
           return work.get();
         });
   }

core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java

@@ -1,173 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
-import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.DatabaseHelper.persistActiveDomain;
-import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
-import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
-import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId;
-import static google.registry.testing.SqlHelper.getMostRecentVerifiedRegistryLockByRepoId;
-import static google.registry.testing.SqlHelper.getRegistryLocksByRegistrarId;
-import static google.registry.testing.SqlHelper.saveRegistryLock;
-import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.truth.Truth8;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.DomainHistory;
-import google.registry.model.domain.RegistryLock;
-import google.registry.model.registrar.Registrar;
-import google.registry.model.reporting.HistoryEntry;
-import google.registry.testing.DeterministicStringGenerator;
-import google.registry.testing.DualDatabaseTest;
-import google.registry.testing.TestOfyAndSql;
-import google.registry.tools.CommandTestCase;
-import google.registry.util.StringGenerator.Alphabets;
-import java.util.Optional;
-import org.joda.time.DateTime;
-import org.joda.time.Duration;
-import org.junit.jupiter.api.BeforeEach;
-
-/** Unit tests for {@link BackfillRegistryLocksCommand}. */
-@DualDatabaseTest
-class BackfillRegistryLocksCommandTest extends CommandTestCase<BackfillRegistryLocksCommand> {
-
-  @BeforeEach
-  void beforeEach() {
-    persistNewRegistrar("adminreg", "Admin Registrar", Registrar.Type.REAL, 693L);
-    createTld("tld");
-    command.registryAdminClientId = "adminreg";
-    command.clock = fakeClock;
-    command.stringGenerator = new DeterministicStringGenerator(Alphabets.BASE_58);
-  }
-
-  @TestOfyAndSql
-  void testSimpleBackfill() throws Exception {
-    DomainBase domain = persistLockedDomain("example.tld");
-    Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty();
-
-    runCommandForced("--domain_roids", domain.getRepoId());
-
-    Optional<RegistryLock> lockOptional = getMostRecentRegistryLockByRepoId(domain.getRepoId());
-    Truth8.assertThat(lockOptional).isPresent();
-    Truth8.assertThat(lockOptional.get().getLockCompletionTime()).isPresent();
-  }
-
-  @TestOfyAndSql
-  void testBackfill_onlyLockedDomains() throws Exception {
-    DomainBase neverLockedDomain = persistActiveDomain("neverlocked.tld");
-    DomainBase previouslyLockedDomain = persistLockedDomain("unlocked.tld");
-    persistResource(previouslyLockedDomain.asBuilder().setStatusValues(ImmutableSet.of()).build());
-    DomainBase lockedDomain = persistLockedDomain("locked.tld");
-
-    runCommandForced(
-        "--domain_roids",
-        String.format(
-            "%s,%s,%s",
-            neverLockedDomain.getRepoId(),
-            previouslyLockedDomain.getRepoId(),
-            lockedDomain.getRepoId()));
-
-    ImmutableList<RegistryLock> locks = getRegistryLocksByRegistrarId("adminreg");
-    assertThat(locks).hasSize(1);
-    assertThat(Iterables.getOnlyElement(locks).getDomainName()).isEqualTo("locked.tld");
-  }
-
-  @TestOfyAndSql
-  void testBackfill_skipsDeletedDomains() throws Exception {
-    DomainBase domain = persistDeletedDomain("example.tld", fakeClock.nowUtc());
-    persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
-    fakeClock.advanceBy(Duration.standardSeconds(1));
-    runCommandForced("--domain_roids", domain.getRepoId());
-    Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty();
-  }
-
-  @TestOfyAndSql
-  void testBackfill_skipsDomains_ifLockAlreadyExists() throws Exception {
-    DomainBase domain = persistLockedDomain("example.tld");
-
-    RegistryLock previousLock =
-        saveRegistryLock(
-            new RegistryLock.Builder()
-                .isSuperuser(true)
-                .setRegistrarId("adminreg")
-                .setRepoId(domain.getRepoId())
-                .setDomainName(domain.getDomainName())
-                .setLockCompletionTime(fakeClock.nowUtc())
-                .setVerificationCode(command.stringGenerator.createString(32))
-                .build());
-
-    fakeClock.advanceBy(Duration.standardDays(1));
-    runCommandForced("--domain_roids", domain.getRepoId());
-
-    assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId()).get().getLockCompletionTime())
-        .isEqualTo(previousLock.getLockCompletionTime());
-  }
-
-  @TestOfyAndSql
-  void testBackfill_usesUrsTime_ifExists() throws Exception {
-    DateTime ursTime = fakeClock.nowUtc();
-    DomainBase ursDomain = persistLockedDomain("urs.tld");
-    persistResource(
-        new DomainHistory.Builder()
-            .setBySuperuser(true)
-            .setRegistrarId("adminreg")
-            .setModificationTime(ursTime)
-            .setDomain(ursDomain)
-            .setReason("Uniform Rapid Suspension")
-            .setType(HistoryEntry.Type.DOMAIN_UPDATE)
-            .setRequestedByRegistrar(false)
-            .build());
-    DomainBase nonUrsDomain = persistLockedDomain("nonurs.tld");
-    persistResource(
-        new DomainHistory.Builder()
-            .setBySuperuser(true)
-            .setRegistrarId("adminreg")
-            .setDomain(nonUrsDomain)
-            .setType(HistoryEntry.Type.DOMAIN_UPDATE)
-            .setRequestedByRegistrar(false)
-            .setModificationTime(ursTime)
-            .build());
-
-    fakeClock.advanceBy(Duration.standardDays(10));
-    runCommandForced(
-        "--domain_roids", String.format("%s,%s", ursDomain.getRepoId(), nonUrsDomain.getRepoId()));
-
-    RegistryLock ursLock = getMostRecentVerifiedRegistryLockByRepoId(ursDomain.getRepoId()).get();
-    assertThat(ursLock.getLockCompletionTime()).hasValue(ursTime);
-    RegistryLock nonUrsLock =
-        getMostRecentVerifiedRegistryLockByRepoId(nonUrsDomain.getRepoId()).get();
-    assertThat(nonUrsLock.getLockCompletionTime()).hasValue(fakeClock.nowUtc());
-  }
-
-  @TestOfyAndSql
-  void testFailure_mustProvideDomainRoids() {
-    assertThat(assertThrows(IllegalArgumentException.class, this::runCommandForced))
-        .hasMessageThat()
-        .isEqualTo("Must provide non-empty domain_roids argument");
-  }
-
-  private static DomainBase persistLockedDomain(String domainName) {
-    DomainBase domain = persistActiveDomain(domainName);
-    return persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
-  }
-}

core/src/test/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommandTest.java

@@ -1,274 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
-import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
-import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.DatabaseHelper.deleteResource;
-import static google.registry.testing.DatabaseHelper.insertInDb;
-import static google.registry.testing.DatabaseHelper.newDomainBase;
-import static google.registry.testing.DatabaseHelper.persistActiveDomain;
-import static google.registry.testing.DatabaseHelper.persistResource;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.reporting.Spec11ThreatMatch;
-import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
-import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
-import google.registry.testing.DualDatabaseTest;
-import google.registry.testing.TestOfyAndSql;
-import google.registry.tools.CommandTestCase;
-import java.io.IOException;
-import org.joda.time.DateTime;
-import org.joda.time.LocalDate;
-import org.junit.jupiter.api.BeforeEach;
-
-/** Tests for {@link BackfillSpec11ThreatMatchesCommand}. */
-@DualDatabaseTest
-public class BackfillSpec11ThreatMatchesCommandTest
-    extends CommandTestCase<BackfillSpec11ThreatMatchesCommand> {
-
-  private static final LocalDate CURRENT_DATE = DateTime.parse("2020-11-22").toLocalDate();
-  private final Spec11RegistrarThreatMatchesParser threatMatchesParser =
-      mock(Spec11RegistrarThreatMatchesParser.class);
-
-  private DomainBase domainA;
-
-  @BeforeEach
-  void beforeEach() throws Exception {
-    createTld("com");
-    domainA = persistActiveDomain("a.com");
-    persistActiveDomain("b.com");
-    persistActiveDomain("c.com");
-    fakeClock.setTo(CURRENT_DATE.toDateTimeAtStartOfDay());
-    command.threatMatchesParser = threatMatchesParser;
-    command.clock = fakeClock;
-    when(threatMatchesParser.getRegistrarThreatMatches(any(LocalDate.class)))
-        .thenReturn(ImmutableSet.of());
-  }
-
-  @TestOfyAndSql
-  void testSuccess_singleFile() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    assertInStdout("Backfill Spec11 results from 692 files?");
-    assertInStdout("Successfully parsed through 692 files with 3 threats.");
-    verifyExactlyThreeEntriesInDbFromLastDay();
-  }
-
-  @TestOfyAndSql
-  void testSuccess_sameDomain_multipleDays() throws Exception {
-    // If the same domains show up on multiple days, there should be multiple entries for them
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    when(threatMatchesParser.getRegistrarThreatMatches(LocalDate.parse("2019-01-01")))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    assertInStdout("Backfill Spec11 results from 692 files?");
-    assertInStdout("Successfully parsed through 692 files with 6 threats.");
-    jpaTm()
-        .transact(
-            () -> {
-              ImmutableList<Spec11ThreatMatch> threatMatches =
-                  jpaTm().loadAllOf(Spec11ThreatMatch.class);
-              assertThat(threatMatches).hasSize(6);
-              assertThat(
-                      threatMatches.stream()
-                          .map(Spec11ThreatMatch::getDomainName)
-                          .collect(toImmutableSet()))
-                  .containsExactly("a.com", "b.com", "c.com");
-              assertThat(
-                      threatMatches.stream()
-                          .map(Spec11ThreatMatch::getCheckDate)
-                          .collect(toImmutableSet()))
-                  .containsExactly(CURRENT_DATE, LocalDate.parse("2019-01-01"));
-            });
-  }
-
-  @TestOfyAndSql
-  void testSuccess_empty() throws Exception {
-    runCommandForced();
-    assertInStdout("Backfill Spec11 results from 692 files?");
-    assertInStdout("Successfully parsed through 692 files with 0 threats.");
-  }
-
-  @TestOfyAndSql
-  void testSuccess_sameDayTwice() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    runCommandForced();
-    verifyExactlyThreeEntriesInDbFromLastDay();
-  }
-
-  @TestOfyAndSql
-  void testSuccess_threeDomainsForDomainName() throws Exception {
-    // We should use the repo ID from the proper DomainBase object at the scan's point in time.
-    // First, domain was created at START_OF_TIME and deleted one year ago
-    DateTime now = fakeClock.nowUtc();
-    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
-
-    // Next, domain was created six months ago and deleted two months ago
-    DomainBase secondSave =
-        persistResource(
-            newDomainBase("a.com")
-                .asBuilder()
-                .setCreationTimeForTest(now.minusMonths(6))
-                .setDeletionTime(now.minusMonths(2))
-                .build());
-
-    // Lastly, domain was created one month ago and is still valid
-    DomainBase thirdSave =
-        persistResource(
-            newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
-
-    // If the scan result was from three months ago, we should use the second save
-    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    String threatMatchRepoId =
-        jpaTm()
-            .transact(
-                () ->
-                    jpaTm().loadAllOf(Spec11ThreatMatch.class).stream()
-                        .filter((match) -> match.getDomainName().equals("a.com"))
-                        .findFirst()
-                        .get()
-                        .getDomainRepoId());
-    assertThat(threatMatchRepoId).isNotEqualTo(domainA.getRepoId());
-    assertThat(threatMatchRepoId).isEqualTo(secondSave.getRepoId());
-    assertThat(threatMatchRepoId).isNotEqualTo(thirdSave.getRepoId());
-  }
-
-  @TestOfyAndSql
-  void testSuccess_skipsExistingDatesWithoutOverwrite() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    Spec11ThreatMatch previous =
-        new Spec11ThreatMatch.Builder()
-            .setCheckDate(CURRENT_DATE)
-            .setDomainName("previous.tld")
-            .setDomainRepoId("1-DOMAIN")
-            .setRegistrarId("TheRegistrar")
-            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
-            .build();
-    insertInDb(previous);
-
-    runCommandForced();
-    ImmutableList<Spec11ThreatMatch> threatMatches =
-        jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class));
-    assertAboutImmutableObjects()
-        .that(Iterables.getOnlyElement(threatMatches))
-        .isEqualExceptFields(previous, "id");
-  }
-
-  @TestOfyAndSql
-  void testSuccess_overwritesExistingDatesWhenSpecified() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    Spec11ThreatMatch previous =
-        new Spec11ThreatMatch.Builder()
-            .setCheckDate(CURRENT_DATE)
-            .setDomainName("previous.tld")
-            .setDomainRepoId("1-DOMAIN")
-            .setRegistrarId("TheRegistrar")
-            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
-            .build();
-    insertInDb(previous);
-
-    runCommandForced("--overwrite_existing_dates");
-    verifyExactlyThreeEntriesInDbFromLastDay();
-  }
-
-  @TestOfyAndSql
-  void testFailure_oneFileFails() throws Exception {
-    // If there are any exceptions, we should fail loud and fast
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE.minusDays(1)))
-        .thenThrow(new IOException("hi"));
-    RuntimeException runtimeException =
-        assertThrows(RuntimeException.class, this::runCommandForced);
-    assertThat(runtimeException.getCause().getClass()).isEqualTo(IOException.class);
-    assertThat(runtimeException).hasCauseThat().hasMessageThat().isEqualTo("hi");
-    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class))).isEmpty();
-  }
-
-  @TestOfyAndSql
-  void testFailure_noDomainForDomainName() throws Exception {
-    deleteResource(domainA);
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
-        .hasMessageThat()
-        .isEqualTo("Domain name a.com had no associated DomainBase objects.");
-  }
-
-  @TestOfyAndSql
-  void testFailure_noDomainAtTimeOfScan() throws Exception {
-    // If the domain existed at some point(s) in time but not the time of the scan, fail.
-    // First, domain was created at START_OF_TIME and deleted one year ago
-    DateTime now = fakeClock.nowUtc();
-    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
-
-    // Second, domain was created one month ago and is still valid
-    persistResource(
-        newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
-
-    // If we have a result for this domain from 3 months ago when it didn't exist, fail.
-    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
-        .thenReturn(sampleThreatMatches());
-    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
-        .hasMessageThat()
-        .isEqualTo("Could not find a DomainBase valid for a.com on day 2020-08-22.");
-  }
-
-  private void verifyExactlyThreeEntriesInDbFromLastDay() {
-    jpaTm()
-        .transact(
-            () -> {
-              ImmutableList<Spec11ThreatMatch> threatMatches =
-                  jpaTm().loadAllOf(Spec11ThreatMatch.class);
-              assertThat(threatMatches)
-                  .comparingElementsUsing(immutableObjectCorrespondence("id", "domainRepoId"))
-                  .containsExactly(
-                      expectedThreatMatch("TheRegistrar", "a.com"),
-                      expectedThreatMatch("NewRegistrar", "b.com"),
-                      expectedThreatMatch("NewRegistrar", "c.com"));
-            });
-  }
-
-  private Spec11ThreatMatch expectedThreatMatch(String registrarId, String domainName) {
-    return new Spec11ThreatMatch.Builder()
-        .setDomainRepoId("ignored")
-        .setDomainName(domainName)
-        .setRegistrarId(registrarId)
-        .setCheckDate(CURRENT_DATE)
-        .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
-        .build();
-  }
-}

release/cloudbuild-nomulus.yaml

@@ -96,7 +96,11 @@ steps:
       google.registry.beam.invoicing.InvoicingPipeline \
       google/registry/beam/invoicing_pipeline_metadata.json \
       google.registry.beam.rde.RdePipeline \
-      google/registry/beam/rde_pipeline_metadata.json
+      google/registry/beam/rde_pipeline_metadata.json \
+      google.registry.beam.comparedb.ValidateDatastorePipeline \
+      google/registry/beam/validate_datastore_pipeline_metadata.json \
+      google.registry.beam.comparedb.ValidateSqlPipeline \
+      google/registry/beam/validate_sql_pipeline_metadata.json
 # Tentatively build and publish Cloud SQL schema jar here, before schema release
 # process is finalized. Also publish nomulus:core jars that are needed for
 # server/schema compatibility tests.