Make EscrowDepositEncryptor work with BRDA deposits (#1512)

Also make it possible to specify a revision number.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1512)
<!-- Reviewable:end -->

core/build.gradle

@@ -708,6 +708,9 @@ createToolTask(
 createToolTask(
     'validateSqlPipeline', 'google.registry.beam.comparedb.ValidateSqlPipeline')
 
+createToolTask(
+    'validateDatastorePipeline', 'google.registry.beam.comparedb.ValidateDatastorePipeline')
+
 
 createToolTask(
     'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
@@ -793,6 +796,16 @@ if (environment == 'alpha') {
               mainClass: 'google.registry.beam.rde.RdePipeline',
               metaData : 'google/registry/beam/rde_pipeline_metadata.json'
           ],
+      validateDatastore  :
+          [
+              mainClass: 'google.registry.beam.comparedb.ValidateDatastorePipeline',
+              metaData: 'google/registry/beam/validate_datastore_pipeline_metadata.json'
+          ],
+      validateSql  :
+          [
+              mainClass: 'google.registry.beam.comparedb.ValidateSqlPipeline',
+              metaData: 'google/registry/beam/validate_sql_pipeline_metadata.json'
+          ],
   ]
   project.tasks.create("stageBeamPipelines") {
     doLast {

core/src/main/java/google/registry/backup/CommitLogCheckpointAction.java

@@ -17,7 +17,7 @@ package google.registry.backup;
 import static google.registry.backup.ExportCommitLogDiffAction.LOWER_CHECKPOINT_TIME_PARAM;
 import static google.registry.backup.ExportCommitLogDiffAction.UPPER_CHECKPOINT_TIME_PARAM;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 
 import com.google.common.collect.ImmutableMultimap;
@@ -67,7 +67,8 @@ public final class CommitLogCheckpointAction implements Runnable {
     final CommitLogCheckpoint checkpoint = strategy.computeCheckpoint();
     logger.atInfo().log(
         "Generated candidate checkpoint for time: %s", checkpoint.getCheckpointTime());
-    tm().transact(
+    ofyTm()
+        .transact(
             () -> {
               DateTime lastWrittenTime = CommitLogCheckpointRoot.loadRoot().getLastWrittenTime();
               if (isBeforeOrAt(checkpoint.getCheckpointTime(), lastWrittenTime)) {

core/src/main/java/google/registry/backup/DeleteOldCommitLogsAction.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static google.registry.mapreduce.MapreduceRunner.PARAM_DRY_RUN;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static java.lang.Boolean.FALSE;
 import static java.lang.Boolean.TRUE;
 
@@ -288,7 +288,8 @@ public final class DeleteOldCommitLogsAction implements Runnable {
       }
 
       DeletionResult deletionResult =
-          tm().transactNew(
+          ofyTm()
+              .transactNew(
                   () -> {
                     CommitLogManifest manifest = auditedOfy().load().key(manifestKey).now();
                     // It is possible that the same manifestKey was run twice, if a shard had to be

core/src/main/java/google/registry/beam/comparedb/LatestDatastoreSnapshotFinder.java

@@ -53,11 +53,11 @@ public class LatestDatastoreSnapshotFinder {
   }
 
   /**
-   * Finds information of the most recent Datastore snapshot, including the GCS folder of the
-   * exported data files and the start and stop times of the export. The folder of the CommitLogs is
-   * also included in the return.
+   * Finds information of the most recent Datastore snapshot that ends strictly before {@code
+   * exportEndTimeUpperBound}, including the GCS folder of the exported data files and the start and
+   * stop times of the export. The folder of the CommitLogs is also included in the return.
    */
-  public DatastoreSnapshotInfo getSnapshotInfo() {
+  public DatastoreSnapshotInfo getSnapshotInfo(Instant exportEndTimeUpperBound) {
     String bucketName = RegistryConfig.getDatastoreBackupsBucket().substring("gs://".length());
     /**
      * Find the bucket-relative path to the overall metadata file of the last Datastore export.
@@ -65,7 +65,8 @@ public class LatestDatastoreSnapshotFinder {
      * return value is like
      * "2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata".
      */
-    Optional<String> metaFilePathOptional = findMostRecentExportMetadataFile(bucketName, 2);
+    Optional<String> metaFilePathOptional =
+        findNewestExportMetadataFileBeforeTime(bucketName, exportEndTimeUpperBound, 2);
     if (!metaFilePathOptional.isPresent()) {
       throw new NoSuchElementException("No exports found over the past 2 days.");
     }
@@ -85,8 +86,9 @@ public class LatestDatastoreSnapshotFinder {
   }
 
   /**
-   * Finds the bucket-relative path of the overall export metadata file, in the given bucket,
-   * searching back up to {@code lookBackDays} days, including today.
+   * Finds the latest Datastore export that ends strictly before {@code endTimeUpperBound} and
+   * returns the bucket-relative path of the overall export metadata file, in the given bucket. The
+   * search goes back for up to {@code lookBackDays} days in time, including today.
    *
    * <p>The overall export metadata file is the last file created during a Datastore export. All
    * data has been exported by the creation time of this file. The name of this file, like that of
@@ -95,7 +97,8 @@ public class LatestDatastoreSnapshotFinder {
    * <p>An example return value: {@code
    * 2021-11-19T06:00:00_76493/2021-11-19T06:00:00_76493.overall_export_metadata}.
    */
-  private Optional<String> findMostRecentExportMetadataFile(String bucketName, int lookBackDays) {
+  private Optional<String> findNewestExportMetadataFileBeforeTime(
+      String bucketName, Instant endTimeUpperBound, int lookBackDays) {
     DateTime today = clock.nowUtc();
     for (int day = 0; day < lookBackDays; day++) {
       String dateString = today.minusDays(day).toString("yyyy-MM-dd");
@@ -107,7 +110,11 @@ public class LatestDatastoreSnapshotFinder {
                 .sorted(Comparator.<String>naturalOrder().reversed())
                 .findFirst();
         if (metaFilePath.isPresent()) {
-          return metaFilePath;
+          BlobInfo blobInfo = gcsUtils.getBlobInfo(BlobId.of(bucketName, metaFilePath.get()));
+          Instant exportEndTime = new Instant(blobInfo.getCreateTime());
+          if (exportEndTime.isBefore(endTimeUpperBound)) {
+            return metaFilePath;
+          }
         }
       } catch (IOException ioe) {
         throw new RuntimeException(ioe);

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipeline.java

@@ -0,0 +1,83 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.beam.common.RegistryPipelineWorkerInitializer;
+import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
+import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import java.util.Optional;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.joda.time.DateTime;
+
+/**
+ * Validates the asynchronous data replication process from Cloud SQL (primary) to Datastore
+ * (secondary).
+ *
+ * <p>This pipeline simply compares the snapshots provided by an invoker, which is responsible for
+ * obtaining two consistent snapshots for the same point of time.
+ */
+// TODO(weiminyu): Implement the invoker action in a followup PR.
+@DeleteAfterMigration
+public class ValidateDatastorePipeline {
+
+  private final ValidateDatastorePipelineOptions options;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
+
+  public ValidateDatastorePipeline(
+      ValidateDatastorePipelineOptions options,
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
+    this.options = options;
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
+  }
+
+  void run(Pipeline pipeline) {
+    DateTime latestCommitLogTime = DateTime.parse(options.getLatestCommitLogTimestamp());
+    DatastoreSnapshotInfo mostRecentExport =
+        datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
+
+    ValidateSqlPipeline.setupPipeline(
+        pipeline,
+        Optional.ofNullable(options.getSqlSnapshotId()),
+        mostRecentExport,
+        latestCommitLogTime,
+        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
+
+    pipeline.run();
+  }
+
+  public static void main(String[] args) {
+    ValidateDatastorePipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args)
+            .withValidation()
+            .as(ValidateDatastorePipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
+
+    // Defensively set important options.
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ);
+    options.setJpaTransactionManagerType(JpaTransactionManagerType.BULK_QUERY);
+
+    // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
+    new RegistryPipelineWorkerInitializer().beforeProcessing(options);
+
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+    new ValidateDatastorePipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
+  }
+}

core/src/main/java/google/registry/beam/comparedb/ValidateDatastorePipelineOptions.java

@@ -0,0 +1,39 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.comparedb;
+
+import google.registry.model.annotations.DeleteAfterMigration;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.options.Description;
+import org.apache.beam.sdk.options.Validation;
+
+/** BEAM pipeline options for {@link ValidateDatastorePipelineOptions}. */
+@DeleteAfterMigration
+public interface ValidateDatastorePipelineOptions extends ValidateSqlPipelineOptions {
+
+  @Description(
+      "The id of the SQL snapshot to be compared with Datastore. "
+          + "If null, the current state of the SQL database is used.")
+  @Nullable
+  String getSqlSnapshotId();
+
+  void setSqlSnapshotId(String snapshotId);
+
+  @Description("The latest CommitLogs to load, in ISO8601 format.")
+  @Validation.Required
+  String getLatestCommitLogTimestamp();
+
+  void setLatestCommitLogTimestamp(String commitLogEndTimestamp);
+}

core/src/main/java/google/registry/beam/comparedb/ValidateSqlPipeline.java

@@ -26,6 +26,9 @@ import google.registry.beam.common.RegistryPipelineWorkerInitializer;
 import google.registry.beam.comparedb.LatestDatastoreSnapshotFinder.DatastoreSnapshotInfo;
 import google.registry.beam.comparedb.ValidateSqlUtils.CompareSqlEntity;
 import google.registry.model.annotations.DeleteAfterMigration;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
+import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import google.registry.model.common.DatabaseMigrationStateSchedule.ReplayDirection;
 import google.registry.model.domain.DomainBase;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.replay.SqlEntity;
@@ -35,6 +38,7 @@ import google.registry.persistence.PersistenceModule.JpaTransactionManagerType;
 import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
 import google.registry.persistence.transaction.TransactionManagerFactory;
 import google.registry.util.RequestStatusChecker;
+import google.registry.util.SystemClock;
 import java.io.Serializable;
 import java.util.Optional;
 import org.apache.beam.sdk.Pipeline;
@@ -76,21 +80,16 @@ public class ValidateSqlPipeline {
       java.time.Duration.ofSeconds(30);
 
   private final ValidateSqlPipelineOptions options;
-  private final DatastoreSnapshotInfo mostRecentExport;
+  private final LatestDatastoreSnapshotFinder datastoreSnapshotFinder;
 
   public ValidateSqlPipeline(
-      ValidateSqlPipelineOptions options, DatastoreSnapshotInfo mostRecentExport) {
+      ValidateSqlPipelineOptions options, LatestDatastoreSnapshotFinder datastoreSnapshotFinder) {
     this.options = options;
-    this.mostRecentExport = mostRecentExport;
-  }
-
-  void run() {
-    run(Pipeline.create(options));
+    this.datastoreSnapshotFinder = datastoreSnapshotFinder;
   }
 
   @VisibleForTesting
   void run(Pipeline pipeline) {
-    // TODO(weiminyu): ensure migration stage is DATASTORE_PRIMARY or DATASTORE_PRIMARY_READ_ONLY
     Optional<Lock> lock = acquireCommitLogReplayLock();
     if (lock.isPresent()) {
       logger.atInfo().log("Acquired CommitLog Replay lock.");
@@ -101,6 +100,8 @@ public class ValidateSqlPipeline {
     try {
       DateTime latestCommitLogTime =
           TransactionManagerFactory.jpaTm().transact(() -> SqlReplayCheckpoint.get());
+      DatastoreSnapshotInfo mostRecentExport =
+          datastoreSnapshotFinder.getSnapshotInfo(latestCommitLogTime.toInstant());
       Preconditions.checkState(
           latestCommitLogTime.isAfter(mostRecentExport.exportInterval().getEnd()),
           "Cannot recreate Datastore snapshot since target time is in the middle of an export.");
@@ -109,7 +110,16 @@ public class ValidateSqlPipeline {
         lock.ifPresent(Lock::releaseSql);
         lock = Optional.empty();
 
-        setupPipeline(pipeline, Optional.of(databaseSnapshot.getSnapshotId()), latestCommitLogTime);
+        logger.atInfo().log(
+            "Starting comparison with export at %s and latestCommitLogTime at %s",
+            mostRecentExport.exportDir(), latestCommitLogTime);
+
+        setupPipeline(
+            pipeline,
+            Optional.of(databaseSnapshot.getSnapshotId()),
+            mostRecentExport,
+            latestCommitLogTime,
+            Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse));
         State state = pipeline.run().waitUntilFinish();
         if (!State.DONE.equals(state)) {
           throw new IllegalStateException("Unexpected pipeline state: " + state);
@@ -120,15 +130,16 @@ public class ValidateSqlPipeline {
     }
   }
 
-  void setupPipeline(
-      Pipeline pipeline, Optional<String> sqlSnapshotId, DateTime latestCommitLogTime) {
+  static void setupPipeline(
+      Pipeline pipeline,
+      Optional<String> sqlSnapshotId,
+      DatastoreSnapshotInfo mostRecentExport,
+      DateTime latestCommitLogTime,
+      Optional<DateTime> compareStartTime) {
     pipeline
         .getCoderRegistry()
         .registerCoderForClass(SqlEntity.class, SerializableCoder.of(Serializable.class));
 
-    Optional<DateTime> compareStartTime =
-        Optional.ofNullable(options.getComparisonStartTimestamp()).map(DateTime::parse);
-
     PCollectionTuple datastoreSnapshot =
         DatastoreSnapshots.loadDatastoreSnapshotByKind(
             pipeline,
@@ -216,6 +227,10 @@ public class ValidateSqlPipeline {
         return "ValidateSqlPipeline";
       }
 
+      LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+          DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+              .datastoreSnapshotInfoFinder();
+
       @Override
       public boolean isRunning(String requestLogId) {
         return true;
@@ -234,11 +249,16 @@ public class ValidateSqlPipeline {
     // Reuse Dataflow worker initialization code to set up JPA in the pipeline harness.
     new RegistryPipelineWorkerInitializer().beforeProcessing(options);
 
-    DatastoreSnapshotInfo mostRecentExport =
-        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
-            .datastoreSnapshotInfoFinder()
-            .getSnapshotInfo();
+    MigrationState state =
+        DatabaseMigrationStateSchedule.getValueAtTime(new SystemClock().nowUtc());
+    if (!state.getReplayDirection().equals(ReplayDirection.DATASTORE_TO_SQL)) {
+      throw new IllegalStateException("This pipeline is not designed for migration phase " + state);
+    }
 
-    new ValidateSqlPipeline(options, mostRecentExport).run(Pipeline.create(options));
+    LatestDatastoreSnapshotFinder datastoreSnapshotFinder =
+        DaggerLatestDatastoreSnapshotFinder_LatestDatastoreSnapshotFinderFinderComponent.create()
+            .datastoreSnapshotInfoFinder();
+
+    new ValidateSqlPipeline(options, datastoreSnapshotFinder).run(Pipeline.create(options));
   }
 }

core/src/main/java/google/registry/beam/comparedb/ValidateSqlUtils.java

@@ -104,6 +104,7 @@ final class ValidateSqlUtils {
     private final HashMap<String, Counter> missingCounters = new HashMap<>();
     private final HashMap<String, Counter> unequalCounters = new HashMap<>();
     private final HashMap<String, Counter> badEntityCounters = new HashMap<>();
+    private final HashMap<String, Counter> duplicateEntityCounters = new HashMap<>();
 
     private volatile boolean logPrinted = false;
 
@@ -120,6 +121,8 @@ final class ValidateSqlUtils {
           counterKey, Metrics.counter("CompareDB", "Missing In One DB: " + counterKey));
       unequalCounters.put(counterKey, Metrics.counter("CompareDB", "Not Equal:" + counterKey));
       badEntityCounters.put(counterKey, Metrics.counter("CompareDB", "Bad Entities:" + counterKey));
+      duplicateEntityCounters.put(
+          counterKey, Metrics.counter("CompareDB", "Duplicate Entities:" + counterKey));
     }
 
     /**
@@ -158,12 +161,18 @@ final class ValidateSqlUtils {
       ImmutableList<SqlEntity> entities = ImmutableList.copyOf(kv.getValue());
 
       verify(!entities.isEmpty(), "Can't happen: no value for key %s.", kv.getKey());
-      verify(entities.size() <= 2, "Unexpected duplicates for key %s", kv.getKey());
 
       String counterKey = getCounterKey(entities.get(0).getClass());
       ensureCounterExists(counterKey);
       totalCounters.get(counterKey).inc();
 
+      if (entities.size() > 2) {
+        // Duplicates may happen with Cursors if imported across projects. Its key in Datastore, the
+        // id field, encodes the project name and is not fixed by the importing job.
+        duplicateEntityCounters.get(counterKey).inc();
+        return;
+      }
+
       if (entities.size() == 1) {
         if (isSpecialCaseProberEntity(entities.get(0))) {
           return;

core/src/main/java/google/registry/beam/rde/RdeIO.java

@@ -248,6 +248,9 @@ public class RdeIO {
       // Now that we're done, output roll the cursor forward.
       if (key.manual()) {
         logger.atInfo().log("Manual operation; not advancing cursor or enqueuing upload task.");
+        // Temporary measure to run RDE in beam in parallel with the daily MapReduce based RDE runs.
+      } else if (tm().isOfy()) {
+        logger.atInfo().log("Ofy is primary TM; not advancing cursor or enqueuing upload task.");
       } else {
         outputReceiver.output(KV.of(key, revision));
       }
@@ -294,6 +297,10 @@ public class RdeIO {
                 logger.atInfo().log(
                     "Rolled forward %s on %s cursor to %s.", key.cursor(), key.tld(), newPosition);
                 RdeRevision.saveRevision(key.tld(), key.watermark(), key.mode(), revision);
+                // Enqueueing a task is a side effect that is not undone if the transaction rolls
+                // back. So this may result in multiple copies of the same task being processed.
+                // This is fine because the RdeUploadAction is guarded by a lock and tracks progress
+                // by cursor. The BrdaCopyAction writes a file to GCS, which is an atomic action.
                 if (key.mode() == RdeMode.FULL) {
                   cloudTasksUtils.enqueue(
                       RDE_UPLOAD_QUEUE,

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -242,6 +242,9 @@ cloudSql:
   jdbcUrl: jdbc:postgresql://localhost
   # This name is used by Cloud SQL when connecting to the database.
   instanceConnectionName: project-id:region:instance-id
+  # If non-null, we will use this instance for certain read-only actions or
+  # pipelines, e.g. RDE, in order to offload some work from the primary
+  # instance. Expect any write actions on this instance to fail.
   replicaInstanceConnectionName: null
 
 cloudDns:

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -36,6 +36,19 @@
     <target>backend</target>
   </cron>
 
+  <cron>
+    <url>/_dr/task/rdeStaging?beam=true</url>
+    <description>
+      This job generates a full RDE escrow deposit as a single gigantic XML
+      document using the Beam pipeline regardless of the current TM
+      configuration and streams it to cloud storage. It does not trigger the
+      subsequent upload tasks and is meant to run parallel with the main cron
+      job in order to compare the results from both runs.
+    </description>
+    <schedule>every 8 hours from 00:07 to 20:00</schedule>
+    <target>backend</target>
+  </cron>
+
   <cron>
     <url><![CDATA[/_dr/cron/fanout?queue=rde-upload&endpoint=/_dr/task/rdeUpload&forEachRealTld]]></url>
     <description>

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -169,6 +169,7 @@ import org.joda.time.Duration;
  * @error {@link DomainFlowUtils.FeesMismatchException}
  * @error {@link DomainFlowUtils.FeesRequiredDuringEarlyAccessProgramException}
  * @error {@link DomainFlowUtils.FeesRequiredForPremiumNameException}
+ * @error {@link DomainFlowUtils.InvalidDsRecordException}
  * @error {@link DomainFlowUtils.InvalidIdnDomainLabelException}
  * @error {@link DomainFlowUtils.InvalidPunycodeException}
  * @error {@link DomainFlowUtils.InvalidTcnIdChecksumException}

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -129,6 +129,7 @@ import google.registry.model.tld.label.ReservedList;
 import google.registry.model.tmch.ClaimsListDao;
 import google.registry.persistence.VKey;
 import google.registry.tldconfig.idn.IdnLabelValidator;
+import google.registry.tools.DigestType;
 import google.registry.util.Idn;
 import java.math.BigDecimal;
 import java.util.Collection;
@@ -144,6 +145,7 @@ import org.joda.money.CurrencyUnit;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
+import org.xbill.DNS.DNSSEC.Algorithm;
 
 /** Static utility functions for domain flows. */
 public class DomainFlowUtils {
@@ -293,13 +295,46 @@ public class DomainFlowUtils {
 
   /** Check that the DS data that will be set on a domain is valid. */
   static void validateDsData(Set<DelegationSignerData> dsData) throws EppException {
-    if (dsData != null && dsData.size() > MAX_DS_RECORDS_PER_DOMAIN) {
-      throw new TooManyDsRecordsException(
-          String.format(
-              "A maximum of %s DS records are allowed per domain.", MAX_DS_RECORDS_PER_DOMAIN));
+    if (dsData != null) {
+      if (dsData.size() > MAX_DS_RECORDS_PER_DOMAIN) {
+        throw new TooManyDsRecordsException(
+            String.format(
+                "A maximum of %s DS records are allowed per domain.", MAX_DS_RECORDS_PER_DOMAIN));
+      }
+      // TODO(sarahbot@): Add signature length verification
+      ImmutableList<DelegationSignerData> invalidAlgorithms =
+          dsData.stream()
+              .filter(ds -> !validateAlgorithm(ds.getAlgorithm()))
+              .collect(toImmutableList());
+      if (!invalidAlgorithms.isEmpty()) {
+        throw new InvalidDsRecordException(
+            String.format(
+                "Domain contains DS record(s) with an invalid algorithm wire value: %s",
+                invalidAlgorithms));
+      }
+      ImmutableList<DelegationSignerData> invalidDigestTypes =
+          dsData.stream()
+              .filter(ds -> !DigestType.fromWireValue(ds.getDigestType()).isPresent())
+              .collect(toImmutableList());
+      if (!invalidDigestTypes.isEmpty()) {
+        throw new InvalidDsRecordException(
+            String.format(
+                "Domain contains DS record(s) with an invalid digest type: %s",
+                invalidDigestTypes));
+      }
     }
   }
 
+  public static boolean validateAlgorithm(int alg) {
+    if (alg > 255 || alg < 0) {
+      return false;
+    }
+    // Algorithms that are reserved or unassigned will just return a string representation of their
+    // integer wire value.
+    String algorithm = Algorithm.string(alg);
+    return !algorithm.equals(Integer.toString(alg));
+  }
+
   /** We only allow specifying years in a period. */
   static Period verifyUnitIsYears(Period period) throws EppException {
     if (!checkNotNull(period).getUnit().equals(Period.Unit.YEARS)) {
@@ -1217,6 +1252,13 @@ public class DomainFlowUtils {
     }
   }
 
+  /** Domain has an invalid DS record. */
+  static class InvalidDsRecordException extends ParameterValuePolicyErrorException {
+    public InvalidDsRecordException(String message) {
+      super(message);
+    }
+  }
+
   /** Domain name is under tld which doesn't exist. */
   static class TldDoesNotExistException extends ParameterValueRangeErrorException {
     public TldDoesNotExistException(String tld) {

core/src/main/java/google/registry/flows/domain/DomainUpdateFlow.java

@@ -114,6 +114,7 @@ import org.joda.time.DateTime;
  * @error {@link DomainFlowUtils.EmptySecDnsUpdateException}
  * @error {@link DomainFlowUtils.FeesMismatchException}
  * @error {@link DomainFlowUtils.FeesRequiredForNonFreeOperationException}
+ * @error {@link DomainFlowUtils.InvalidDsRecordException}
  * @error {@link DomainFlowUtils.LinkedResourcesDoNotExistException}
  * @error {@link DomainFlowUtils.LinkedResourceInPendingDeleteProhibitsOperationException}
  * @error {@link DomainFlowUtils.MaxSigLifeChangeNotSupportedException}

core/src/main/java/google/registry/model/EppResource.java

@@ -21,6 +21,7 @@ import static com.google.common.collect.Sets.union;
 import static google.registry.config.RegistryConfig.getEppResourceCachingDuration;
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.nullToEmpty;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
@@ -380,13 +381,13 @@ public abstract class EppResource extends BackupGroupRoot implements Buildable {
 
         @Override
         public EppResource load(VKey<? extends EppResource> key) {
-          return tm().doTransactionless(() -> tm().loadByKey(key));
+          return replicaTm().doTransactionless(() -> replicaTm().loadByKey(key));
         }
 
         @Override
         public Map<VKey<? extends EppResource>, EppResource> loadAll(
             Iterable<? extends VKey<? extends EppResource>> keys) {
-          return tm().doTransactionless(() -> tm().loadByKeys(keys));
+          return replicaTm().doTransactionless(() -> replicaTm().loadByKeys(keys));
         }
       };
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -34,6 +34,9 @@ import javax.persistence.AccessType;
 @ReportedOn
 @Entity
 @javax.persistence.Entity(name = "Host")
+@javax.persistence.Table(
+    name = "Host",
+    indexes = {@javax.persistence.Index(columnList = "hostName")})
 @ExternalMessagingName("host")
 @WithStringVKey
 @Access(AccessType.FIELD) // otherwise it'll use the default if the repoId (property)

core/src/main/java/google/registry/model/index/ForeignKeyIndex.java

@@ -21,6 +21,7 @@ import static google.registry.config.RegistryConfig.getEppResourceCachingDuratio
 import static google.registry.config.RegistryConfig.getEppResourceMaxCachedEntries;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.entriesToImmutableMap;
 import static google.registry.util.TypeUtils.instantiate;
@@ -51,6 +52,7 @@ import google.registry.model.host.HostResource;
 import google.registry.model.replay.DatastoreOnlyEntity;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.CriteriaQueryBuilder;
+import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.util.NonFinalForTesting;
 import java.util.Collection;
 import java.util.Comparator;
@@ -198,7 +200,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    */
   public static <E extends EppResource> ImmutableMap<String, ForeignKeyIndex<E>> load(
       Class<E> clazz, Collection<String> foreignKeys, final DateTime now) {
-    return loadIndexesFromStore(clazz, foreignKeys, true).entrySet().stream()
+    return loadIndexesFromStore(clazz, foreignKeys, true, false).entrySet().stream()
         .filter(e -> now.isBefore(e.getValue().getDeletionTime()))
         .collect(entriesToImmutableMap());
   }
@@ -217,7 +219,10 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
    */
   private static <E extends EppResource>
       ImmutableMap<String, ForeignKeyIndex<E>> loadIndexesFromStore(
-          Class<E> clazz, Collection<String> foreignKeys, boolean inTransaction) {
+          Class<E> clazz,
+          Collection<String> foreignKeys,
+          boolean inTransaction,
+          boolean useReplicaJpaTm) {
     if (tm().isOfy()) {
       Class<ForeignKeyIndex<E>> fkiClass = mapToFkiClass(clazz);
       return ImmutableMap.copyOf(
@@ -226,17 +231,18 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               : tm().doTransactionless(() -> auditedOfy().load().type(fkiClass).ids(foreignKeys)));
     } else {
       String property = RESOURCE_CLASS_TO_FKI_PROPERTY.get(clazz);
+      JpaTransactionManager jpaTmToUse = useReplicaJpaTm ? replicaJpaTm() : jpaTm();
       ImmutableList<ForeignKeyIndex<E>> indexes =
-          tm().transact(
-                  () ->
-                      jpaTm()
-                          .criteriaQuery(
-                              CriteriaQueryBuilder.create(clazz)
-                                  .whereFieldIsIn(property, foreignKeys)
-                                  .build())
-                          .getResultStream()
-                          .map(e -> ForeignKeyIndex.create(e, e.getDeletionTime()))
-                          .collect(toImmutableList()));
+          jpaTmToUse.transact(
+              () ->
+                  jpaTmToUse
+                      .criteriaQuery(
+                          CriteriaQueryBuilder.create(clazz)
+                              .whereFieldIsIn(property, foreignKeys)
+                              .build())
+                      .getResultStream()
+                      .map(e -> ForeignKeyIndex.create(e, e.getDeletionTime()))
+                      .collect(toImmutableList()));
       // We need to find and return the entities with the maximum deletionTime for each foreign key.
       return Multimaps.index(indexes, ForeignKeyIndex::getForeignKey).asMap().entrySet().stream()
           .map(
@@ -260,7 +266,8 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               loadIndexesFromStore(
                       RESOURCE_CLASS_TO_FKI_CLASS.inverse().get(key.getKind()),
                       ImmutableSet.of(foreignKey),
-                      false)
+                      false,
+                      true)
                   .get(foreignKey));
         }
 
@@ -276,7 +283,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
               Streams.stream(keys).map(v -> v.getSqlKey().toString()).collect(toImmutableSet());
           ImmutableSet<VKey<ForeignKeyIndex<?>>> typedKeys = ImmutableSet.copyOf(keys);
           ImmutableMap<String, ? extends ForeignKeyIndex<? extends EppResource>> existingFkis =
-              loadIndexesFromStore(resourceClass, foreignKeys, false);
+              loadIndexesFromStore(resourceClass, foreignKeys, false, true);
           // ofy omits keys that don't have values in Datastore, so re-add them in
           // here with Optional.empty() values.
           return Maps.asMap(
@@ -336,7 +343,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
     // Safe to cast VKey<FKI<E>> to VKey<FKI<?>>
     @SuppressWarnings("unchecked")
     ImmutableList<VKey<ForeignKeyIndex<?>>> fkiVKeys =
-        Streams.stream(foreignKeys)
+        foreignKeys.stream()
             .map(fk -> (VKey<ForeignKeyIndex<?>>) VKey.create(fkiClass, fk))
             .collect(toImmutableList());
     try {

core/src/main/java/google/registry/module/pubapi/PubApiComponent.java

@@ -30,6 +30,7 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.module.pubapi.PubApiRequestComponent.PubApiRequestComponentModule;
 import google.registry.monitoring.whitebox.StackdriverModule;
+import google.registry.persistence.PersistenceModule;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
@@ -56,6 +57,7 @@ import javax.inject.Singleton;
       KeyringModule.class,
       KmsModule.class,
       NetHttpTransportModule.class,
+      PersistenceModule.class,
       PubApiRequestComponentModule.class,
       SecretManagerModule.class,
       ServerTridProviderModule.class,

core/src/main/java/google/registry/persistence/PersistenceModule.java

@@ -277,6 +277,8 @@ public abstract class PersistenceModule {
     setSqlCredential(credentialStore, new RobotUser(RobotId.NOMULUS), overrides);
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
+    overrides.put(
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 
@@ -291,6 +293,8 @@ public abstract class PersistenceModule {
     HashMap<String, String> overrides = Maps.newHashMap(beamCloudSqlConfigs);
     replicaInstanceConnectionName.ifPresent(
         name -> overrides.put(HIKARI_DS_CLOUD_SQL_INSTANCE, name));
+    overrides.put(
+        Environment.ISOLATION, TransactionIsolationLevel.TRANSACTION_READ_COMMITTED.name());
     return new JpaTransactionManagerImpl(create(overrides), clock);
   }
 

core/src/main/java/google/registry/persistence/transaction/CriteriaQueryBuilder.java

@@ -18,7 +18,6 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
 
 import com.google.common.collect.ImmutableList;
 import java.util.Collection;
-import javax.persistence.EntityManager;
 import javax.persistence.criteria.CriteriaBuilder;
 import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Expression;
@@ -42,12 +41,14 @@ public class CriteriaQueryBuilder<T> {
 
   private final CriteriaQuery<T> query;
   private final Root<?> root;
+  private final JpaTransactionManager jpaTm;
   private final ImmutableList.Builder<Predicate> predicates = new ImmutableList.Builder<>();
   private final ImmutableList.Builder<Order> orders = new ImmutableList.Builder<>();
 
-  private CriteriaQueryBuilder(CriteriaQuery<T> query, Root<?> root) {
+  private CriteriaQueryBuilder(CriteriaQuery<T> query, Root<?> root, JpaTransactionManager jpaTm) {
     this.query = query;
     this.root = root;
+    this.jpaTm = jpaTm;
   }
 
   /** Adds a WHERE clause to the query, given the specified operation, field, and value. */
@@ -75,18 +76,18 @@ public class CriteriaQueryBuilder<T> {
    */
   public <V> CriteriaQueryBuilder<T> whereFieldContains(String fieldName, Object value) {
     return where(
-        jpaTm().getEntityManager().getCriteriaBuilder().isMember(value, root.get(fieldName)));
+        jpaTm.getEntityManager().getCriteriaBuilder().isMember(value, root.get(fieldName)));
   }
 
   /** Orders the result by the given field ascending. */
   public CriteriaQueryBuilder<T> orderByAsc(String fieldName) {
-    orders.add(jpaTm().getEntityManager().getCriteriaBuilder().asc(root.get(fieldName)));
+    orders.add(jpaTm.getEntityManager().getCriteriaBuilder().asc(root.get(fieldName)));
     return this;
   }
 
   /** Orders the result by the given field descending. */
   public CriteriaQueryBuilder<T> orderByDesc(String fieldName) {
-    orders.add(jpaTm().getEntityManager().getCriteriaBuilder().desc(root.get(fieldName)));
+    orders.add(jpaTm.getEntityManager().getCriteriaBuilder().desc(root.get(fieldName)));
     return this;
   }
 
@@ -103,23 +104,24 @@ public class CriteriaQueryBuilder<T> {
 
   /** Creates a query builder that will SELECT from the given class. */
   public static <T> CriteriaQueryBuilder<T> create(Class<T> clazz) {
-    return create(jpaTm().getEntityManager(), clazz);
+    return create(jpaTm(), clazz);
   }
 
   /** Creates a query builder for the given entity manager. */
-  public static <T> CriteriaQueryBuilder<T> create(EntityManager em, Class<T> clazz) {
-    CriteriaQuery<T> query = em.getCriteriaBuilder().createQuery(clazz);
+  public static <T> CriteriaQueryBuilder<T> create(JpaTransactionManager jpaTm, Class<T> clazz) {
+    CriteriaQuery<T> query = jpaTm.getEntityManager().getCriteriaBuilder().createQuery(clazz);
     Root<T> root = query.from(clazz);
     query = query.select(root);
-    return new CriteriaQueryBuilder<>(query, root);
+    return new CriteriaQueryBuilder<>(query, root, jpaTm);
   }
 
   /** Creates a "count" query for the table for the class. */
-  public static <T> CriteriaQueryBuilder<Long> createCount(EntityManager em, Class<T> clazz) {
-    CriteriaBuilder builder = em.getCriteriaBuilder();
+  public static <T> CriteriaQueryBuilder<Long> createCount(
+      JpaTransactionManager jpaTm, Class<T> clazz) {
+    CriteriaBuilder builder = jpaTm.getEntityManager().getCriteriaBuilder();
     CriteriaQuery<Long> query = builder.createQuery(Long.class);
     Root<T> root = query.from(clazz);
     query = query.select(builder.count(root));
-    return new CriteriaQueryBuilder<>(query, root);
+    return new CriteriaQueryBuilder<>(query, root, jpaTm);
   }
 }

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -141,14 +141,15 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     // Postgresql-specific: 'set transaction' command must be called inside a transaction
     assertInTransaction();
 
-    EntityManager entityManager = getEntityManager();
+    ReadOnlyCheckingEntityManager entityManager =
+        (ReadOnlyCheckingEntityManager) getEntityManager();
     // Isolation is hardcoded to REPEATABLE READ, as specified by parent's Javadoc.
     entityManager
         .createNativeQuery("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
-        .executeUpdate();
+        .executeUpdateIgnoringReadOnly();
     entityManager
         .createNativeQuery(String.format("SET TRANSACTION SNAPSHOT '%s'", snapshotId))
-        .executeUpdate();
+        .executeUpdateIgnoringReadOnly();
     return this;
   }
 
@@ -1130,7 +1131,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
 
     private TypedQuery<T> buildQuery() {
       CriteriaQueryBuilder<T> queryBuilder =
-          CriteriaQueryBuilder.create(getEntityManager(), entityClass);
+          CriteriaQueryBuilder.create(JpaTransactionManagerImpl.this, entityClass);
       return addCriteria(queryBuilder);
     }
 
@@ -1177,7 +1178,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
     @Override
     public long count() {
       CriteriaQueryBuilder<Long> queryBuilder =
-          CriteriaQueryBuilder.createCount(getEntityManager(), entityClass);
+          CriteriaQueryBuilder.createCount(JpaTransactionManagerImpl.this, entityClass);
       return addCriteria(queryBuilder).getSingleResult();
     }
 

core/src/main/java/google/registry/persistence/transaction/ReadOnlyCheckingEntityManager.java

@@ -206,7 +206,7 @@ public class ReadOnlyCheckingEntityManager implements EntityManager {
   }
 
   @Override
-  public Query createNativeQuery(String sqlString) {
+  public ReadOnlyCheckingQuery createNativeQuery(String sqlString) {
     return new ReadOnlyCheckingQuery(delegate.createNativeQuery(sqlString));
   }
 

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -14,8 +14,8 @@
 
 package google.registry.persistence.transaction;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.appengine.api.utils.SystemProperty;
@@ -47,6 +47,10 @@ public final class TransactionManagerFactory {
   private static Supplier<JpaTransactionManager> jpaTm =
       Suppliers.memoize(TransactionManagerFactory::createJpaTransactionManager);
 
+  @NonFinalForTesting
+  private static Supplier<JpaTransactionManager> replicaJpaTm =
+      Suppliers.memoize(TransactionManagerFactory::createReplicaJpaTransactionManager);
+
   private static boolean onBeam = false;
 
   private TransactionManagerFactory() {}
@@ -61,6 +65,14 @@ public final class TransactionManagerFactory {
     }
   }
 
+  private static JpaTransactionManager createReplicaJpaTransactionManager() {
+    if (isInAppEngine()) {
+      return DaggerPersistenceComponent.create().readOnlyReplicaJpaTransactionManager();
+    } else {
+      return DummyJpaTransactionManager.create();
+    }
+  }
+
   private static DatastoreTransactionManager createTransactionManager() {
     return new DatastoreTransactionManager(null);
   }
@@ -108,6 +120,21 @@ public final class TransactionManagerFactory {
     return jpaTm.get();
   }
 
+  /** Returns a read-only {@link JpaTransactionManager} instance if configured. */
+  public static JpaTransactionManager replicaJpaTm() {
+    return replicaJpaTm.get();
+  }
+
+  /**
+   * Returns a {@link TransactionManager} that uses a replica database if one exists.
+   *
+   * <p>In Datastore mode, this is unchanged from the regular transaction manager. In SQL mode,
+   * however, this will be a reference to the read-only replica database if one is configured.
+   */
+  public static TransactionManager replicaTm() {
+    return tm().isOfy() ? tm() : replicaJpaTm();
+  }
+
   /** Returns {@link DatastoreTransactionManager} instance. */
   @VisibleForTesting
   public static DatastoreTransactionManager ofyTm() {
@@ -116,7 +143,7 @@ public final class TransactionManagerFactory {
 
   /** Sets the return of {@link #jpaTm()} to the given instance of {@link JpaTransactionManager}. */
   public static void setJpaTm(Supplier<JpaTransactionManager> jpaTmSupplier) {
-    checkNotNull(jpaTmSupplier, "jpaTmSupplier");
+    checkArgumentNotNull(jpaTmSupplier, "jpaTmSupplier");
     checkState(
         RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
             || RegistryToolEnvironment.get() != null,
@@ -124,13 +151,23 @@ public final class TransactionManagerFactory {
     jpaTm = Suppliers.memoize(jpaTmSupplier::get);
   }
 
+  /** Sets the value of {@link #replicaJpaTm()} to the given {@link JpaTransactionManager}. */
+  public static void setReplicaJpaTm(Supplier<JpaTransactionManager> replicaJpaTmSupplier) {
+    checkArgumentNotNull(replicaJpaTmSupplier, "replicaJpaTmSupplier");
+    checkState(
+        RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
+            || RegistryToolEnvironment.get() != null,
+        "setReplicaJpaTm() should only be called by tools and tests.");
+    replicaJpaTm = Suppliers.memoize(replicaJpaTmSupplier::get);
+  }
+
   /**
    * Makes {@link #jpaTm()} return the {@link JpaTransactionManager} instance provided by {@code
    * jpaTmSupplier} from now on. This method should only be called by an implementor of {@link
    * org.apache.beam.sdk.harness.JvmInitializer}.
    */
   public static void setJpaTmOnBeamWorker(Supplier<JpaTransactionManager> jpaTmSupplier) {
-    checkNotNull(jpaTmSupplier, "jpaTmSupplier");
+    checkArgumentNotNull(jpaTmSupplier, "jpaTmSupplier");
     jpaTm = Suppliers.memoize(jpaTmSupplier::get);
     onBeam = true;
   }

core/src/main/java/google/registry/rdap/RdapDomainSearchAction.java

@@ -18,7 +18,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.HEAD;
@@ -91,7 +91,9 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
   @Inject @Parameter("name") Optional<String> nameParam;
   @Inject @Parameter("nsLdhName") Optional<String> nsLdhNameParam;
   @Inject @Parameter("nsIp") Optional<String> nsIpParam;
-  @Inject public RdapDomainSearchAction() {
+
+  @Inject
+  public RdapDomainSearchAction() {
     super("domain search", EndpointType.DOMAINS);
   }
 
@@ -223,13 +225,13 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
       resultSet = getMatchingResources(query, true, querySizeLimit);
     } else {
       resultSet =
-          jpaTm()
+          replicaJpaTm()
               .transact(
                   () -> {
                     CriteriaBuilder criteriaBuilder =
-                        jpaTm().getEntityManager().getCriteriaBuilder();
+                        replicaJpaTm().getEntityManager().getCriteriaBuilder();
                     CriteriaQueryBuilder<DomainBase> queryBuilder =
-                        CriteriaQueryBuilder.create(DomainBase.class)
+                        CriteriaQueryBuilder.create(replicaJpaTm(), DomainBase.class)
                             .where(
                                 "fullyQualifiedDomainName",
                                 criteriaBuilder::like,
@@ -270,7 +272,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
       resultSet = getMatchingResources(query, true, querySizeLimit);
     } else {
       resultSet =
-          jpaTm()
+          replicaJpaTm()
               .transact(
                   () -> {
                     CriteriaQueryBuilder<DomainBase> builder =
@@ -354,7 +356,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
           .map(VKey::from)
           .collect(toImmutableSet());
     } else {
-      return jpaTm()
+      return replicaJpaTm()
           .transact(
               () -> {
                 CriteriaQueryBuilder<HostResource> builder =
@@ -368,11 +370,12 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
                   builder =
                       builder.where(
                           "currentSponsorClientId",
-                          jpaTm().getEntityManager().getCriteriaBuilder()::equal,
+                          replicaJpaTm().getEntityManager().getCriteriaBuilder()::equal,
                           desiredRegistrar.get());
                 }
                 return getMatchingResourcesSql(builder, true, maxNameserversInFirstStage)
-                    .resources().stream()
+                    .resources()
+                    .stream()
                     .map(HostResource::createVKey)
                     .collect(toImmutableSet());
               });
@@ -509,11 +512,11 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         parameters.put("desiredRegistrar", desiredRegistrar.get());
       }
       hostKeys =
-          jpaTm()
+          replicaJpaTm()
               .transact(
                   () -> {
                     javax.persistence.Query query =
-                        jpaTm()
+                        replicaJpaTm()
                             .getEntityManager()
                             .createNativeQuery(queryBuilder.toString())
                             .setMaxResults(maxNameserversInFirstStage);
@@ -568,16 +571,16 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
         }
         stream.forEach(domainSetBuilder::add);
       } else {
-        jpaTm()
+        replicaJpaTm()
             .transact(
                 () -> {
                   for (VKey<HostResource> hostKey : hostKeys) {
                     CriteriaQueryBuilder<DomainBase> queryBuilder =
-                        CriteriaQueryBuilder.create(DomainBase.class)
+                        CriteriaQueryBuilder.create(replicaJpaTm(), DomainBase.class)
                             .whereFieldContains("nsHosts", hostKey)
                             .orderByAsc("fullyQualifiedDomainName");
                     CriteriaBuilder criteriaBuilder =
-                        jpaTm().getEntityManager().getCriteriaBuilder();
+                        replicaJpaTm().getEntityManager().getCriteriaBuilder();
                     if (!shouldIncludeDeleted()) {
                       queryBuilder =
                           queryBuilder.where(
@@ -590,7 +593,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
                               criteriaBuilder::greaterThan,
                               cursorString.get());
                     }
-                    jpaTm()
+                    replicaJpaTm()
                         .criteriaQuery(queryBuilder.build())
                         .getResultStream()
                         .filter(this::isAuthorized)

core/src/main/java/google/registry/rdap/RdapEntitySearchAction.java

@@ -15,7 +15,7 @@
 package google.registry.rdap;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
 import static google.registry.rdap.RdapUtils.getRegistrarByIanaIdentifier;
@@ -277,7 +277,7 @@ public class RdapEntitySearchAction extends RdapSearchActionBase {
           resultSet = getMatchingResources(query, false, rdapResultSetMaxSize + 1);
         } else {
           resultSet =
-              jpaTm()
+              replicaJpaTm()
                   .transact(
                       () -> {
                         CriteriaQueryBuilder<ContactResource> builder =
@@ -399,7 +399,7 @@ public class RdapEntitySearchAction extends RdapSearchActionBase {
                   querySizeLimit);
         } else {
           contactResultSet =
-              jpaTm()
+              replicaJpaTm()
                   .transact(
                       () ->
                           getMatchingResourcesSql(

core/src/main/java/google/registry/rdap/RdapNameserverSearchAction.java

@@ -15,7 +15,7 @@
 package google.registry.rdap;
 
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.GET;
 import static google.registry.request.Action.Method.HEAD;
@@ -233,7 +233,7 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase {
       return makeSearchResults(
           getMatchingResources(query, shouldIncludeDeleted(), querySizeLimit), CursorType.NAME);
     } else {
-      return jpaTm()
+      return replicaJpaTm()
           .transact(
               () -> {
                 CriteriaQueryBuilder<HostResource> queryBuilder =
@@ -290,11 +290,11 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase {
       }
       queryBuilder.append(" ORDER BY repo_id ASC");
       rdapResultSet =
-          jpaTm()
+          replicaJpaTm()
               .transact(
                   () -> {
                     javax.persistence.Query query =
-                        jpaTm()
+                        replicaJpaTm()
                             .getEntityManager()
                             .createNativeQuery(queryBuilder.toString(), HostResource.class)
                             .setMaxResults(querySizeLimit);

core/src/main/java/google/registry/rdap/RdapSearchActionBase.java

@@ -16,7 +16,7 @@ package google.registry.rdap;
 
 import static com.google.common.base.Charsets.UTF_8;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 
 import com.google.common.collect.ImmutableList;
@@ -193,16 +193,17 @@ public abstract class RdapSearchActionBase extends RdapActionBase {
    */
   <T extends EppResource> RdapResultSet<T> getMatchingResourcesSql(
       CriteriaQueryBuilder<T> builder, boolean checkForVisibility, int querySizeLimit) {
-    jpaTm().assertInTransaction();
+    replicaJpaTm().assertInTransaction();
     Optional<String> desiredRegistrar = getDesiredRegistrar();
     if (desiredRegistrar.isPresent()) {
       builder =
           builder.where(
-              "currentSponsorClientId", jpaTm().getEntityManager().getCriteriaBuilder()::equal,
+              "currentSponsorClientId",
+              replicaJpaTm().getEntityManager().getCriteriaBuilder()::equal,
               desiredRegistrar.get());
     }
     List<T> queryResult =
-        jpaTm().criteriaQuery(builder.build()).setMaxResults(querySizeLimit).getResultList();
+        replicaJpaTm().criteriaQuery(builder.build()).setMaxResults(querySizeLimit).getResultList();
     if (checkForVisibility) {
       return filterResourcesByVisibility(queryResult, querySizeLimit);
     } else {
@@ -395,7 +396,7 @@ public abstract class RdapSearchActionBase extends RdapActionBase {
       RdapSearchPattern partialStringQuery,
       Optional<String> cursorString,
       DeletedItemHandling deletedItemHandling) {
-    jpaTm().assertInTransaction();
+    replicaJpaTm().assertInTransaction();
     if (partialStringQuery.getInitialString().length()
         < RdapSearchPattern.MIN_INITIAL_STRING_LENGTH) {
       throw new UnprocessableEntityException(
@@ -403,8 +404,8 @@ public abstract class RdapSearchActionBase extends RdapActionBase {
               "Initial search string must be at least %d characters",
               RdapSearchPattern.MIN_INITIAL_STRING_LENGTH));
     }
-    CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
-    CriteriaQueryBuilder<T> builder = CriteriaQueryBuilder.create(clazz);
+    CriteriaBuilder criteriaBuilder = replicaJpaTm().getEntityManager().getCriteriaBuilder();
+    CriteriaQueryBuilder<T> builder = CriteriaQueryBuilder.create(replicaJpaTm(), clazz);
     if (partialStringQuery.getHasWildcard()) {
       builder =
           builder.where(
@@ -493,9 +494,9 @@ public abstract class RdapSearchActionBase extends RdapActionBase {
               "Initial search string must be at least %d characters",
               RdapSearchPattern.MIN_INITIAL_STRING_LENGTH));
     }
-    jpaTm().assertInTransaction();
-    CriteriaQueryBuilder<T> builder = CriteriaQueryBuilder.create(clazz);
-    CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
+    replicaJpaTm().assertInTransaction();
+    CriteriaQueryBuilder<T> builder = CriteriaQueryBuilder.create(replicaJpaTm(), clazz);
+    CriteriaBuilder criteriaBuilder = replicaJpaTm().getEntityManager().getCriteriaBuilder();
     builder = builder.where(filterField, criteriaBuilder::equal, queryString);
     if (cursorString.isPresent()) {
       if (cursorField.isPresent()) {
@@ -544,7 +545,7 @@ public abstract class RdapSearchActionBase extends RdapActionBase {
       RdapSearchPattern partialStringQuery,
       Optional<String> cursorString,
       DeletedItemHandling deletedItemHandling) {
-    jpaTm().assertInTransaction();
+    replicaJpaTm().assertInTransaction();
     return queryItemsSql(clazz, "repoId", partialStringQuery, cursorString, deletedItemHandling);
   }
 
@@ -553,7 +554,9 @@ public abstract class RdapSearchActionBase extends RdapActionBase {
     if (!Objects.equals(deletedItemHandling, DeletedItemHandling.INCLUDE)) {
       builder =
           builder.where(
-              "deletionTime", jpaTm().getEntityManager().getCriteriaBuilder()::equal, END_OF_TIME);
+              "deletionTime",
+              replicaJpaTm().getEntityManager().getCriteriaBuilder()::equal,
+              END_OF_TIME);
     }
     return builder;
   }

core/src/main/java/google/registry/rde/BrdaCopyAction.java

@@ -24,6 +24,7 @@ import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.model.rde.RdeNamingUtils;
+import google.registry.model.rde.RdeRevision;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
 import google.registry.request.RequestParameters;
@@ -86,7 +87,13 @@ public final class BrdaCopyAction implements Runnable {
   }
 
   private void copyAsRyde() throws IOException {
-    String nameWithoutPrefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, 0);
+    // TODO(b/217772483): consider guarding this action with a lock and check if there is work.
+    // Not urgent since file writes on GCS are atomic.
+    int revision =
+        RdeRevision.getCurrentRevision(tld, watermark, THIN)
+            .orElseThrow(
+                () -> new IllegalStateException("RdeRevision was not set on generated deposit"));
+    String nameWithoutPrefix = RdeNamingUtils.makeRydeFilename(tld, watermark, THIN, 1, revision);
     String name = prefix.orElse("") + nameWithoutPrefix;
     BlobId xmlFilename = BlobId.of(stagingBucket, name + ".xml.ghostryde");
     BlobId xmlLengthFilename = BlobId.of(stagingBucket, name + ".xml.length");

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -391,9 +391,6 @@ public final class RdeStagingAction implements Runnable {
     if (revision.isPresent()) {
       throw new BadRequestException("Revision parameter not allowed in standard operation");
     }
-    if (beam) {
-      throw new BadRequestException("Beam parameter not allowed in standard operation");
-    }
 
     return ImmutableSetMultimap.copyOf(
         Multimaps.filterValues(

core/src/main/java/google/registry/rde/RdeStagingReducer.java

@@ -14,8 +14,6 @@
 
 package google.registry.rde;
 
-import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
-import static com.google.appengine.api.taskqueue.TaskOptions.Builder.withUrl;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Verify.verify;
 import static google.registry.model.common.Cursor.getCursorTimeOrStartOfTime;
@@ -26,6 +24,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import com.google.appengine.tools.mapreduce.Reducer;
 import com.google.appengine.tools.mapreduce.ReducerInput;
 import com.google.cloud.storage.BlobId;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.gcs.GcsUtils;
@@ -36,10 +35,11 @@ import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.model.rde.RdeRevision;
 import google.registry.model.tld.Registry;
+import google.registry.request.Action.Service;
 import google.registry.request.RequestParameters;
 import google.registry.request.lock.LockHandler;
 import google.registry.tldconfig.idn.IdnTableEnum;
-import google.registry.util.TaskQueueUtils;
+import google.registry.util.CloudTasksUtils;
 import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdeheader.XjcRdeHeaderElement;
 import google.registry.xml.ValidationMode;
@@ -65,7 +65,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
 
-  private final TaskQueueUtils taskQueueUtils;
+  private final CloudTasksUtils cloudTasksUtils;
   private final LockHandler lockHandler;
   private final String bucket;
   private final Duration lockTimeout;
@@ -74,14 +74,14 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
   private final GcsUtils gcsUtils;
 
   RdeStagingReducer(
-      TaskQueueUtils taskQueueUtils,
+      CloudTasksUtils cloudTasksUtils,
       LockHandler lockHandler,
       String bucket,
       Duration lockTimeout,
       byte[] stagingKeyBytes,
       ValidationMode validationMode,
       GcsUtils gcsUtils) {
-    this.taskQueueUtils = taskQueueUtils;
+    this.cloudTasksUtils = cloudTasksUtils;
     this.lockHandler = lockHandler;
     this.bucket = bucket;
     this.lockTimeout = lockTimeout;
@@ -226,23 +226,35 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
               logger.atInfo().log(
                   "Rolled forward %s on %s cursor to %s.", key.cursor(), tld, newPosition);
               RdeRevision.saveRevision(tld, watermark, mode, revision);
+              // Enqueueing a task is a side effect that is not undone if the transaction rolls
+              // back. So this may result in multiple copies of the same task being processed. This
+              // is fine because the RdeUploadAction is guarded by a lock and tracks progress by
+              // cursor. The BrdaCopyAction writes a file to GCS, which is an atomic action.
               if (mode == RdeMode.FULL) {
-                taskQueueUtils.enqueue(
-                    getQueue("rde-upload"),
-                    withUrl(RdeUploadAction.PATH).param(RequestParameters.PARAM_TLD, tld));
+                cloudTasksUtils.enqueue(
+                    "rde-upload",
+                    CloudTasksUtils.createPostTask(
+                        RdeUploadAction.PATH,
+                        Service.BACKEND.toString(),
+                        ImmutableMultimap.of(RequestParameters.PARAM_TLD, tld)));
               } else {
-                taskQueueUtils.enqueue(
-                    getQueue("brda"),
-                    withUrl(BrdaCopyAction.PATH)
-                        .param(RequestParameters.PARAM_TLD, tld)
-                        .param(RdeModule.PARAM_WATERMARK, watermark.toString()));
+                cloudTasksUtils.enqueue(
+                    "brda",
+                    CloudTasksUtils.createPostTask(
+                        BrdaCopyAction.PATH,
+                        Service.BACKEND.toString(),
+                        ImmutableMultimap.of(
+                            RequestParameters.PARAM_TLD,
+                            tld,
+                            RdeModule.PARAM_WATERMARK,
+                            watermark.toString())));
               }
             });
   }
 
   /** Injectible factory for creating {@link RdeStagingReducer}. */
   static class Factory {
-    @Inject TaskQueueUtils taskQueueUtils;
+    @Inject CloudTasksUtils cloudTasksUtils;
     @Inject LockHandler lockHandler;
     @Inject @Config("rdeBucket") String bucket;
     @Inject @Config("rdeStagingLockTimeout") Duration lockTimeout;
@@ -252,7 +264,7 @@ public final class RdeStagingReducer extends Reducer<PendingDeposit, DepositFrag
 
     RdeStagingReducer create(ValidationMode validationMode, GcsUtils gcsUtils) {
       return new RdeStagingReducer(
-          taskQueueUtils,
+          cloudTasksUtils,
           lockHandler,
           bucket,
           lockTimeout,

core/src/main/java/google/registry/reporting/billing/GenerateInvoicesAction.java

@@ -32,6 +32,7 @@ import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.config.RegistryEnvironment;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
+import google.registry.persistence.PersistenceModule;
 import google.registry.reporting.ReportingModule;
 import google.registry.request.Action;
 import google.registry.request.Parameter;
@@ -120,17 +121,16 @@ public class GenerateInvoicesAction implements Runnable {
               .setContainerSpecGcsPath(
                   String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
               .setParameters(
-                  ImmutableMap.of(
-                      "yearMonth",
-                      yearMonth.toString("yyyy-MM"),
-                      "invoiceFilePrefix",
-                      invoiceFilePrefix,
-                      "database",
-                      database.name(),
-                      "billingBucketUrl",
-                      billingBucketUrl,
-                      "registryEnvironment",
-                      RegistryEnvironment.get().name()));
+                  new ImmutableMap.Builder<String, String>()
+                      .put("yearMonth", yearMonth.toString("yyyy-MM"))
+                      .put("invoiceFilePrefix", invoiceFilePrefix)
+                      .put("database", database.name())
+                      .put("billingBucketUrl", billingBucketUrl)
+                      .put("registryEnvironment", RegistryEnvironment.get().name())
+                      .put(
+                          "jpaTransactionManagerType",
+                          PersistenceModule.JpaTransactionManagerType.READ_ONLY_REPLICA.toString())
+                      .build());
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/java/google/registry/tools/DigestType.java

@@ -0,0 +1,58 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import java.util.Optional;
+
+/**
+ * Enumerates the DNSSEC digest types for use with Delegation Signer records.
+ *
+ * <p>This also enforces the set of types that are valid for use with Cloud DNS. Customers cannot
+ * create DS records containing any other digest type.
+ *
+ * <p>The complete list can be found here:
+ * https://www.iana.org/assignments/ds-rr-types/ds-rr-types.xhtml
+ */
+public enum DigestType {
+  SHA1(1),
+  SHA256(2),
+  // Algorithm number 3 is GOST R 34.11-94 and is deliberately NOT SUPPORTED.
+  // This algorithm was reviewed by ise-crypto and deemed academically broken (b/207029800).
+  // In addition, RFC 8624 specifies that this algorithm MUST NOT be used for DNSSEC delegations.
+  // TODO(sarhabot@): Add note in Cloud DNS code to notify the Registry of any new changes to
+  // supported digest types.
+  SHA384(4);
+
+  private final int wireValue;
+
+  DigestType(int wireValue) {
+    this.wireValue = wireValue;
+  }
+
+  /** Fetches a DigestType enumeration constant by its IANA assigned value. */
+  public static Optional<DigestType> fromWireValue(int wireValue) {
+    for (DigestType alg : DigestType.values()) {
+      if (alg.getWireValue() == wireValue) {
+        return Optional.of(alg);
+      }
+    }
+    return Optional.empty();
+  }
+
+  /** Fetches a value in the range [0, 255] that encodes this DS digest type on the wire. */
+  public int getWireValue() {
+    return wireValue;
+  }
+}

core/src/main/java/google/registry/tools/DsRecord.java

@@ -16,6 +16,7 @@ package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableList.toImmutableList;
+import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
 
 import com.beust.jcommander.IStringConverter;
 import com.google.auto.value.AutoValue;
@@ -25,6 +26,7 @@ import com.google.common.base.Splitter;
 import com.google.common.io.BaseEncoding;
 import com.google.template.soy.data.SoyListData;
 import com.google.template.soy.data.SoyMapData;
+import google.registry.flows.domain.DomainFlowUtils;
 import java.util.List;
 
 @AutoValue
@@ -46,6 +48,15 @@ abstract class DsRecord {
         "digest should be even-lengthed hex, but is %s (length %s)",
         digest,
         digest.length());
+    checkArgumentPresent(
+        DigestType.fromWireValue(digestType),
+        String.format("DS record uses an unrecognized digest type: %d", digestType));
+
+    if (!DomainFlowUtils.validateAlgorithm(alg)) {
+      throw new IllegalArgumentException(
+          String.format("DS record uses an unrecognized algorithm: %d", alg));
+    }
+
     return new AutoValue_DsRecord(keyTag, alg, digestType, digest);
   }
 

core/src/main/java/google/registry/tools/EncryptEscrowDepositCommand.java

@@ -18,6 +18,7 @@ import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
+import google.registry.model.rde.RdeMode;
 import google.registry.tools.params.PathParameter;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -46,11 +47,20 @@ class EncryptEscrowDepositCommand implements CommandWithRemoteApi {
       validateWith = PathParameter.OutputDirectory.class)
   private Path outdir = Paths.get(".");
 
-  @Inject
-  EscrowDepositEncryptor encryptor;
+  @Parameter(
+      names = {"-m", "--mode"},
+      description = "Specify the escrow mode, FULL for RDE and THIN for BRDA.")
+  private RdeMode mode = RdeMode.FULL;
+
+  @Parameter(
+      names = {"-r", "--revision"},
+      description = "Specify the revision.")
+  private int revision = 0;
+
+  @Inject EscrowDepositEncryptor encryptor;
 
   @Override
   public final void run() throws Exception {
-    encryptor.encrypt(canonicalizeDomainName(tld), input, outdir);
+    encryptor.encrypt(mode, canonicalizeDomainName(tld), revision, input, outdir);
   }
 }

core/src/main/java/google/registry/tools/EscrowDepositEncryptor.java

@@ -18,6 +18,7 @@ import static google.registry.model.rde.RdeMode.FULL;
 
 import com.google.common.io.ByteStreams;
 import google.registry.keyring.api.KeyModule.Key;
+import google.registry.model.rde.RdeMode;
 import google.registry.model.rde.RdeNamingUtils;
 import google.registry.rde.RdeUtil;
 import google.registry.rde.RydeEncoder;
@@ -42,26 +43,44 @@ final class EscrowDepositEncryptor {
 
   @Inject @Key("rdeSigningKey") Provider<PGPKeyPair> rdeSigningKey;
   @Inject @Key("rdeReceiverKey") Provider<PGPPublicKey> rdeReceiverKey;
+
+  @Inject
+  @Key("brdaSigningKey")
+  Provider<PGPKeyPair> brdaSigningKey;
+
+  @Inject
+  @Key("brdaReceiverKey")
+  Provider<PGPPublicKey> brdaReceiverKey;
+
   @Inject EscrowDepositEncryptor() {}
 
   /** Creates a {@code .ryde} and {@code .sig} file, provided an XML deposit file. */
-  void encrypt(String tld, Path xmlFile, Path outdir)
+  void encrypt(RdeMode mode, String tld, Integer revision, Path xmlFile, Path outdir)
       throws IOException, XmlException {
     try (InputStream xmlFileInput = Files.newInputStream(xmlFile);
         BufferedInputStream xmlInput = new BufferedInputStream(xmlFileInput, PEEK_BUFFER_SIZE)) {
       DateTime watermark = RdeUtil.peekWatermark(xmlInput);
-      String name = RdeNamingUtils.makeRydeFilename(tld, watermark, FULL, 1, 0);
+      String name = RdeNamingUtils.makeRydeFilename(tld, watermark, mode, 1, revision);
       Path rydePath = outdir.resolve(name + ".ryde");
       Path sigPath = outdir.resolve(name + ".sig");
       Path pubPath = outdir.resolve(tld + ".pub");
-      PGPKeyPair signingKey = rdeSigningKey.get();
+      PGPKeyPair signingKey;
+      PGPPublicKey receiverKey;
+      if (mode == FULL) {
+        signingKey = rdeSigningKey.get();
+        receiverKey = rdeReceiverKey.get();
+      } else {
+        signingKey = brdaSigningKey.get();
+        receiverKey = brdaReceiverKey.get();
+      }
       try (OutputStream rydeOutput = Files.newOutputStream(rydePath);
           OutputStream sigOutput = Files.newOutputStream(sigPath);
-          RydeEncoder rydeEncoder = new RydeEncoder.Builder()
-              .setRydeOutput(rydeOutput, rdeReceiverKey.get())
-              .setSignatureOutput(sigOutput, signingKey)
-              .setFileMetadata(name, Files.size(xmlFile), watermark)
-              .build()) {
+          RydeEncoder rydeEncoder =
+              new RydeEncoder.Builder()
+                  .setRydeOutput(rydeOutput, receiverKey)
+                  .setSignatureOutput(sigOutput, signingKey)
+                  .setFileMetadata(name, Files.size(xmlFile), watermark)
+                  .build()) {
         ByteStreams.copy(xmlInput, rydeEncoder);
       }
       try (OutputStream pubOutput = Files.newOutputStream(pubPath);

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -15,14 +15,8 @@
 package google.registry.tools;
 
 import com.google.common.collect.ImmutableMap;
-import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
-import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
-import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.tools.javascrap.HardDeleteHostCommand;
-import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
-import google.registry.tools.javascrap.RemoveIpAddressCommand;
-import google.registry.tools.javascrap.ResaveAllTldsCommand;
 
 /** Container class to create and run remote commands against a Datastore instance. */
 public final class RegistryTool {
@@ -36,8 +30,6 @@ public final class RegistryTool {
   public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
       new ImmutableMap.Builder<String, Class<? extends Command>>()
           .put("ack_poll_messages", AckPollMessagesCommand.class)
-          .put("backfill_registry_locks", BackfillRegistryLocksCommand.class)
-          .put("backfill_spec11_threat_matches", BackfillSpec11ThreatMatchesCommand.class)
           .put("canonicalize_labels", CanonicalizeLabelsCommand.class)
           .put("check_domain", CheckDomainCommand.class)
           .put("check_domain_claims", CheckDomainClaimsCommand.class)
@@ -57,7 +49,6 @@ public final class RegistryTool {
           .put("curl", CurlCommand.class)
           .put("dedupe_one_time_billing_event_ids", DedupeOneTimeBillingEventIdsCommand.class)
           .put("delete_allocation_tokens", DeleteAllocationTokensCommand.class)
-          .put("delete_contact_by_roid", DeleteContactByRoidCommand.class)
           .put("delete_domain", DeleteDomainCommand.class)
           .put("delete_host", DeleteHostCommand.class)
           .put("delete_premium_list", DeletePremiumListCommand.class)
@@ -107,12 +98,9 @@ public final class RegistryTool {
           .put("login", LoginCommand.class)
           .put("logout", LogoutCommand.class)
           .put("pending_escrow", PendingEscrowCommand.class)
-          .put("populate_null_registrar_fields", PopulateNullRegistrarFieldsCommand.class)
           .put("registrar_contact", RegistrarContactCommand.class)
-          .put("remove_ip_address", RemoveIpAddressCommand.class)
           .put("remove_registry_one_key", RemoveRegistryOneKeyCommand.class)
           .put("renew_domain", RenewDomainCommand.class)
-          .put("resave_all_tlds", ResaveAllTldsCommand.class)
           .put("resave_entities", ResaveEntitiesCommand.class)
           .put("resave_environment_entities", ResaveEnvironmentEntitiesCommand.class)
           .put("resave_epp_resource", ResaveEppResourceCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -42,9 +42,7 @@ import google.registry.request.Modules.URLFetchServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
-import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
-import google.registry.tools.javascrap.DeleteContactByRoidCommand;
 import google.registry.tools.javascrap.HardDeleteHostCommand;
 import google.registry.util.UtilsModule;
 import google.registry.whois.NonCachingWhoisModule;
@@ -90,8 +88,6 @@ import javax.inject.Singleton;
 interface RegistryToolComponent {
   void inject(AckPollMessagesCommand command);
 
-  void inject(BackfillRegistryLocksCommand command);
-
   void inject(CheckDomainClaimsCommand command);
 
   void inject(CheckDomainCommand command);
@@ -112,8 +108,6 @@ interface RegistryToolComponent {
 
   void inject(CreateTldCommand command);
 
-  void inject(DeleteContactByRoidCommand command);
-
   void inject(EncryptEscrowDepositCommand command);
 
   void inject(EnqueuePollMessageCommand command);

core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java

@@ -1,157 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
-import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.flogger.FluentLogger;
-import google.registry.config.RegistryConfig.Config;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.RegistryLock;
-import google.registry.model.reporting.HistoryEntry;
-import google.registry.model.reporting.HistoryEntryDao;
-import google.registry.model.tld.RegistryLockDao;
-import google.registry.persistence.VKey;
-import google.registry.tools.CommandWithRemoteApi;
-import google.registry.tools.ConfirmingCommand;
-import google.registry.util.Clock;
-import google.registry.util.StringGenerator;
-import java.util.Comparator;
-import java.util.List;
-import javax.inject.Inject;
-import javax.inject.Named;
-import org.joda.time.DateTime;
-
-/**
- * Scrap tool to backfill {@link RegistryLock}s for domains previously locked.
- *
- * <p>This will save new objects for all existing domains that are locked but don't have any
- * corresponding lock objects already in the database.
- */
-@Parameters(
-    separators = " =",
-    commandDescription =
-        "Backfills RegistryLock objects for specified domain resource IDs that are locked but don't"
-            + " already have a corresponding RegistryLock object.")
-public class BackfillRegistryLocksCommand extends ConfirmingCommand
-    implements CommandWithRemoteApi {
-
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final int VERIFICATION_CODE_LENGTH = 32;
-
-  @Parameter(
-      names = {"--domain_roids"},
-      description = "Comma-separated list of domain roids to check")
-  protected List<String> roids;
-
-  // Inject here so that we can create the command automatically for tests
-  @Inject Clock clock;
-
-  @Inject
-  @Config("registryAdminClientId")
-  String registryAdminClientId;
-
-  @Inject
-  @Named("base58StringGenerator")
-  StringGenerator stringGenerator;
-
-  private ImmutableList<DomainBase> lockedDomains;
-
-  @Override
-  protected String prompt() {
-    checkArgument(
-        roids != null && !roids.isEmpty(), "Must provide non-empty domain_roids argument");
-    lockedDomains =
-        jpaTm().transact(() -> getLockedDomainsWithoutLocks(jpaTm().getTransactionTime()));
-    ImmutableList<String> lockedDomainNames =
-        lockedDomains.stream().map(DomainBase::getDomainName).collect(toImmutableList());
-    return String.format(
-        "Locked domains for which there does not exist a RegistryLock object: %s",
-        lockedDomainNames);
-  }
-
-  @Override
-  protected String execute() {
-    ImmutableSet.Builder<String> failedDomainsBuilder = new ImmutableSet.Builder<>();
-    jpaTm()
-        .transact(
-            () -> {
-              for (DomainBase domainBase : lockedDomains) {
-                try {
-                  RegistryLockDao.save(
-                      new RegistryLock.Builder()
-                          .isSuperuser(true)
-                          .setRegistrarId(registryAdminClientId)
-                          .setRepoId(domainBase.getRepoId())
-                          .setDomainName(domainBase.getDomainName())
-                          .setLockCompletionTime(
-                              getLockCompletionTimestamp(domainBase, jpaTm().getTransactionTime()))
-                          .setVerificationCode(
-                              stringGenerator.createString(VERIFICATION_CODE_LENGTH))
-                          .build());
-                } catch (Throwable t) {
-                  logger.atSevere().withCause(t).log(
-                      "Error when creating lock object for domain '%s'.",
-                      domainBase.getDomainName());
-                  failedDomainsBuilder.add(domainBase.getDomainName());
-                }
-              }
-            });
-    ImmutableSet<String> failedDomains = failedDomainsBuilder.build();
-    if (failedDomains.isEmpty()) {
-      return String.format(
-          "Successfully created lock objects for %d domains.", lockedDomains.size());
-    } else {
-      return String.format(
-          "Successfully created lock objects for %d domains. We failed to create locks "
-              + "for the following domains: %s",
-          lockedDomains.size() - failedDomains.size(), failedDomains);
-    }
-  }
-
-  private DateTime getLockCompletionTimestamp(DomainBase domainBase, DateTime now) {
-    // Best-effort, if a domain was URS-locked we should use that time
-    // If we can't find that, return now.
-    return HistoryEntryDao.loadHistoryObjectsForResource(domainBase.createVKey()).stream()
-        // sort by modification time descending so we get the most recent one if it was locked twice
-        .sorted(Comparator.comparing(HistoryEntry::getModificationTime).reversed())
-        .filter(entry -> "Uniform Rapid Suspension".equals(entry.getReason()))
-        .findFirst()
-        .map(HistoryEntry::getModificationTime)
-        .orElse(now);
-  }
-
-  private ImmutableList<DomainBase> getLockedDomainsWithoutLocks(DateTime now) {
-    ImmutableList<VKey<DomainBase>> domainKeys =
-        roids.stream().map(roid -> VKey.create(DomainBase.class, roid)).collect(toImmutableList());
-    ImmutableCollection<DomainBase> domains =
-        transactIfJpaTm(() -> tm().loadByKeys(domainKeys)).values();
-    return domains.stream()
-        .filter(d -> d.getDeletionTime().isAfter(now))
-        .filter(d -> d.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES))
-        .filter(d -> !RegistryLockDao.getMostRecentByRepoId(d.getRepoId()).isPresent())
-        .collect(toImmutableList());
-  }
-}

core/src/main/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommand.java

@@ -1,223 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableListMultimap.flatteningToImmutableListMultimap;
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.ImmutableSet;
-import google.registry.beam.spec11.ThreatMatch;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.reporting.Spec11ThreatMatch;
-import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
-import google.registry.model.reporting.Spec11ThreatMatchDao;
-import google.registry.persistence.transaction.QueryComposer;
-import google.registry.reporting.spec11.RegistrarThreatMatches;
-import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
-import google.registry.tools.CommandWithRemoteApi;
-import google.registry.tools.ConfirmingCommand;
-import google.registry.util.Clock;
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.function.Function;
-import javax.inject.Inject;
-import org.joda.time.LocalDate;
-
-/**
- * Scrap tool to backfill {@link Spec11ThreatMatch} objects from prior days.
- *
- * <p>This will load the previously-existing Spec11 files from GCS (looking back to 2019-01-01 (a
- * rough estimate of when we started using this format) and convert those RegistrarThreatMatches
- * objects into the new Spec11ThreatMatch format. It will then insert these entries into SQL.
- *
- * <p>Note that the script will attempt to find the corresponding {@link DomainBase} object for each
- * domain name on the day of the scan. It will fail if it cannot find a corresponding domain object,
- * or if the domain objects were not active at the time of the scan.
- */
-@Parameters(
-    commandDescription =
-        "Backfills Spec11 threat match entries from the old and deprecated GCS JSON files to the "
-            + "Cloud SQL database.")
-public class BackfillSpec11ThreatMatchesCommand extends ConfirmingCommand
-    implements CommandWithRemoteApi {
-
-  private static final LocalDate START_DATE = new LocalDate(2019, 1, 1);
-
-  @Parameter(
-      names = {"-o", "--overwrite_existing_dates"},
-      description =
-          "Whether the command will overwrite data that already exists for dates that exist in the "
-              + "GCS bucket. Defaults to false.")
-  private boolean overrideExistingDates;
-
-  @Inject Spec11RegistrarThreatMatchesParser threatMatchesParser;
-  // Inject the clock for testing purposes
-  @Inject Clock clock;
-
-  @Override
-  protected String prompt() {
-    return String.format("Backfill Spec11 results from %d files?", getDatesToBackfill().size());
-  }
-
-  @Override
-  protected String execute() {
-    ImmutableList<LocalDate> dates = getDatesToBackfill();
-    ImmutableListMultimap.Builder<LocalDate, RegistrarThreatMatches> threatMatchesBuilder =
-        new ImmutableListMultimap.Builder<>();
-    for (LocalDate date : dates) {
-      try {
-        // It's OK if the file doesn't exist for a particular date; the result will be empty.
-        threatMatchesBuilder.putAll(date, threatMatchesParser.getRegistrarThreatMatches(date));
-      } catch (IOException e) {
-        throw new RuntimeException(
-            String.format("Error parsing through file with date %s.", date), e);
-      }
-    }
-    ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatches =
-        threatMatchesBuilder.build();
-    // Look up all possible DomainBases for these domain names, any of which can be in the past
-    ImmutableListMultimap<String, DomainBase> domainsByDomainName =
-        getDomainsByDomainName(threatMatches);
-
-    // For each date, convert all threat matches with the proper domain repo ID
-    int totalNumThreats = 0;
-    for (LocalDate date : threatMatches.keySet()) {
-      ImmutableList.Builder<Spec11ThreatMatch> spec11ThreatsBuilder = new ImmutableList.Builder<>();
-      for (RegistrarThreatMatches rtm : threatMatches.get(date)) {
-        rtm.threatMatches().stream()
-            .map(
-                threatMatch ->
-                    threatMatchToCloudSqlObject(
-                        threatMatch, date, rtm.clientId(), domainsByDomainName))
-            .forEach(spec11ThreatsBuilder::add);
-      }
-      ImmutableList<Spec11ThreatMatch> spec11Threats = spec11ThreatsBuilder.build();
-      jpaTm()
-          .transact(
-              () -> {
-                Spec11ThreatMatchDao.deleteEntriesByDate(jpaTm(), date);
-                jpaTm().putAll(spec11Threats);
-              });
-      totalNumThreats += spec11Threats.size();
-    }
-    return String.format(
-        "Successfully parsed through %d files with %d threats.", dates.size(), totalNumThreats);
-  }
-
-  /** Returns a per-domain list of possible DomainBase objects, starting with the most recent. */
-  private ImmutableListMultimap<String, DomainBase> getDomainsByDomainName(
-      ImmutableListMultimap<LocalDate, RegistrarThreatMatches> threatMatchesByDate) {
-    return threatMatchesByDate.values().stream()
-        .map(RegistrarThreatMatches::threatMatches)
-        .flatMap(ImmutableList::stream)
-        .map(ThreatMatch::fullyQualifiedDomainName)
-        .distinct()
-        .collect(
-            flatteningToImmutableListMultimap(
-                Function.identity(),
-                (domainName) -> {
-                  ImmutableList<DomainBase> domains = loadDomainsForFqdn(domainName);
-                  checkState(
-                      !domains.isEmpty(),
-                      "Domain name %s had no associated DomainBase objects.",
-                      domainName);
-                  return domains.stream()
-                      .sorted(Comparator.comparing(DomainBase::getCreationTime).reversed());
-                }));
-  }
-
-  /** Loads in all {@link DomainBase} objects for a given FQDN. */
-  private ImmutableList<DomainBase> loadDomainsForFqdn(String fullyQualifiedDomainName) {
-    return transactIfJpaTm(
-        () ->
-            tm().createQueryComposer(DomainBase.class)
-                .where(
-                    "fullyQualifiedDomainName",
-                    QueryComposer.Comparator.EQ,
-                    fullyQualifiedDomainName)
-                .list());
-  }
-
-  /** Converts the previous {@link ThreatMatch} object to {@link Spec11ThreatMatch}. */
-  private Spec11ThreatMatch threatMatchToCloudSqlObject(
-      ThreatMatch threatMatch,
-      LocalDate date,
-      String registrarId,
-      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
-    DomainBase domain =
-        findDomainAsOfDateOrThrow(
-            threatMatch.fullyQualifiedDomainName(), date, domainsByDomainName);
-    return new Spec11ThreatMatch.Builder()
-        .setThreatTypes(ImmutableSet.of(ThreatType.valueOf(threatMatch.threatType())))
-        .setCheckDate(date)
-        .setRegistrarId(registrarId)
-        .setDomainName(threatMatch.fullyQualifiedDomainName())
-        .setDomainRepoId(domain.getRepoId())
-        .build();
-  }
-
-  /** Returns the DomainBase object as of the particular date, which is likely in the past. */
-  private DomainBase findDomainAsOfDateOrThrow(
-      String domainName,
-      LocalDate date,
-      ImmutableListMultimap<String, DomainBase> domainsByDomainName) {
-    ImmutableList<DomainBase> domains = domainsByDomainName.get(domainName);
-    for (DomainBase domain : domains) {
-      // We only know the date (not datetime) of the threat scan, so we approximate
-      LocalDate creationDate = domain.getCreationTime().toLocalDate();
-      LocalDate deletionDate = domain.getDeletionTime().toLocalDate();
-      if (!date.isBefore(creationDate) && !date.isAfter(deletionDate)) {
-        return domain;
-      }
-    }
-    throw new IllegalStateException(
-        String.format("Could not find a DomainBase valid for %s on day %s.", domainName, date));
-  }
-
-  /** Returns the list of dates between {@link #START_DATE} and now (UTC), inclusive. */
-  private ImmutableList<LocalDate> getDatesToBackfill() {
-    ImmutableSet<LocalDate> datesToSkip =
-        overrideExistingDates ? ImmutableSet.of() : getExistingDates();
-    ImmutableList.Builder<LocalDate> result = new ImmutableList.Builder<>();
-    LocalDate endDate = clock.nowUtc().toLocalDate();
-    for (LocalDate currentDate = START_DATE;
-        !currentDate.isAfter(endDate);
-        currentDate = currentDate.plusDays(1)) {
-      if (!datesToSkip.contains(currentDate)) {
-        result.add(currentDate);
-      }
-    }
-    return result.build();
-  }
-
-  private ImmutableSet<LocalDate> getExistingDates() {
-    return jpaTm()
-        .transact(
-            () ->
-                jpaTm()
-                    .query(
-                        "SELECT DISTINCT stm.checkDate FROM Spec11ThreatMatch stm", LocalDate.class)
-                    .getResultStream()
-                    .collect(toImmutableSet()));
-  }
-}

core/src/main/java/google/registry/tools/javascrap/DeleteContactByRoidCommand.java

@@ -1,115 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.Verify.verify;
-import static google.registry.model.ofy.ObjectifyService.auditedOfy;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.googlecode.objectify.Key;
-import google.registry.model.contact.ContactResource;
-import google.registry.model.eppcommon.StatusValue;
-import google.registry.model.index.EppResourceIndex;
-import google.registry.model.index.ForeignKeyIndex;
-import google.registry.tools.CommandWithRemoteApi;
-import google.registry.tools.ConfirmingCommand;
-import google.registry.util.SystemClock;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * Deletes a {@link google.registry.model.contact.ContactResource} by its ROID.
- *
- * <p>This is a short-term tool for race condition clean up while the bug is being fixed.
- */
-@Parameters(separators = " =", commandDescription = "Delete a contact by its ROID.")
-public class DeleteContactByRoidCommand extends ConfirmingCommand implements CommandWithRemoteApi {
-
-  @Parameter(names = "--roid", description = "The roid of the contact to be deleted.")
-  String roid;
-
-  @Parameter(
-      names = "--contact_id",
-      description = "The user provided contactId, for verification purpose.")
-  String contactId;
-
-  ImmutableList<Key<?>> toDelete;
-
-  @Override
-  protected void init() {
-    System.out.printf("Deleting %s, which refers to %s.\n", roid, contactId);
-    tm().transact(
-            () -> {
-              Key<ContactResource> targetKey = Key.create(ContactResource.class, roid);
-              ContactResource targetContact = auditedOfy().load().key(targetKey).now();
-              verify(
-                  Objects.equals(targetContact.getContactId(), contactId),
-                  "contactId does not match.");
-              verify(
-                  Objects.equals(targetContact.getStatusValues(), ImmutableSet.of(StatusValue.OK)));
-              System.out.println("Target contact has the expected contactId");
-              String canonicalResource =
-                  ForeignKeyIndex.load(ContactResource.class, contactId, new SystemClock().nowUtc())
-                      .getResourceKey()
-                      .getOfyKey()
-                      .getName();
-              verify(!Objects.equals(canonicalResource, roid), "Contact still in ForeignKeyIndex.");
-              System.out.printf(
-                  "It is safe to delete %s, since the contactId is mapped to a different entry in"
-                      + " the Foreign key index (%s).\n\n",
-                  roid, canonicalResource);
-
-              List<Object> ancestors =
-                  auditedOfy().load().ancestor(Key.create(ContactResource.class, roid)).list();
-
-              System.out.println("Ancestor query returns: ");
-              for (Object entity : ancestors) {
-                System.out.println(Key.create(entity));
-              }
-
-              ImmutableSet<String> deletetableKinds =
-                  ImmutableSet.of("HistoryEntry", "ContactResource");
-              toDelete =
-                  ancestors.stream()
-                      .map(Key::create)
-                      .filter(key -> deletetableKinds.contains(key.getKind()))
-                      .collect(ImmutableList.toImmutableList());
-
-              EppResourceIndex eppResourceIndex =
-                  auditedOfy().load().entity(EppResourceIndex.create(targetKey)).now();
-              verify(eppResourceIndex.getKey().equals(targetKey), "Wrong EppResource Index loaded");
-              System.out.printf("\n\nEppResourceIndex found (%s).\n", Key.create(eppResourceIndex));
-
-              toDelete =
-                  new ImmutableList.Builder<Key<?>>()
-                      .addAll(toDelete)
-                      .add(Key.create(eppResourceIndex))
-                      .build();
-
-              System.out.printf("\n\nAbout to delete %s entities:\n", toDelete.size());
-              toDelete.forEach(System.out::println);
-            });
-  }
-
-  @Override
-  protected String execute() {
-    tm().transact(() -> auditedOfy().delete().keys(toDelete).now());
-    return "Done";
-  }
-}

core/src/main/java/google/registry/tools/javascrap/PopulateNullRegistrarFieldsCommand.java

@@ -1,70 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.base.MoreObjects.firstNonNull;
-
-import com.beust.jcommander.Parameters;
-import com.google.common.collect.ImmutableList;
-import google.registry.model.registrar.Registrar;
-import google.registry.model.registrar.RegistrarAddress;
-import google.registry.tools.MutatingCommand;
-import java.util.Objects;
-
-/**
- * Scrap tool to update Registrars with null registrarName or localizedAddress fields.
- *
- * <p>This sets a null registrarName to the key name, and null localizedAddress fields to fake data.
- */
-@Parameters(
-  separators = " =",
-  commandDescription = "Populate previously null required registrar fields."
-)
-public class PopulateNullRegistrarFieldsCommand extends MutatingCommand {
-
-  @Override
-  protected void init() {
-    for (Registrar registrar : Registrar.loadAll()) {
-      Registrar.Builder changeBuilder = registrar.asBuilder();
-      changeBuilder.setRegistrarName(
-          firstNonNull(registrar.getRegistrarName(), registrar.getRegistrarId()));
-
-      RegistrarAddress address = registrar.getLocalizedAddress();
-      if (address == null) {
-        changeBuilder.setLocalizedAddress(
-            new RegistrarAddress.Builder()
-                .setCity("Fakington")
-                .setCountryCode("US")
-                .setState("FL")
-                .setZip("12345")
-                .setStreet(ImmutableList.of("123 Fake Street"))
-                .build());
-      } else {
-        changeBuilder.setLocalizedAddress(
-            new RegistrarAddress.Builder()
-                .setCity(firstNonNull(address.getCity(), "Fakington"))
-                .setCountryCode(firstNonNull(address.getCountryCode(), "US"))
-                .setState(firstNonNull(address.getState(), "FL"))
-                .setZip(firstNonNull(address.getZip(), "12345"))
-                .setStreet(firstNonNull(address.getStreet(), ImmutableList.of("123 Fake Street")))
-                .build());
-      }
-      Registrar changedRegistrar = changeBuilder.build();
-      if (!Objects.equals(registrar, changedRegistrar)) {
-        stageEntityChange(registrar, changedRegistrar);
-      }
-    }
-  }
-}

core/src/main/java/google/registry/tools/javascrap/RemoveIpAddressCommand.java

@@ -1,88 +0,0 @@
-// Copyright 2017 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import com.google.template.soy.data.SoyMapData;
-import google.registry.model.host.HostResource;
-import google.registry.persistence.VKey;
-import google.registry.tools.MutatingEppToolCommand;
-import google.registry.tools.params.PathParameter;
-import google.registry.tools.soy.RemoveIpAddressSoyInfo;
-import java.net.Inet6Address;
-import java.net.InetAddress;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-/**
- * Command to remove external IP Addresses from HostResources identified by text file listing
- * resource ids, one per line.
- *
- * <p>Written for b/23757755 so we can clean up records with IP addresses that should always be
- * resolved by hostname.
- *
- * <p>The JSON file should contain a list of objects each of which has a "roid" attribute.
- */
-@Parameters(separators = " =", commandDescription = "Remove all IP Addresses.")
-public class RemoveIpAddressCommand extends MutatingEppToolCommand {
-  public static String registrarId = "CharlestonRoad";
-
-  @Parameter(names = "--roids_file",
-             description = "Text file containing a list of HostResource roids to remove",
-             required = true,
-             validateWith = PathParameter.InputFile.class)
-  private Path roidsFilePath;
-
-  @Override
-  protected void initMutatingEppToolCommand() throws Exception {
-    List<String> roids = Files.readAllLines(roidsFilePath, UTF_8);
-
-    for (String roid : roids) {
-      // Look up the HostResource from its roid.
-      Optional<HostResource> host =
-          transactIfJpaTm(() -> tm().loadByKeyIfPresent(VKey.create(HostResource.class, roid)));
-      if (!host.isPresent()) {
-        System.err.printf("Record for %s not found.\n", roid);
-        continue;
-      }
-
-      ArrayList<SoyMapData> ipAddresses = new ArrayList<>();
-      for (InetAddress address : host.get().getInetAddresses()) {
-        SoyMapData dataMap = new SoyMapData(
-            "address", address.getHostAddress(),
-            "version", address instanceof Inet6Address ? "v6" : "v4");
-        ipAddresses.add(dataMap);
-      }
-
-      // Build and execute the EPP command.
-      setSoyTemplate(
-          RemoveIpAddressSoyInfo.getInstance(), RemoveIpAddressSoyInfo.REMOVE_IP_ADDRESS);
-      addSoyRecord(
-          registrarId,
-          new SoyMapData(
-              "name", host.get().getHostName(),
-              "ipAddresses", ipAddresses,
-              "requestedByRegistrar", registrarId));
-    }
-  }
-}

core/src/main/java/google/registry/tools/javascrap/ResaveAllTldsCommand.java

@@ -1,30 +0,0 @@
-// Copyright 2021 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
-
-import com.beust.jcommander.Parameters;
-import google.registry.model.tld.Registry;
-import google.registry.tools.CommandWithRemoteApi;
-
-/** Scrap command to resave all Registry entities. */
-@Parameters(commandDescription = "Resave all TLDs")
-public class ResaveAllTldsCommand implements CommandWithRemoteApi {
-  @Override
-  public void run() throws Exception {
-    tm().transact(() -> tm().putAll(tm().loadAllOf(Registry.class)));
-  }
-}

core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json

@@ -69,6 +69,12 @@
       "regexes": [
         "^DATASTORE|CLOUD_SQL$"
       ]
+    },
+    {
+      "name": "jpaTransactionManagerType",
+      "label": "The type of JPA transaction manager to use if using SQL",
+      "helpText": "The standard SQL instance or a read-only replica may be used",
+      "regexes": ["^REGULAR|READ_ONLY_REPLICA$"]
     }
   ]
 }

core/src/main/resources/google/registry/beam/validate_datastore_pipeline_metadata.json

@@ -0,0 +1,42 @@
+{
+  "name": "Validate Datastore with Cloud SQL",
+  "description": "An Apache Beam batch pipeline that compares Datastore with the primary Cloud SQL database.",
+  "parameters": [
+    {
+      "name": "registryEnvironment",
+      "label": "The Registry environment.",
+      "helpText": "The Registry environment.",
+      "is_optional": false,
+      "regexes": [
+        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
+      ]
+    },
+    {
+      "name": "isolationOverride",
+      "label": "The desired SQL transaction isolation level.",
+      "helpText": "The desired SQL transaction isolation level.",
+      "is_optional": true,
+      "regexes": [
+        "^[0-9A-Z_]+$"
+      ]
+    },
+    {
+      "name": "sqlSnapshotId",
+      "label": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
+      "helpText": "The ID of an exported Cloud SQL (Postgresql) snapshot.",
+      "is_optional": true
+    },
+    {
+      "name": "latestCommitLogTimestamp",
+      "label": "Nomulus CommitLog start time",
+      "helpText": "The latest entity update time allowed for inclusion in validation, in ISO8601 format.",
+      "is_optional": false
+    },
+    {
+      "name": "comparisonStartTimestamp",
+      "label": "Only entities updated at or after this time are included for validation.",
+      "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
+      "is_optional": true
+    }
+  ]
+}

core/src/main/resources/google/registry/beam/validate_sql_pipeline_metadata.json

@@ -0,0 +1,21 @@
+{
+  "name": "Validate Cloud SQL with Datastore being primary",
+  "description": "An Apache Beam batch pipeline that compares Cloud SQL with the primary Datastore.",
+  "parameters": [
+    {
+      "name": "registryEnvironment",
+      "label": "The Registry environment.",
+      "helpText": "The Registry environment.",
+      "is_optional": false,
+      "regexes": [
+        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
+      ]
+    },
+    {
+      "name": "comparisonStartTimestamp",
+      "label": "Only entities updated at or after this time are included for validation.",
+      "helpText": "The earliest entity update time allowed for inclusion in validation, in ISO8601 format.",
+      "is_optional": true
+    }
+  ]
+}

core/src/test/java/google/registry/cron/TldFanoutActionTest.java

@@ -98,11 +98,7 @@ class TldFanoutActionTest {
   }
 
   private void assertTaskWithoutTld() {
-    cloudTasksHelper.assertTasksEnqueued(
-        QUEUE,
-        new TaskMatcher()
-            .url(ENDPOINT)
-            .header("content-type", "application/x-www-form-urlencoded"));
+    cloudTasksHelper.assertTasksEnqueued(QUEUE, new TaskMatcher().url(ENDPOINT));
   }
 
   @Test

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -106,6 +106,7 @@ import google.registry.flows.domain.DomainFlowUtils.FeeDescriptionParseException
 import google.registry.flows.domain.DomainFlowUtils.FeesMismatchException;
 import google.registry.flows.domain.DomainFlowUtils.FeesRequiredDuringEarlyAccessProgramException;
 import google.registry.flows.domain.DomainFlowUtils.FeesRequiredForPremiumNameException;
+import google.registry.flows.domain.DomainFlowUtils.InvalidDsRecordException;
 import google.registry.flows.domain.DomainFlowUtils.InvalidIdnDomainLabelException;
 import google.registry.flows.domain.DomainFlowUtils.InvalidPunycodeException;
 import google.registry.flows.domain.DomainFlowUtils.InvalidTcnIdChecksumException;
@@ -1002,6 +1003,22 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
 
+  @TestOfyAndSql
+  void testFailure_secDnsInvalidDigestType() throws Exception {
+    setEppInput("domain_create_dsdata_bad_digest_types.xml");
+    persistContactsAndHosts();
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
+  @TestOfyAndSql
+  void testFailure_secDnsInvalidAlgorithm() throws Exception {
+    setEppInput("domain_create_dsdata_bad_algorithms.xml");
+    persistContactsAndHosts();
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
   @TestOfyAndSql
   void testFailure_wrongExtension() {
     setEppInput("domain_create_wrong_extension.xml");

core/src/test/java/google/registry/flows/domain/DomainUpdateFlowTest.java

@@ -72,6 +72,7 @@ import google.registry.flows.domain.DomainFlowUtils.DuplicateContactForRoleExcep
 import google.registry.flows.domain.DomainFlowUtils.EmptySecDnsUpdateException;
 import google.registry.flows.domain.DomainFlowUtils.FeesMismatchException;
 import google.registry.flows.domain.DomainFlowUtils.FeesRequiredForNonFreeOperationException;
+import google.registry.flows.domain.DomainFlowUtils.InvalidDsRecordException;
 import google.registry.flows.domain.DomainFlowUtils.LinkedResourceInPendingDeleteProhibitsOperationException;
 import google.registry.flows.domain.DomainFlowUtils.LinkedResourcesDoNotExistException;
 import google.registry.flows.domain.DomainFlowUtils.MaxSigLifeChangeNotSupportedException;
@@ -122,7 +123,7 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, DomainBase> {
 
   private static final DelegationSignerData SOME_DSDATA =
-      DelegationSignerData.create(1, 2, 3, base16().decode("0123"));
+      DelegationSignerData.create(1, 2, 2, base16().decode("0123"));
   private static final ImmutableMap<String, String> OTHER_DSDATA_TEMPLATE_MAP =
       ImmutableMap.of(
           "KEY_TAG", "12346",
@@ -536,7 +537,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "3", "DIGEST", "0123"));
+        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "0123"));
   }
 
   @TestOfyAndSql
@@ -556,8 +557,8 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
         ImmutableSet.of(
-            SOME_DSDATA, DelegationSignerData.create(12346, 2, 3, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "12346", "ALG", "2", "DIGEST_TYPE", "3", "DIGEST", "0123"));
+            SOME_DSDATA, DelegationSignerData.create(12346, 2, 2, base16().decode("0123"))),
+        ImmutableMap.of("KEY_TAG", "12346", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "0123"));
   }
 
   @TestOfyAndSql
@@ -565,8 +566,8 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 8, 3, base16().decode("0123"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "8", "DIGEST_TYPE", "3", "DIGEST", "0123"));
+        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 8, 2, base16().decode("0123"))),
+        ImmutableMap.of("KEY_TAG", "1", "ALG", "8", "DIGEST_TYPE", "2", "DIGEST", "0123"));
   }
 
   @TestOfyAndSql
@@ -583,15 +584,15 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
     doSecDnsSuccessfulTest(
         "domain_update_dsdata_add.xml",
         ImmutableSet.of(SOME_DSDATA),
-        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 2, 3, base16().decode("4567"))),
-        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "3", "DIGEST", "4567"));
+        ImmutableSet.of(SOME_DSDATA, DelegationSignerData.create(1, 2, 2, base16().decode("4567"))),
+        ImmutableMap.of("KEY_TAG", "1", "ALG", "2", "DIGEST_TYPE", "2", "DIGEST", "4567"));
   }
 
   @TestOfyAndSql
   void testSuccess_secDnsAddToMaxRecords() throws Exception {
     ImmutableSet.Builder<DelegationSignerData> builder = new ImmutableSet.Builder<>();
     for (int i = 0; i < 7; ++i) {
-      builder.add(DelegationSignerData.create(i, 2, 3, new byte[] {0, 1, 2}));
+      builder.add(DelegationSignerData.create(i, 2, 2, new byte[] {0, 1, 2}));
     }
     ImmutableSet<DelegationSignerData> commonDsData = builder.build();
 
@@ -643,7 +644,7 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
   void testSuccess_secDnsAddRemoveToMaxRecords() throws Exception {
     ImmutableSet.Builder<DelegationSignerData> builder = new ImmutableSet.Builder<>();
     for (int i = 0; i < 7; ++i) {
-      builder.add(DelegationSignerData.create(i, 2, 3, new byte[] {0, 1, 2}));
+      builder.add(DelegationSignerData.create(i, 2, 2, new byte[] {0, 1, 2}));
     }
     ImmutableSet<DelegationSignerData> commonDsData = builder.build();
 
@@ -813,11 +814,69 @@ class DomainUpdateFlowTest extends ResourceFlowTestCase<DomainUpdateFlow, Domain
         MaxSigLifeChangeNotSupportedException.class, "domain_update_maxsiglife.xml");
   }
 
+  @TestOfyAndSql
+  void testFailure_secDnsInvalidDigestType() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
+  @TestOfyAndSql
+  void testFailure_secDnsMultipleInvalidDigestTypes() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(
+                ImmutableSet.of(
+                    DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2}),
+                    DelegationSignerData.create(2, 2, 6, new byte[] {0, 1, 2})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertThat(thrown).hasMessageThat().contains("digestType=3");
+    assertThat(thrown).hasMessageThat().contains("digestType=6");
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
+  @TestOfyAndSql
+  void testFailure_secDnsInvalidAlgorithm() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(ImmutableSet.of(DelegationSignerData.create(1, 99, 2, new byte[] {0, 1, 2})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
+  @TestOfyAndSql
+  void testFailure_secDnsMultipleInvalidAlgorithms() throws Exception {
+    setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);
+    persistResource(
+        newDomainBase(getUniqueIdFromCommand())
+            .asBuilder()
+            .setDsData(
+                ImmutableSet.of(
+                    DelegationSignerData.create(1, 998, 2, new byte[] {0, 1, 2}),
+                    DelegationSignerData.create(2, 99, 2, new byte[] {0, 1, 2})))
+            .build());
+    EppException thrown = assertThrows(InvalidDsRecordException.class, this::runFlow);
+    assertThat(thrown).hasMessageThat().contains("algorithm=998");
+    assertThat(thrown).hasMessageThat().contains("algorithm=99");
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
   @TestOfyAndSql
   void testFailure_secDnsTooManyDsRecords() throws Exception {
     ImmutableSet.Builder<DelegationSignerData> builder = new ImmutableSet.Builder<>();
     for (int i = 0; i < 8; ++i) {
-      builder.add(DelegationSignerData.create(i, 2, 3, new byte[] {0, 1, 2}));
+      builder.add(DelegationSignerData.create(i, 2, 2, new byte[] {0, 1, 2}));
     }
 
     setEppInput("domain_update_dsdata_add.xml", OTHER_DSDATA_TEMPLATE_MAP);

core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtension.java

@@ -217,11 +217,14 @@ abstract class JpaTransactionManagerExtension implements BeforeEachCallback, Aft
     JpaTransactionManagerImpl txnManager = new JpaTransactionManagerImpl(emf, clock);
     cachedTm = TransactionManagerFactory.jpaTm();
     TransactionManagerFactory.setJpaTm(Suppliers.ofInstance(txnManager));
+    TransactionManagerFactory.setReplicaJpaTm(
+        Suppliers.ofInstance(new ReplicaSimulatingJpaTransactionManager(txnManager)));
   }
 
   @Override
   public void afterEach(ExtensionContext context) {
     TransactionManagerFactory.setJpaTm(Suppliers.ofInstance(cachedTm));
+    TransactionManagerFactory.setReplicaJpaTm(Suppliers.ofInstance(cachedTm));
     // Even though we didn't set this, reset it to make sure no other tests are affected
     JpaTransactionManagerImpl.removeReplaySqlToDsOverrideForTest();
     cachedTm = null;

core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerExtensionTest.java

@@ -16,6 +16,7 @@ package google.registry.persistence.transaction;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
 import static google.registry.testing.DatabaseHelper.insertInDb;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -62,6 +63,17 @@ public class JpaTransactionManagerExtensionTest {
             });
   }
 
+  @Test
+  void testReplicaJpaTm() {
+    TestEntity testEntity = new TestEntity("foo", "bar");
+    assertThat(
+            assertThrows(
+                PersistenceException.class,
+                () -> replicaJpaTm().transact(() -> replicaJpaTm().put(testEntity))))
+        .hasMessageThat()
+        .isEqualTo("Error while committing the transaction");
+  }
+
   @Test
   void testExtraParameters() {
     // This test verifies that 1) withEntityClass() has registered TestEntity and 2) The table

core/src/test/java/google/registry/persistence/transaction/ReplicaSimulatingJpaTransactionManager.java

@@ -0,0 +1,357 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.transaction;
+
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.stream.Stream;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
+import javax.persistence.criteria.CriteriaQuery;
+import org.joda.time.DateTime;
+
+/**
+ * A {@link JpaTransactionManager} that simulates a read-only replica SQL instance.
+ *
+ * <p>We accomplish this by delegating all calls to the standard transaction manager except for
+ * calls that start transactions. For these, we create a transaction like normal but set it to READ
+ * ONLY mode before doing any work. This is similar to how the read-only Postgres replica works; it
+ * treats all transactions as read-only transactions.
+ */
+public class ReplicaSimulatingJpaTransactionManager implements JpaTransactionManager {
+
+  private final JpaTransactionManager delegate;
+
+  public ReplicaSimulatingJpaTransactionManager(JpaTransactionManager delegate) {
+    this.delegate = delegate;
+  }
+
+  @Override
+  public void teardown() {
+    delegate.teardown();
+  }
+
+  @Override
+  public EntityManager getStandaloneEntityManager() {
+    return delegate.getStandaloneEntityManager();
+  }
+
+  @Override
+  public EntityManager getEntityManager() {
+    return delegate.getEntityManager();
+  }
+
+  @Override
+  public JpaTransactionManager setDatabaseSnapshot(String snapshotId) {
+    return delegate.setDatabaseSnapshot(snapshotId);
+  }
+
+  @Override
+  public <T> TypedQuery<T> query(String sqlString, Class<T> resultClass) {
+    return delegate.query(sqlString, resultClass);
+  }
+
+  @Override
+  public <T> TypedQuery<T> criteriaQuery(CriteriaQuery<T> criteriaQuery) {
+    return delegate.criteriaQuery(criteriaQuery);
+  }
+
+  @Override
+  public Query query(String sqlString) {
+    return delegate.query(sqlString);
+  }
+
+  @Override
+  public boolean inTransaction() {
+    return delegate.inTransaction();
+  }
+
+  @Override
+  public void assertInTransaction() {
+    delegate.assertInTransaction();
+  }
+
+  @Override
+  public <T> T transact(Supplier<T> work) {
+    if (delegate.inTransaction()) {
+      return work.get();
+    }
+    return delegate.transact(
+        () -> {
+          delegate
+              .getEntityManager()
+              .createNativeQuery("SET TRANSACTION READ ONLY")
+              .executeUpdate();
+          return work.get();
+        });
+  }
+
+  @Override
+  public <T> T transactWithoutBackup(Supplier<T> work) {
+    return transact(work);
+  }
+
+  @Override
+  public <T> T transactNoRetry(Supplier<T> work) {
+    return transact(work);
+  }
+
+  @Override
+  public void transact(Runnable work) {
+    transact(
+        () -> {
+          work.run();
+          return null;
+        });
+  }
+
+  @Override
+  public void transactNoRetry(Runnable work) {
+    transact(work);
+  }
+
+  @Override
+  public <T> T transactNew(Supplier<T> work) {
+    return transact(work);
+  }
+
+  @Override
+  public void transactNew(Runnable work) {
+    transact(work);
+  }
+
+  @Override
+  public <T> T transactNewReadOnly(Supplier<T> work) {
+    return transact(work);
+  }
+
+  @Override
+  public void transactNewReadOnly(Runnable work) {
+    transact(work);
+  }
+
+  @Override
+  public <T> T doTransactionless(Supplier<T> work) {
+    return delegate.doTransactionless(work);
+  }
+
+  @Override
+  public DateTime getTransactionTime() {
+    return delegate.getTransactionTime();
+  }
+
+  @Override
+  public void insert(Object entity) {
+    delegate.insert(entity);
+  }
+
+  @Override
+  public void insertAll(ImmutableCollection<?> entities) {
+    delegate.insertAll(entities);
+  }
+
+  @Override
+  public void insertAll(ImmutableObject... entities) {
+    delegate.insertAll(entities);
+  }
+
+  @Override
+  public void insertWithoutBackup(ImmutableObject entity) {
+    delegate.insertWithoutBackup(entity);
+  }
+
+  @Override
+  public void insertAllWithoutBackup(ImmutableCollection<?> entities) {
+    delegate.insertAllWithoutBackup(entities);
+  }
+
+  @Override
+  public void put(Object entity) {
+    delegate.put(entity);
+  }
+
+  @Override
+  public void putAll(ImmutableObject... entities) {
+    delegate.putAll(entities);
+  }
+
+  @Override
+  public void putAll(ImmutableCollection<?> entities) {
+    delegate.putAll(entities);
+  }
+
+  @Override
+  public void putWithoutBackup(ImmutableObject entity) {
+    delegate.putWithoutBackup(entity);
+  }
+
+  @Override
+  public void putAllWithoutBackup(ImmutableCollection<?> entities) {
+    delegate.putAllWithoutBackup(entities);
+  }
+
+  @Override
+  public void update(Object entity) {
+    delegate.update(entity);
+  }
+
+  @Override
+  public void updateAll(ImmutableCollection<?> entities) {
+    delegate.updateAll(entities);
+  }
+
+  @Override
+  public void updateAll(ImmutableObject... entities) {
+    delegate.updateAll(entities);
+  }
+
+  @Override
+  public void updateWithoutBackup(ImmutableObject entity) {
+    delegate.updateWithoutBackup(entity);
+  }
+
+  @Override
+  public void updateAllWithoutBackup(ImmutableCollection<?> entities) {
+    delegate.updateAllWithoutBackup(entities);
+  }
+
+  @Override
+  public <T> boolean exists(VKey<T> key) {
+    return delegate.exists(key);
+  }
+
+  @Override
+  public boolean exists(Object entity) {
+    return delegate.exists(entity);
+  }
+
+  @Override
+  public <T> Optional<T> loadByKeyIfPresent(VKey<T> key) {
+    return delegate.loadByKeyIfPresent(key);
+  }
+
+  @Override
+  public <T> ImmutableMap<VKey<? extends T>, T> loadByKeysIfPresent(
+      Iterable<? extends VKey<? extends T>> vKeys) {
+    return delegate.loadByKeysIfPresent(vKeys);
+  }
+
+  @Override
+  public <T> ImmutableList<T> loadByEntitiesIfPresent(Iterable<T> entities) {
+    return delegate.loadByEntitiesIfPresent(entities);
+  }
+
+  @Override
+  public <T> T loadByKey(VKey<T> key) {
+    return delegate.loadByKey(key);
+  }
+
+  @Override
+  public <T> ImmutableMap<VKey<? extends T>, T> loadByKeys(
+      Iterable<? extends VKey<? extends T>> vKeys) {
+    return delegate.loadByKeys(vKeys);
+  }
+
+  @Override
+  public <T> T loadByEntity(T entity) {
+    return delegate.loadByEntity(entity);
+  }
+
+  @Override
+  public <T> ImmutableList<T> loadByEntities(Iterable<T> entities) {
+    return delegate.loadByEntities(entities);
+  }
+
+  @Override
+  public <T> ImmutableList<T> loadAllOf(Class<T> clazz) {
+    return delegate.loadAllOf(clazz);
+  }
+
+  @Override
+  public <T> Stream<T> loadAllOfStream(Class<T> clazz) {
+    return delegate.loadAllOfStream(clazz);
+  }
+
+  @Override
+  public <T> Optional<T> loadSingleton(Class<T> clazz) {
+    return delegate.loadSingleton(clazz);
+  }
+
+  @Override
+  public void delete(VKey<?> key) {
+    delegate.delete(key);
+  }
+
+  @Override
+  public void delete(Iterable<? extends VKey<?>> vKeys) {
+    delegate.delete(vKeys);
+  }
+
+  @Override
+  public <T> T delete(T entity) {
+    return delegate.delete(entity);
+  }
+
+  @Override
+  public void deleteWithoutBackup(VKey<?> key) {
+    delegate.deleteWithoutBackup(key);
+  }
+
+  @Override
+  public void deleteWithoutBackup(Iterable<? extends VKey<?>> keys) {
+    delegate.deleteWithoutBackup(keys);
+  }
+
+  @Override
+  public void deleteWithoutBackup(Object entity) {
+    delegate.deleteWithoutBackup(entity);
+  }
+
+  @Override
+  public <T> QueryComposer<T> createQueryComposer(Class<T> entity) {
+    return delegate.createQueryComposer(entity);
+  }
+
+  @Override
+  public void clearSessionCache() {
+    delegate.clearSessionCache();
+  }
+
+  @Override
+  public boolean isOfy() {
+    return delegate.isOfy();
+  }
+
+  @Override
+  public void putIgnoringReadOnlyWithoutBackup(Object entity) {
+    delegate.putIgnoringReadOnlyWithoutBackup(entity);
+  }
+
+  @Override
+  public void deleteIgnoringReadOnlyWithoutBackup(VKey<?> key) {
+    delegate.deleteIgnoringReadOnlyWithoutBackup(key);
+  }
+
+  @Override
+  public <T> void assertDelete(VKey<T> key) {
+    delegate.assertDelete(key);
+  }
+}

core/src/test/java/google/registry/rdap/RdapDomainSearchActionTest.java

@@ -93,39 +93,27 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
   }
 
   private JsonObject generateActualJson(RequestType requestType, String paramValue, String cursor) {
-    action.requestPath = actionPath;
-    action.requestMethod = POST;
-    String requestTypeParam = null;
+    String requestTypeParam;
     switch (requestType) {
       case NAME:
         action.nameParam = Optional.of(paramValue);
-        action.nsLdhNameParam = Optional.empty();
-        action.nsIpParam = Optional.empty();
         requestTypeParam = "name";
         break;
       case NS_LDH_NAME:
-        action.nameParam = Optional.empty();
         action.nsLdhNameParam = Optional.of(paramValue);
-        action.nsIpParam = Optional.empty();
         requestTypeParam = "nsLdhName";
         break;
       case NS_IP:
-        action.nameParam = Optional.empty();
-        action.nsLdhNameParam = Optional.empty();
         action.nsIpParam = Optional.of(paramValue);
         requestTypeParam = "nsIp";
         break;
       default:
-        action.nameParam = Optional.empty();
-        action.nsLdhNameParam = Optional.empty();
-        action.nsIpParam = Optional.empty();
         requestTypeParam = "";
         break;
     }
     if (paramValue != null) {
       if (cursor == null) {
         action.parameterMap = ImmutableListMultimap.of(requestTypeParam, paramValue);
-        action.cursorTokenParam = Optional.empty();
       } else {
         action.parameterMap =
             ImmutableListMultimap.of(requestTypeParam, paramValue, "cursor", cursor);
@@ -381,6 +369,11 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
             clock.nowUtc()));
 
     action.requestMethod = POST;
+    action.nameParam = Optional.empty();
+    action.nsLdhNameParam = Optional.empty();
+    action.nsIpParam = Optional.empty();
+    action.cursorTokenParam = Optional.empty();
+    action.requestPath = actionPath;
   }
 
   private JsonObject generateExpectedJsonForTwoDomainsNsReply() {

core/src/test/java/google/registry/rde/BrdaCopyActionTest.java

@@ -16,6 +16,7 @@ package google.registry.rde;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.SystemInfo.hasCommand;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
@@ -27,6 +28,8 @@ import com.google.common.io.CharStreams;
 import com.google.common.io.Files;
 import google.registry.gcs.GcsUtils;
 import google.registry.keyring.api.Keyring;
+import google.registry.model.rde.RdeMode;
+import google.registry.model.rde.RdeRevision;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.BouncyCastleProviderExtension;
 import google.registry.testing.FakeKeyringModule;
@@ -109,6 +112,10 @@ public class BrdaCopyActionTest {
     action.receiverKey = receiverKey;
     action.signingKey = signingKey;
     action.stagingDecryptionKey = decryptKey;
+    tm().transact(
+            () -> {
+              RdeRevision.saveRevision("lol", DateTime.parse("2010-10-17TZ"), RdeMode.THIN, 0);
+            });
   }
 
   @ParameterizedTest

core/src/test/java/google/registry/rde/RdeStagingActionDatastoreTest.java

@@ -27,7 +27,6 @@ import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.testing.DatabaseHelper.persistResourceWithCommitLog;
 import static google.registry.testing.TaskQueueHelper.assertAtLeastOneTaskIsEnqueued;
 import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.testing.TestDataHelper.loadFile;
 import static google.registry.tldconfig.idn.IdnTableEnum.EXTENDED_LATIN;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -50,17 +49,15 @@ import google.registry.model.ofy.Ofy;
 import google.registry.model.tld.Registry;
 import google.registry.request.HttpException.BadRequestException;
 import google.registry.request.RequestParameters;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeKeyringModule;
 import google.registry.testing.FakeLockHandler;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.InjectExtension;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
 import google.registry.testing.mapreduce.MapreduceTestCase;
 import google.registry.tldconfig.idn.IdnTableEnum;
-import google.registry.util.Retrier;
-import google.registry.util.SystemSleeper;
-import google.registry.util.TaskQueueUtils;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.rde.XjcRdeContentType;
 import google.registry.xjc.rde.XjcRdeDeposit;
@@ -100,10 +97,19 @@ public class RdeStagingActionDatastoreTest extends MapreduceTestCase<RdeStagingA
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
 
-  private final FakeClock clock = new FakeClock();
+  /**
+   * Without autoIncrement mode, the fake clock won't advance between Mapper and Reducer
+   * transactions when action is invoked, resulting in rolled back reducer transaction (due to
+   * TimestampInversionException if both transactions are mapped to the same CommitLog bucket) and
+   * multiple RdeUplaod/BrdaCopy tasks being enqueued (due to transaction retries, since Cloud Tasks
+   * enqueuing is not transactional with Datastore transactions).
+   */
+  private final FakeClock clock = new FakeClock().setAutoIncrementByOneMilli();
+
   private final FakeResponse response = new FakeResponse();
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
   private final List<? super XjcRdeContentType> alreadyExtracted = new ArrayList<>();
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
 
   private static PGPPublicKey encryptKey;
   private static PGPPrivateKey decryptKey;
@@ -124,7 +130,7 @@ public class RdeStagingActionDatastoreTest extends MapreduceTestCase<RdeStagingA
     action.mrRunner = makeDefaultRunner();
     action.lenient = false;
     action.reducerFactory = new RdeStagingReducer.Factory();
-    action.reducerFactory.taskQueueUtils = new TaskQueueUtils(new Retrier(new SystemSleeper(), 1));
+    action.reducerFactory.cloudTasksUtils = cloudTasksHelper.getTestCloudTasksUtils();
     action.reducerFactory.lockHandler = new FakeLockHandler(true);
     action.reducerFactory.bucket = "rde-bucket";
     action.reducerFactory.lockTimeout = Duration.standardHours(1);
@@ -467,11 +473,13 @@ public class RdeStagingActionDatastoreTest extends MapreduceTestCase<RdeStagingA
     clock.setTo(DateTime.parse("2000-01-04TZ")); // Tuesday
     action.run();
     executeTasksUntilEmpty("mapreduce", clock);
-    assertTasksEnqueued("rde-upload",
-        new TaskMatcher()
-            .url(RdeUploadAction.PATH)
-            .param(RequestParameters.PARAM_TLD, "lol"));
-    assertTasksEnqueued("brda",
+    // TODO(b/217773051): duplicate tasks are possible though unlikely. Consider if below calls are
+    // appropriate since they don't allow duplicates.
+    cloudTasksHelper.assertTasksEnqueued(
+        "rde-upload",
+        new TaskMatcher().url(RdeUploadAction.PATH).param(RequestParameters.PARAM_TLD, "lol"));
+    cloudTasksHelper.assertTasksEnqueued(
+        "brda",
         new TaskMatcher()
             .url(BrdaCopyAction.PATH)
             .param(RequestParameters.PARAM_TLD, "lol")

core/src/test/java/google/registry/rde/RdeStagingReducerTest.java

@@ -20,8 +20,6 @@ import static google.registry.model.rde.RdeMode.FULL;
 import static google.registry.model.rde.RdeMode.THIN;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.TaskQueueHelper.assertNoTasksEnqueued;
-import static google.registry.testing.TaskQueueHelper.assertTasksEnqueued;
 import static google.registry.util.ResourceUtils.readResourceUtf8;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -41,13 +39,10 @@ import google.registry.model.rde.RdeRevision;
 import google.registry.model.tld.Registry;
 import google.registry.request.RequestParameters;
 import google.registry.testing.AppEngineExtension;
-import google.registry.testing.FakeClock;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.FakeKeyringModule;
 import google.registry.testing.FakeLockHandler;
-import google.registry.testing.FakeSleeper;
-import google.registry.testing.TaskQueueHelper.TaskMatcher;
-import google.registry.util.Retrier;
-import google.registry.util.TaskQueueUtils;
 import google.registry.xml.ValidationMode;
 import java.io.IOException;
 import java.util.Iterator;
@@ -74,6 +69,7 @@ class RdeStagingReducerTest {
   private static final PGPPublicKey encryptionKey =
       new FakeKeyringModule().get().getRdeStagingEncryptionKey();
   private static final DateTime now = DateTime.parse("2000-01-01TZ");
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper();
 
   private Fragments brdaFragments =
       new Fragments(
@@ -94,7 +90,7 @@ class RdeStagingReducerTest {
 
   private RdeStagingReducer reducer =
       new RdeStagingReducer(
-          new TaskQueueUtils(new Retrier(new FakeSleeper(new FakeClock()), 1)),
+          cloudTasksHelper.getTestCloudTasksUtils(),
           new FakeLockHandler(true),
           GCS_BUCKET,
           Duration.ZERO,
@@ -133,7 +129,7 @@ class RdeStagingReducerTest {
     assertThat(loadCursorTime(CursorType.BRDA))
         .isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
     assertThat(loadRevision(THIN)).isEqualTo(1);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "brda",
         new TaskMatcher()
             .url(BrdaCopyAction.PATH)
@@ -159,7 +155,7 @@ class RdeStagingReducerTest {
     // No extra operations in manual mode.
     assertThat(loadCursorTime(CursorType.BRDA)).isEquivalentAccordingToCompareTo(now);
     assertThat(loadRevision(THIN)).isEqualTo(0);
-    assertNoTasksEnqueued("brda");
+    cloudTasksHelper.assertNoTasksEnqueued("brda");
   }
 
   @Test
@@ -179,7 +175,7 @@ class RdeStagingReducerTest {
     assertThat(loadCursorTime(CursorType.RDE_STAGING))
         .isEquivalentAccordingToCompareTo(now.plus(Duration.standardDays(1)));
     assertThat(loadRevision(FULL)).isEqualTo(1);
-    assertTasksEnqueued(
+    cloudTasksHelper.assertTasksEnqueued(
         "rde-upload",
         new TaskMatcher().url(RdeUploadAction.PATH).param(RequestParameters.PARAM_TLD, "soy"));
   }
@@ -200,7 +196,7 @@ class RdeStagingReducerTest {
     // No extra operations in manual mode.
     assertThat(loadCursorTime(CursorType.RDE_STAGING)).isEquivalentAccordingToCompareTo(now);
     assertThat(loadRevision(FULL)).isEqualTo(0);
-    assertNoTasksEnqueued("rde-upload");
+    cloudTasksHelper.assertNoTasksEnqueued("rde-upload");
   }
 
   private static void compareLength(String outputFile, String lengthFilename) throws IOException {

core/src/test/java/google/registry/testing/CloudTasksHelper.java

@@ -39,6 +39,7 @@ import com.google.common.collect.Multimaps;
 import com.google.common.net.HttpHeaders;
 import com.google.common.net.MediaType;
 import com.google.common.truth.Truth8;
+import com.google.protobuf.Timestamp;
 import google.registry.model.ImmutableObject;
 import google.registry.util.CloudTasksUtils;
 import google.registry.util.Retrier;
@@ -195,6 +196,7 @@ public class CloudTasksHelper implements Serializable {
     // tests.
     HttpMethod method = HttpMethod.POST;
     String url;
+    Timestamp scheduleTime;
     Multimap<String, String> headers = ArrayListMultimap.create();
     Multimap<String, String> params = ArrayListMultimap.create();
 
@@ -216,6 +218,7 @@ public class CloudTasksHelper implements Serializable {
           Ascii.toLowerCase(task.getAppEngineHttpRequest().getAppEngineRouting().getService());
       method = task.getAppEngineHttpRequest().getHttpMethod();
       url = uri.getPath();
+      scheduleTime = task.getScheduleTime();
       ImmutableMultimap.Builder<String, String> headerBuilder = new ImmutableMultimap.Builder<>();
       task.getAppEngineHttpRequest()
           .getHeadersMap()
@@ -227,22 +230,20 @@ public class CloudTasksHelper implements Serializable {
               });
       headers = headerBuilder.build();
       ImmutableMultimap.Builder<String, String> paramBuilder = new ImmutableMultimap.Builder<>();
-      String query = null;
+      // Note that UriParameters.parse() does not throw an IAE on a bad query string (e.g. one
+      // where parameters are not properly URL-encoded); it always does a best-effort parse.
       if (method == HttpMethod.GET) {
-        query = uri.getQuery();
-      } else if (method == HttpMethod.POST) {
+        paramBuilder.putAll(UriParameters.parse(uri.getQuery()));
+      } else if (method == HttpMethod.POST && !task.getAppEngineHttpRequest().getBody().isEmpty()) {
         assertThat(
                 headers.containsEntry(
                     Ascii.toLowerCase(HttpHeaders.CONTENT_TYPE), MediaType.FORM_DATA.toString()))
             .isTrue();
-        query = task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8);
-      }
-      if (query != null) {
-        // Note that UriParameters.parse() does not throw an IAE on a bad query string (e.g. one
-        // where parameters are not properly URL-encoded); it always does a best-effort parse.
-        paramBuilder.putAll(UriParameters.parse(query));
-        params = paramBuilder.build();
+        paramBuilder.putAll(
+            UriParameters.parse(
+                task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8)));
       }
+      params = paramBuilder.build();
     }
 
     public Map<String, Object> toMap() {
@@ -253,6 +254,7 @@ public class CloudTasksHelper implements Serializable {
       builder.put("url", url);
       builder.put("headers", headers);
       builder.put("params", params);
+      builder.put("scheduleTime", scheduleTime);
       return Maps.filterValues(builder, not(in(asList(null, "", Collections.EMPTY_MAP))));
     }
   }
@@ -295,6 +297,11 @@ public class CloudTasksHelper implements Serializable {
       return this;
     }
 
+    public TaskMatcher scheduleTime(Timestamp scheduleTime) {
+      expected.scheduleTime = scheduleTime;
+      return this;
+    }
+
     public TaskMatcher param(String key, String value) {
       checkNotNull(value, "Test error: A param can never have a null value, so don't assert it");
       expected.params.put(key, value);
@@ -318,6 +325,8 @@ public class CloudTasksHelper implements Serializable {
      *
      * <p>Match fails if any headers or params expected on the TaskMatcher are not found on the
      * Task. Note that the inverse is not true (i.e. there may be extra headers on the Task).
+     *
+     * <p>Schedule time by default is Timestamp.getDefaultInstance() or null.
      */
     @Override
     public boolean test(@Nonnull Task task) {
@@ -326,6 +335,8 @@ public class CloudTasksHelper implements Serializable {
           && (expected.url == null || Objects.equals(expected.url, actual.url))
           && (expected.method == null || Objects.equals(expected.method, actual.method))
           && (expected.service == null || Objects.equals(expected.service, actual.service))
+          && (expected.scheduleTime == null
+              || Objects.equals(expected.scheduleTime, actual.scheduleTime))
           && containsEntries(actual.params, expected.params)
           && containsEntries(actual.headers, expected.headers);
     }

core/src/test/java/google/registry/tools/CreateDomainCommandTest.java

@@ -50,7 +50,7 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
         "--admins=crr-admin",
         "--techs=crr-tech",
         "--password=2fooBAR",
-        "--ds_records=1 2 3 abcd,4 5 6 EF01",
+        "--ds_records=1 2 2 abcd,4 5 1 EF01",
         "--ds_records=60485 5  2  D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
         "example.tld");
     eppVerifier.verifySent("domain_create_complete.xml");
@@ -66,7 +66,7 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
         "--admins=crr-admin",
         "--techs=crr-tech",
         "--password=2fooBAR",
-        "--ds_records=1 2 3 abcd,4 5 6 EF01",
+        "--ds_records=1 2 2 abcd,4 5 1 EF01",
         "--ds_records=60485 5  2  D4B7D520E7BB5F0F67674A0CCEB1E3E0614B93C4F9E99B8383F6A1E4469DA50A",
         "example.tld");
     eppVerifier.verifySent("domain_create_complete.xml");
@@ -314,6 +314,38 @@ class CreateDomainCommandTest extends EppToolCommandTestCase<CreateDomainCommand
     assertThat(thrown).hasMessageThat().contains("--period");
   }
 
+  @Test
+  void testFailure_invalidDigestType() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar",
+                    "--registrant=crr-admin",
+                    "--admins=crr-admin",
+                    "--techs=crr-tech",
+                    "--ds_records=1 2 3 abcd",
+                    "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized digest type: 3");
+  }
+
+  @Test
+  void testFailure_invalidAlgorithm() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar",
+                    "--registrant=crr-admin",
+                    "--admins=crr-admin",
+                    "--techs=crr-tech",
+                    "--ds_records=1 999 4 abcd",
+                    "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized algorithm: 999");
+  }
+
   @Test
   void testFailure_dsRecordsNot4Parts() {
     IllegalArgumentException thrown =

core/src/test/java/google/registry/tools/EncryptEscrowDepositCommandTest.java

@@ -40,6 +40,8 @@ public class EncryptEscrowDepositCommandTest
     EscrowDepositEncryptor res = new EscrowDepositEncryptor();
     res.rdeReceiverKey = () -> new FakeKeyringModule().get().getRdeReceiverKey();
     res.rdeSigningKey = () -> new FakeKeyringModule().get().getRdeSigningKey();
+    res.brdaReceiverKey = () -> new FakeKeyringModule().get().getBrdaReceiverKey();
+    res.brdaSigningKey = () -> new FakeKeyringModule().get().getBrdaSigningKey();
     return res;
   }
 
@@ -61,4 +63,34 @@ public class EncryptEscrowDepositCommandTest
             "lol_2010-10-17_full_S1_R0.sig",
             "lol.pub");
   }
+
+  @Test
+  void testSuccess_brda() throws Exception {
+    Path depositFile = tmpDir.resolve("deposit.xml");
+    Files.write(depositXml.read(), depositFile.toFile());
+    runCommand(
+        "--mode=THIN", "--tld=lol", "--input=" + depositFile, "--outdir=" + tmpDir.toString());
+    assertThat(tmpDir.toFile().list())
+        .asList()
+        .containsExactly(
+            "deposit.xml",
+            "lol_2010-10-17_thin_S1_R0.ryde",
+            "lol_2010-10-17_thin_S1_R0.sig",
+            "lol.pub");
+  }
+
+  @Test
+  void testSuccess_revision() throws Exception {
+    Path depositFile = tmpDir.resolve("deposit.xml");
+    Files.write(depositXml.read(), depositFile.toFile());
+    runCommand(
+        "--revision=1", "--tld=lol", "--input=" + depositFile, "--outdir=" + tmpDir.toString());
+    assertThat(tmpDir.toFile().list())
+        .asList()
+        .containsExactly(
+            "deposit.xml",
+            "lol_2010-10-17_full_S1_R1.ryde",
+            "lol_2010-10-17_full_S1_R1.sig",
+            "lol.pub");
+  }
 }

core/src/test/java/google/registry/tools/UpdateDomainCommandTest.java

@@ -86,12 +86,12 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 3 abcd,4 5 6 EF01",
+        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
         "--remove_nameservers=ns3.zdns.google,ns4.zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 9 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld");
@@ -106,12 +106,12 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 3 abcd,4 5 6 EF01",
+        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
         "--remove_nameservers=ns[3-4].zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 9 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld");
@@ -128,12 +128,12 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 3 abcd,4 5 6 EF01",
+        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
         "--remove_nameservers=ns[3-4].zdns.google",
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 9 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
         "--registrant=crr-admin",
         "--password=2fooBAR",
         "example.tld",
@@ -186,7 +186,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--add_admins=crr-admin2",
         "--add_techs=crr-tech2",
         "--add_statuses=serverDeleteProhibited",
-        "--add_ds_records=1 2 3 abcd,4 5 6 EF01",
+        "--add_ds_records=1 2 2 abcd,4 5 1 EF01",
         "example.tld");
     eppVerifier.verifySent("domain_update_add.xml");
   }
@@ -199,7 +199,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
         "--remove_admins=crr-admin1",
         "--remove_techs=crr-tech1",
         "--remove_statuses=serverHold",
-        "--remove_ds_records=7 8 9 12ab,6 5 4 34CD",
+        "--remove_ds_records=7 8 1 12ab,6 5 4 34CD",
         "example.tld");
     eppVerifier.verifySent("domain_update_remove.xml");
   }
@@ -277,8 +277,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
 
   @TestOfyAndSql
   void testSuccess_setDsRecords() throws Exception {
-    runCommandForced(
-        "--client=NewRegistrar", "--ds_records=1 2 3 abcd,4 5 6 EF01", "example.tld");
+    runCommandForced("--client=NewRegistrar", "--ds_records=1 2 2 abcd,4 5 1 EF01", "example.tld");
     eppVerifier.verifySent("domain_update_set_ds_records.xml");
   }
 
@@ -286,7 +285,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
   void testSuccess_setDsRecords_withUnneededClear() throws Exception {
     runCommandForced(
         "--client=NewRegistrar",
-        "--ds_records=1 2 3 abcd,4 5 6 EF01",
+        "--ds_records=1 2 2 abcd,4 5 1 EF01",
         "--clear_ds_records",
         "example.tld");
     eppVerifier.verifySent("domain_update_set_ds_records.xml");
@@ -630,6 +629,28 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
                 + "you cannot use the add_statuses and remove_statuses flags.");
   }
 
+  @TestOfyAndSql
+  void testFailure_invalidDsRecordAlgorithm() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar", "--add_ds_records=1 299 2 abcd", "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized algorithm: 299");
+  }
+
+  @TestOfyAndSql
+  void testFailure_invalidDsRecordDigestType() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--client=NewRegistrar", "--add_ds_records=1 2 3 abcd", "example.tld"));
+    assertThat(thrown).hasMessageThat().isEqualTo("DS record uses an unrecognized digest type: 3");
+  }
+
   @TestOfyAndSql
   void testFailure_provideDsRecordsAndAddDsRecords() {
     IllegalArgumentException thrown =
@@ -638,8 +659,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--add_ds_records=1 2 3 abcd",
-                    "--ds_records=4 5 6 EF01",
+                    "--add_ds_records=1 2 2 abcd",
+                    "--ds_records=4 5 1 EF01",
                     "example.tld"));
     assertThat(thrown)
         .hasMessageThat()
@@ -656,8 +677,8 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--remove_ds_records=7 8 9 12ab",
-                    "--ds_records=4 5 6 EF01",
+                    "--remove_ds_records=7 8 1 12ab",
+                    "--ds_records=4 5 1 EF01",
                     "example.tld"));
     assertThat(thrown)
         .hasMessageThat()
@@ -674,7 +695,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--add_ds_records=1 2 3 abcd",
+                    "--add_ds_records=1 2 2 abcd",
                     "--clear_ds_records",
                     "example.tld"));
     assertThat(thrown)
@@ -692,7 +713,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
             () ->
                 runCommandForced(
                     "--client=NewRegistrar",
-                    "--remove_ds_records=7 8 9 12ab",
+                    "--remove_ds_records=7 8 1 12ab",
                     "--clear_ds_records",
                     "example.tld"));
     assertThat(thrown)

core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java

@@ -1,173 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.truth.Truth.assertThat;
-import static com.google.common.truth.Truth8.assertThat;
-import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.DatabaseHelper.persistActiveDomain;
-import static google.registry.testing.DatabaseHelper.persistDeletedDomain;
-import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
-import static google.registry.testing.DatabaseHelper.persistResource;
-import static google.registry.testing.SqlHelper.getMostRecentRegistryLockByRepoId;
-import static google.registry.testing.SqlHelper.getMostRecentVerifiedRegistryLockByRepoId;
-import static google.registry.testing.SqlHelper.getRegistryLocksByRegistrarId;
-import static google.registry.testing.SqlHelper.saveRegistryLock;
-import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.truth.Truth8;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.domain.DomainHistory;
-import google.registry.model.domain.RegistryLock;
-import google.registry.model.registrar.Registrar;
-import google.registry.model.reporting.HistoryEntry;
-import google.registry.testing.DeterministicStringGenerator;
-import google.registry.testing.DualDatabaseTest;
-import google.registry.testing.TestOfyAndSql;
-import google.registry.tools.CommandTestCase;
-import google.registry.util.StringGenerator.Alphabets;
-import java.util.Optional;
-import org.joda.time.DateTime;
-import org.joda.time.Duration;
-import org.junit.jupiter.api.BeforeEach;
-
-/** Unit tests for {@link BackfillRegistryLocksCommand}. */
-@DualDatabaseTest
-class BackfillRegistryLocksCommandTest extends CommandTestCase<BackfillRegistryLocksCommand> {
-
-  @BeforeEach
-  void beforeEach() {
-    persistNewRegistrar("adminreg", "Admin Registrar", Registrar.Type.REAL, 693L);
-    createTld("tld");
-    command.registryAdminClientId = "adminreg";
-    command.clock = fakeClock;
-    command.stringGenerator = new DeterministicStringGenerator(Alphabets.BASE_58);
-  }
-
-  @TestOfyAndSql
-  void testSimpleBackfill() throws Exception {
-    DomainBase domain = persistLockedDomain("example.tld");
-    Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty();
-
-    runCommandForced("--domain_roids", domain.getRepoId());
-
-    Optional<RegistryLock> lockOptional = getMostRecentRegistryLockByRepoId(domain.getRepoId());
-    Truth8.assertThat(lockOptional).isPresent();
-    Truth8.assertThat(lockOptional.get().getLockCompletionTime()).isPresent();
-  }
-
-  @TestOfyAndSql
-  void testBackfill_onlyLockedDomains() throws Exception {
-    DomainBase neverLockedDomain = persistActiveDomain("neverlocked.tld");
-    DomainBase previouslyLockedDomain = persistLockedDomain("unlocked.tld");
-    persistResource(previouslyLockedDomain.asBuilder().setStatusValues(ImmutableSet.of()).build());
-    DomainBase lockedDomain = persistLockedDomain("locked.tld");
-
-    runCommandForced(
-        "--domain_roids",
-        String.format(
-            "%s,%s,%s",
-            neverLockedDomain.getRepoId(),
-            previouslyLockedDomain.getRepoId(),
-            lockedDomain.getRepoId()));
-
-    ImmutableList<RegistryLock> locks = getRegistryLocksByRegistrarId("adminreg");
-    assertThat(locks).hasSize(1);
-    assertThat(Iterables.getOnlyElement(locks).getDomainName()).isEqualTo("locked.tld");
-  }
-
-  @TestOfyAndSql
-  void testBackfill_skipsDeletedDomains() throws Exception {
-    DomainBase domain = persistDeletedDomain("example.tld", fakeClock.nowUtc());
-    persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
-    fakeClock.advanceBy(Duration.standardSeconds(1));
-    runCommandForced("--domain_roids", domain.getRepoId());
-    Truth8.assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId())).isEmpty();
-  }
-
-  @TestOfyAndSql
-  void testBackfill_skipsDomains_ifLockAlreadyExists() throws Exception {
-    DomainBase domain = persistLockedDomain("example.tld");
-
-    RegistryLock previousLock =
-        saveRegistryLock(
-            new RegistryLock.Builder()
-                .isSuperuser(true)
-                .setRegistrarId("adminreg")
-                .setRepoId(domain.getRepoId())
-                .setDomainName(domain.getDomainName())
-                .setLockCompletionTime(fakeClock.nowUtc())
-                .setVerificationCode(command.stringGenerator.createString(32))
-                .build());
-
-    fakeClock.advanceBy(Duration.standardDays(1));
-    runCommandForced("--domain_roids", domain.getRepoId());
-
-    assertThat(getMostRecentRegistryLockByRepoId(domain.getRepoId()).get().getLockCompletionTime())
-        .isEqualTo(previousLock.getLockCompletionTime());
-  }
-
-  @TestOfyAndSql
-  void testBackfill_usesUrsTime_ifExists() throws Exception {
-    DateTime ursTime = fakeClock.nowUtc();
-    DomainBase ursDomain = persistLockedDomain("urs.tld");
-    persistResource(
-        new DomainHistory.Builder()
-            .setBySuperuser(true)
-            .setRegistrarId("adminreg")
-            .setModificationTime(ursTime)
-            .setDomain(ursDomain)
-            .setReason("Uniform Rapid Suspension")
-            .setType(HistoryEntry.Type.DOMAIN_UPDATE)
-            .setRequestedByRegistrar(false)
-            .build());
-    DomainBase nonUrsDomain = persistLockedDomain("nonurs.tld");
-    persistResource(
-        new DomainHistory.Builder()
-            .setBySuperuser(true)
-            .setRegistrarId("adminreg")
-            .setDomain(nonUrsDomain)
-            .setType(HistoryEntry.Type.DOMAIN_UPDATE)
-            .setRequestedByRegistrar(false)
-            .setModificationTime(ursTime)
-            .build());
-
-    fakeClock.advanceBy(Duration.standardDays(10));
-    runCommandForced(
-        "--domain_roids", String.format("%s,%s", ursDomain.getRepoId(), nonUrsDomain.getRepoId()));
-
-    RegistryLock ursLock = getMostRecentVerifiedRegistryLockByRepoId(ursDomain.getRepoId()).get();
-    assertThat(ursLock.getLockCompletionTime()).hasValue(ursTime);
-    RegistryLock nonUrsLock =
-        getMostRecentVerifiedRegistryLockByRepoId(nonUrsDomain.getRepoId()).get();
-    assertThat(nonUrsLock.getLockCompletionTime()).hasValue(fakeClock.nowUtc());
-  }
-
-  @TestOfyAndSql
-  void testFailure_mustProvideDomainRoids() {
-    assertThat(assertThrows(IllegalArgumentException.class, this::runCommandForced))
-        .hasMessageThat()
-        .isEqualTo("Must provide non-empty domain_roids argument");
-  }
-
-  private static DomainBase persistLockedDomain(String domainName) {
-    DomainBase domain = persistActiveDomain(domainName);
-    return persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build());
-  }
-}

core/src/test/java/google/registry/tools/javascrap/BackfillSpec11ThreatMatchesCommandTest.java

@@ -1,274 +0,0 @@
-// Copyright 2020 The Nomulus Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package google.registry.tools.javascrap;
-
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.truth.Truth.assertThat;
-import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
-import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParserTest.sampleThreatMatches;
-import static google.registry.testing.DatabaseHelper.createTld;
-import static google.registry.testing.DatabaseHelper.deleteResource;
-import static google.registry.testing.DatabaseHelper.insertInDb;
-import static google.registry.testing.DatabaseHelper.newDomainBase;
-import static google.registry.testing.DatabaseHelper.persistActiveDomain;
-import static google.registry.testing.DatabaseHelper.persistResource;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import google.registry.model.domain.DomainBase;
-import google.registry.model.reporting.Spec11ThreatMatch;
-import google.registry.model.reporting.Spec11ThreatMatch.ThreatType;
-import google.registry.reporting.spec11.Spec11RegistrarThreatMatchesParser;
-import google.registry.testing.DualDatabaseTest;
-import google.registry.testing.TestOfyAndSql;
-import google.registry.tools.CommandTestCase;
-import java.io.IOException;
-import org.joda.time.DateTime;
-import org.joda.time.LocalDate;
-import org.junit.jupiter.api.BeforeEach;
-
-/** Tests for {@link BackfillSpec11ThreatMatchesCommand}. */
-@DualDatabaseTest
-public class BackfillSpec11ThreatMatchesCommandTest
-    extends CommandTestCase<BackfillSpec11ThreatMatchesCommand> {
-
-  private static final LocalDate CURRENT_DATE = DateTime.parse("2020-11-22").toLocalDate();
-  private final Spec11RegistrarThreatMatchesParser threatMatchesParser =
-      mock(Spec11RegistrarThreatMatchesParser.class);
-
-  private DomainBase domainA;
-
-  @BeforeEach
-  void beforeEach() throws Exception {
-    createTld("com");
-    domainA = persistActiveDomain("a.com");
-    persistActiveDomain("b.com");
-    persistActiveDomain("c.com");
-    fakeClock.setTo(CURRENT_DATE.toDateTimeAtStartOfDay());
-    command.threatMatchesParser = threatMatchesParser;
-    command.clock = fakeClock;
-    when(threatMatchesParser.getRegistrarThreatMatches(any(LocalDate.class)))
-        .thenReturn(ImmutableSet.of());
-  }
-
-  @TestOfyAndSql
-  void testSuccess_singleFile() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    assertInStdout("Backfill Spec11 results from 692 files?");
-    assertInStdout("Successfully parsed through 692 files with 3 threats.");
-    verifyExactlyThreeEntriesInDbFromLastDay();
-  }
-
-  @TestOfyAndSql
-  void testSuccess_sameDomain_multipleDays() throws Exception {
-    // If the same domains show up on multiple days, there should be multiple entries for them
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    when(threatMatchesParser.getRegistrarThreatMatches(LocalDate.parse("2019-01-01")))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    assertInStdout("Backfill Spec11 results from 692 files?");
-    assertInStdout("Successfully parsed through 692 files with 6 threats.");
-    jpaTm()
-        .transact(
-            () -> {
-              ImmutableList<Spec11ThreatMatch> threatMatches =
-                  jpaTm().loadAllOf(Spec11ThreatMatch.class);
-              assertThat(threatMatches).hasSize(6);
-              assertThat(
-                      threatMatches.stream()
-                          .map(Spec11ThreatMatch::getDomainName)
-                          .collect(toImmutableSet()))
-                  .containsExactly("a.com", "b.com", "c.com");
-              assertThat(
-                      threatMatches.stream()
-                          .map(Spec11ThreatMatch::getCheckDate)
-                          .collect(toImmutableSet()))
-                  .containsExactly(CURRENT_DATE, LocalDate.parse("2019-01-01"));
-            });
-  }
-
-  @TestOfyAndSql
-  void testSuccess_empty() throws Exception {
-    runCommandForced();
-    assertInStdout("Backfill Spec11 results from 692 files?");
-    assertInStdout("Successfully parsed through 692 files with 0 threats.");
-  }
-
-  @TestOfyAndSql
-  void testSuccess_sameDayTwice() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    runCommandForced();
-    verifyExactlyThreeEntriesInDbFromLastDay();
-  }
-
-  @TestOfyAndSql
-  void testSuccess_threeDomainsForDomainName() throws Exception {
-    // We should use the repo ID from the proper DomainBase object at the scan's point in time.
-    // First, domain was created at START_OF_TIME and deleted one year ago
-    DateTime now = fakeClock.nowUtc();
-    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
-
-    // Next, domain was created six months ago and deleted two months ago
-    DomainBase secondSave =
-        persistResource(
-            newDomainBase("a.com")
-                .asBuilder()
-                .setCreationTimeForTest(now.minusMonths(6))
-                .setDeletionTime(now.minusMonths(2))
-                .build());
-
-    // Lastly, domain was created one month ago and is still valid
-    DomainBase thirdSave =
-        persistResource(
-            newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
-
-    // If the scan result was from three months ago, we should use the second save
-    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
-        .thenReturn(sampleThreatMatches());
-    runCommandForced();
-    String threatMatchRepoId =
-        jpaTm()
-            .transact(
-                () ->
-                    jpaTm().loadAllOf(Spec11ThreatMatch.class).stream()
-                        .filter((match) -> match.getDomainName().equals("a.com"))
-                        .findFirst()
-                        .get()
-                        .getDomainRepoId());
-    assertThat(threatMatchRepoId).isNotEqualTo(domainA.getRepoId());
-    assertThat(threatMatchRepoId).isEqualTo(secondSave.getRepoId());
-    assertThat(threatMatchRepoId).isNotEqualTo(thirdSave.getRepoId());
-  }
-
-  @TestOfyAndSql
-  void testSuccess_skipsExistingDatesWithoutOverwrite() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    Spec11ThreatMatch previous =
-        new Spec11ThreatMatch.Builder()
-            .setCheckDate(CURRENT_DATE)
-            .setDomainName("previous.tld")
-            .setDomainRepoId("1-DOMAIN")
-            .setRegistrarId("TheRegistrar")
-            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
-            .build();
-    insertInDb(previous);
-
-    runCommandForced();
-    ImmutableList<Spec11ThreatMatch> threatMatches =
-        jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class));
-    assertAboutImmutableObjects()
-        .that(Iterables.getOnlyElement(threatMatches))
-        .isEqualExceptFields(previous, "id");
-  }
-
-  @TestOfyAndSql
-  void testSuccess_overwritesExistingDatesWhenSpecified() throws Exception {
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    Spec11ThreatMatch previous =
-        new Spec11ThreatMatch.Builder()
-            .setCheckDate(CURRENT_DATE)
-            .setDomainName("previous.tld")
-            .setDomainRepoId("1-DOMAIN")
-            .setRegistrarId("TheRegistrar")
-            .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
-            .build();
-    insertInDb(previous);
-
-    runCommandForced("--overwrite_existing_dates");
-    verifyExactlyThreeEntriesInDbFromLastDay();
-  }
-
-  @TestOfyAndSql
-  void testFailure_oneFileFails() throws Exception {
-    // If there are any exceptions, we should fail loud and fast
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE.minusDays(1)))
-        .thenThrow(new IOException("hi"));
-    RuntimeException runtimeException =
-        assertThrows(RuntimeException.class, this::runCommandForced);
-    assertThat(runtimeException.getCause().getClass()).isEqualTo(IOException.class);
-    assertThat(runtimeException).hasCauseThat().hasMessageThat().isEqualTo("hi");
-    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(Spec11ThreatMatch.class))).isEmpty();
-  }
-
-  @TestOfyAndSql
-  void testFailure_noDomainForDomainName() throws Exception {
-    deleteResource(domainA);
-    when(threatMatchesParser.getRegistrarThreatMatches(CURRENT_DATE))
-        .thenReturn(sampleThreatMatches());
-    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
-        .hasMessageThat()
-        .isEqualTo("Domain name a.com had no associated DomainBase objects.");
-  }
-
-  @TestOfyAndSql
-  void testFailure_noDomainAtTimeOfScan() throws Exception {
-    // If the domain existed at some point(s) in time but not the time of the scan, fail.
-    // First, domain was created at START_OF_TIME and deleted one year ago
-    DateTime now = fakeClock.nowUtc();
-    domainA = persistResource(domainA.asBuilder().setDeletionTime(now.minusYears(1)).build());
-
-    // Second, domain was created one month ago and is still valid
-    persistResource(
-        newDomainBase("a.com").asBuilder().setCreationTimeForTest(now.minusMonths(1)).build());
-
-    // If we have a result for this domain from 3 months ago when it didn't exist, fail.
-    when(threatMatchesParser.getRegistrarThreatMatches(now.toLocalDate().minusMonths(3)))
-        .thenReturn(sampleThreatMatches());
-    assertThat(assertThrows(IllegalStateException.class, this::runCommandForced))
-        .hasMessageThat()
-        .isEqualTo("Could not find a DomainBase valid for a.com on day 2020-08-22.");
-  }
-
-  private void verifyExactlyThreeEntriesInDbFromLastDay() {
-    jpaTm()
-        .transact(
-            () -> {
-              ImmutableList<Spec11ThreatMatch> threatMatches =
-                  jpaTm().loadAllOf(Spec11ThreatMatch.class);
-              assertThat(threatMatches)
-                  .comparingElementsUsing(immutableObjectCorrespondence("id", "domainRepoId"))
-                  .containsExactly(
-                      expectedThreatMatch("TheRegistrar", "a.com"),
-                      expectedThreatMatch("NewRegistrar", "b.com"),
-                      expectedThreatMatch("NewRegistrar", "c.com"));
-            });
-  }
-
-  private Spec11ThreatMatch expectedThreatMatch(String registrarId, String domainName) {
-    return new Spec11ThreatMatch.Builder()
-        .setDomainRepoId("ignored")
-        .setDomainName(domainName)
-        .setRegistrarId(registrarId)
-        .setCheckDate(CURRENT_DATE)
-        .setThreatTypes(ImmutableSet.of(ThreatType.MALWARE))
-        .build();
-  }
-}

core/src/test/resources/google/registry/flows/domain/domain_create_dsdata_bad_algorithms.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <command>
+    <create>
+      <domain:create
+          xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>example.tld</domain:name>
+        <domain:period unit="y">2</domain:period>
+        <domain:ns>
+          <domain:hostObj>ns1.example.net</domain:hostObj>
+          <domain:hostObj>ns2.example.net</domain:hostObj>
+        </domain:ns>
+        <domain:registrant>jd1234</domain:registrant>
+        <domain:contact type="admin">sh8013</domain:contact>
+        <domain:contact type="tech">sh8013</domain:contact>
+        <domain:authInfo>
+          <domain:pw>2fooBAR</domain:pw>
+        </domain:authInfo>
+      </domain:create>
+    </create>
+    <extension>
+      <secDNS:create
+          xmlns:secDNS="urn:ietf:params:xml:ns:secDNS-1.1">
+        <secDNS:dsData>
+          <secDNS:keyTag>12345</secDNS:keyTag>
+          <secDNS:alg>99</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12346</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12347</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12348</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12349</secDNS:keyTag>
+          <secDNS:alg>98</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12350</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12351</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12352</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+      </secDNS:create>
+    </extension>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

core/src/test/resources/google/registry/flows/domain/domain_create_dsdata_bad_digest_types.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <command>
+    <create>
+      <domain:create
+          xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>example.tld</domain:name>
+        <domain:period unit="y">2</domain:period>
+        <domain:ns>
+          <domain:hostObj>ns1.example.net</domain:hostObj>
+          <domain:hostObj>ns2.example.net</domain:hostObj>
+        </domain:ns>
+        <domain:registrant>jd1234</domain:registrant>
+        <domain:contact type="admin">sh8013</domain:contact>
+        <domain:contact type="tech">sh8013</domain:contact>
+        <domain:authInfo>
+          <domain:pw>2fooBAR</domain:pw>
+        </domain:authInfo>
+      </domain:create>
+    </create>
+    <extension>
+      <secDNS:create
+          xmlns:secDNS="urn:ietf:params:xml:ns:secDNS-1.1">
+        <secDNS:dsData>
+          <secDNS:keyTag>12345</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>100</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12346</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>3</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12347</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12348</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12349</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12350</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12351</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+        <secDNS:dsData>
+          <secDNS:keyTag>12352</secDNS:keyTag>
+          <secDNS:alg>3</secDNS:alg>
+          <secDNS:digestType>1</secDNS:digestType>
+          <secDNS:digest>49FD46E6C4B45C55D4AC</secDNS:digest>
+        </secDNS:dsData>
+      </secDNS:create>
+    </extension>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

core/src/test/resources/google/registry/tools/server/domain_create_complete.xml

@@ -25,13 +25,13 @@
         <secDNS:dsData>
           <secDNS:keyTag>1</secDNS:keyTag>
           <secDNS:alg>2</secDNS:alg>
-          <secDNS:digestType>3</secDNS:digestType>
+          <secDNS:digestType>2</secDNS:digestType>
           <secDNS:digest>ABCD</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>
           <secDNS:keyTag>4</secDNS:keyTag>
           <secDNS:alg>5</secDNS:alg>
-          <secDNS:digestType>6</secDNS:digestType>
+          <secDNS:digestType>1</secDNS:digestType>
           <secDNS:digest>EF01</secDNS:digest>
         </secDNS:dsData>
         <secDNS:dsData>

core/src/test/resources/google/registry/tools/server/domain_update_add.xml

@@ -22,13 +22,13 @@
           <secDNS:dsData>
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
-            <secDNS:digestType>3</secDNS:digestType>
+            <secDNS:digestType>2</secDNS:digestType>
             <secDNS:digest>ABCD</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
-            <secDNS:digestType>6</secDNS:digestType>
+            <secDNS:digestType>1</secDNS:digestType>
             <secDNS:digest>EF01</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>

core/src/test/resources/google/registry/tools/server/domain_update_complete.xml

@@ -37,7 +37,7 @@
           <secDNS:dsData>
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
-            <secDNS:digestType>9</secDNS:digestType>
+            <secDNS:digestType>1</secDNS:digestType>
             <secDNS:digest>12AB</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
@@ -51,13 +51,13 @@
           <secDNS:dsData>
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
-            <secDNS:digestType>3</secDNS:digestType>
+            <secDNS:digestType>2</secDNS:digestType>
             <secDNS:digest>ABCD</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
-            <secDNS:digestType>6</secDNS:digestType>
+            <secDNS:digestType>1</secDNS:digestType>
             <secDNS:digest>EF01</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>

core/src/test/resources/google/registry/tools/server/domain_update_complete_abc.xml

@@ -37,7 +37,7 @@
           <secDNS:dsData>
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
-            <secDNS:digestType>9</secDNS:digestType>
+            <secDNS:digestType>1</secDNS:digestType>
             <secDNS:digest>12AB</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
@@ -51,13 +51,13 @@
           <secDNS:dsData>
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
-            <secDNS:digestType>3</secDNS:digestType>
+            <secDNS:digestType>2</secDNS:digestType>
             <secDNS:digest>ABCD</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
-            <secDNS:digestType>6</secDNS:digestType>
+            <secDNS:digestType>1</secDNS:digestType>
             <secDNS:digest>EF01</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>

core/src/test/resources/google/registry/tools/server/domain_update_remove.xml

@@ -21,7 +21,7 @@
           <secDNS:dsData>
             <secDNS:keyTag>7</secDNS:keyTag>
             <secDNS:alg>8</secDNS:alg>
-            <secDNS:digestType>9</secDNS:digestType>
+            <secDNS:digestType>1</secDNS:digestType>
             <secDNS:digest>12AB</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>

core/src/test/resources/google/registry/tools/server/domain_update_set_ds_records.xml

@@ -16,13 +16,13 @@
           <secDNS:dsData>
             <secDNS:keyTag>1</secDNS:keyTag>
             <secDNS:alg>2</secDNS:alg>
-            <secDNS:digestType>3</secDNS:digestType>
+            <secDNS:digestType>2</secDNS:digestType>
             <secDNS:digest>ABCD</secDNS:digest>
           </secDNS:dsData>
           <secDNS:dsData>
             <secDNS:keyTag>4</secDNS:keyTag>
             <secDNS:alg>5</secDNS:alg>
-            <secDNS:digestType>6</secDNS:digestType>
+            <secDNS:digestType>1</secDNS:digestType>
             <secDNS:digest>EF01</secDNS:digest>
           </secDNS:dsData>
         </secDNS:add>

db/src/main/resources/sql/er_diagram/full_er_diagram.html

@@ -261,19 +261,11 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-<<<<<<< HEAD
-     <td class="property_value">2021-11-23 20:43:39.672931</td> 
+     <td class="property_value">2022-02-03 18:23:54.985476</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V104__add_transfer_response_host_id_to_poll_message.sql</td>
-=======
-     <td class="property_value">2021-11-17 21:23:10.198048</td> 
-    </tr> 
-    <tr>
-     <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V103__add_transfer_response_host_id_to_poll_message.sql</td>
->>>>>>> 19aa29939 (Fix missing hostPendingActionNotificationResponses in PollMessage.OneTime)
+     <td id="lastFlywayFile" class="property_value">V105__add_index_on_host_name_in_host_table.sql</td>
     </tr>
    </tbody>
   </table> 
@@ -292,11 +284,7 @@ td.section {
      generated on
     </text> 
     <text text-anchor="start" x="4755.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-<<<<<<< HEAD
-     2021-11-23 20:43:39.672931
-=======
-     2021-11-17 21:23:10.198048
->>>>>>> 19aa29939 (Fix missing hostPendingActionNotificationResponses in PollMessage.OneTime)
+     2022-02-03 18:23:54.985476
     </text> 
     <polygon fill="none" stroke="#888888" points="4668.02,-4 4668.02,-44 4933.02,-44 4933.02,-4 4668.02,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 
@@ -11179,6 +11167,18 @@ td.section {
     <tr> 
      <td colspan="3"></td> 
     </tr> 
+    <tr> 
+     <td colspan="2" class="name">idxkpkh68n6dy5v51047yr6b0e9l</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">host_name</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
     <tr> 
      <td colspan="2" class="name">Host_pkey</td> 
      <td class="description right">[unique index]</td> 

db/src/main/resources/sql/flyway.txt

@@ -102,3 +102,4 @@ V101__domain_add_dns_refresh_request_time.sql
 V102__add_indexes_to_domain_history_sub_tables.sql
 V103__creation_time_not_null.sql
 V104__add_transfer_response_host_id_to_poll_message.sql
+V105__add_index_on_host_name_in_host_table.sql

db/src/main/resources/sql/flyway/V105__add_index_on_host_name_in_host_table.sql

@@ -0,0 +1,15 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IDXkpkh68n6dy5v51047yr6b0e9l ON "Host" (host_name);

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -804,6 +804,7 @@ create index IDX6w3qbtgce93cal2orjg1tw7b7 on "DomainHistory" (history_modificati
        add constraint UKat9erbh52e4lg3jw6ai9wkjj9 unique (domain_repo_id, host_repo_id);
 create index IDXj1mtx98ndgbtb1bkekahms18w on "GracePeriod" (domain_repo_id);
 create index IDXd01j17vrpjxaerxdmn8bwxs7s on "GracePeriodHistory" (domain_repo_id);
+create index IDXkpkh68n6dy5v51047yr6b0e9l on "Host" (host_name);
 create index IDXfg2nnjlujxo6cb9fha971bq2n on "HostHistory" (creation_time);
 create index IDX1iy7njgb7wjmj9piml4l2g0qi on "HostHistory" (history_registrar_id);
 create index IDXkkwbwcwvrdkkqothkiye4jiff on "HostHistory" (host_name);

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -1728,6 +1728,13 @@ CREATE INDEX idxkjt9yaq92876dstimd93hwckh ON public."Domain" USING btree (curren
 CREATE INDEX idxknk8gmj7s47q56cwpa6rmpt5l ON public."HostHistory" USING btree (history_type);
 
 
+--
+-- Name: idxkpkh68n6dy5v51047yr6b0e9l; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idxkpkh68n6dy5v51047yr6b0e9l ON public."Host" USING btree (host_name);
+
+
 --
 -- Name: idxlrq7v63pc21uoh3auq6eybyhl; Type: INDEX; Schema: public; Owner: -
 --

docs/flows.md

@@ -398,6 +398,7 @@ An EPP flow that creates a new domain resource.
     *   The fee description passed in the transform command matches multiple fee
         types.
     *   The fee description passed in the transform command cannot be parsed.
+    *   Domain has an invalid DS record.
     *   Domain name starts with xn-- but is not a valid IDN.
     *   The specified trademark validator is not supported.
     *   Domain labels cannot begin with a dash.
@@ -810,6 +811,7 @@ statuses are updated at once.
 *   2306
     *   Cannot add and remove the same value.
     *   More than one contact for a given role is not allowed.
+    *   Domain has an invalid DS record.
     *   Missing type attribute for contact.
     *   The secDNS:all element must have value 'true' if present.
     *   Too many DS records set on a domain.

release/cloudbuild-delete.yaml

@@ -52,8 +52,8 @@ steps:
     do
       for version in $(gcloud app versions list \
         --filter="SERVICE:$service AND SERVING_STATUS:STOPPED" \
-        --sort-by=LAST_DEPLOYED --format="value(VERSION.ID)" \
-        --project=$project_id | head -n -3)
+        --format="value(VERSION.ID,LAST_DEPLOYED)" \
+        --project=$project_id | sort -k 2 | head -n -3)
       do
         gcloud app versions delete $version --service=$service \
           --project=$project_id --quiet;

release/cloudbuild-nomulus.yaml

@@ -96,7 +96,11 @@ steps:
       google.registry.beam.invoicing.InvoicingPipeline \
       google/registry/beam/invoicing_pipeline_metadata.json \
       google.registry.beam.rde.RdePipeline \
-      google/registry/beam/rde_pipeline_metadata.json
+      google/registry/beam/rde_pipeline_metadata.json \
+      google.registry.beam.comparedb.ValidateDatastorePipeline \
+      google/registry/beam/validate_datastore_pipeline_metadata.json \
+      google.registry.beam.comparedb.ValidateSqlPipeline \
+      google/registry/beam/validate_sql_pipeline_metadata.json
 # Tentatively build and publish Cloud SQL schema jar here, before schema release
 # process is finalized. Also publish nomulus:core jars that are needed for
 # server/schema compatibility tests.

util/src/main/java/google/registry/util/CloudTasksUtils.java

@@ -43,6 +43,7 @@ import java.util.Arrays;
 import java.util.Optional;
 import java.util.Random;
 import java.util.function.Supplier;
+import org.joda.time.Duration;
 
 /** Utilities for dealing with Cloud Tasks. */
 public class CloudTasksUtils implements Serializable {
@@ -106,30 +107,34 @@ public class CloudTasksUtils implements Serializable {
     checkArgument(
         path != null && !path.isEmpty() && path.charAt(0) == '/',
         "The path must start with a '/'.");
+    checkArgument(
+        method.equals(HttpMethod.GET) || method.equals(HttpMethod.POST),
+        "HTTP method %s is used. Only GET and POST are allowed.",
+        method);
     AppEngineHttpRequest.Builder requestBuilder =
         AppEngineHttpRequest.newBuilder()
             .setHttpMethod(method)
             .setAppEngineRouting(AppEngineRouting.newBuilder().setService(service).build());
-    Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
-    String encodedParams =
-        Joiner.on("&")
-            .join(
-                params.entries().stream()
-                    .map(
-                        entry ->
-                            String.format(
-                                "%s=%s",
-                                escaper.escape(entry.getKey()), escaper.escape(entry.getValue())))
-                    .collect(toImmutableList()));
-    if (method == HttpMethod.GET) {
-      path = String.format("%s?%s", path, encodedParams);
-    } else if (method == HttpMethod.POST) {
-      requestBuilder
-          .putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
-          .setBody(ByteString.copyFrom(encodedParams, StandardCharsets.UTF_8));
-    } else {
-      throw new IllegalArgumentException(
-          String.format("HTTP method %s is used. Only GET and POST are allowed.", method));
+
+    if (!CollectionUtils.isNullOrEmpty(params)) {
+      Escaper escaper = UrlEscapers.urlPathSegmentEscaper();
+      String encodedParams =
+          Joiner.on("&")
+              .join(
+                  params.entries().stream()
+                      .map(
+                          entry ->
+                              String.format(
+                                  "%s=%s",
+                                  escaper.escape(entry.getKey()), escaper.escape(entry.getValue())))
+                      .collect(toImmutableList()));
+      if (method == HttpMethod.GET) {
+        path = String.format("%s?%s", path, encodedParams);
+      } else {
+        requestBuilder
+            .putHeaders(HttpHeaders.CONTENT_TYPE, MediaType.FORM_DATA.toString())
+            .setBody(ByteString.copyFrom(encodedParams, StandardCharsets.UTF_8));
+      }
     }
     requestBuilder.setRelativeUri(path);
     return Task.newBuilder().setAppEngineHttpRequest(requestBuilder.build()).build();
@@ -163,12 +168,45 @@ public class CloudTasksUtils implements Serializable {
     if (!jitterSeconds.isPresent() || jitterSeconds.get() <= 0) {
       return createTask(path, method, service, params);
     }
-    Instant scheduleTime =
-        Instant.ofEpochMilli(
-            clock
-                .nowUtc()
-                .plusMillis(random.nextInt((int) SECONDS.toMillis(jitterSeconds.get())))
-                .getMillis());
+    return createTask(
+        path,
+        method,
+        service,
+        params,
+        clock,
+        Duration.millis(random.nextInt((int) SECONDS.toMillis(jitterSeconds.get()))));
+  }
+
+  /**
+   * Create a {@link Task} to be enqueued with delay of {@code duration}.
+   *
+   * @param path the relative URI (staring with a slash and ending without one).
+   * @param method the HTTP method to be used for the request, only GET and POST are supported.
+   * @param service the App Engine service to route the request to. Note that with App Engine Task
+   *     Queue API if no service is specified, the service which enqueues the task will be used to
+   *     process the task. Cloud Tasks API does not support this feature so the service will always
+   *     needs to be explicitly specified.
+   * @param params a multi-map of URL query parameters. Duplicate keys are saved as is, and it is up
+   *     to the server to process the duplicate keys.
+   * @param clock a source of time.
+   * @param delay the amount of time that a task needs to delayed for.
+   * @return the enqueued task.
+   * @see <a
+   *     href=ttps://cloud.google.com/appengine/docs/standard/java/taskqueue/push/creating-tasks#target>Specifyinig
+   *     the worker service</a>
+   */
+  private static Task createTask(
+      String path,
+      HttpMethod method,
+      String service,
+      Multimap<String, String> params,
+      Clock clock,
+      Duration delay) {
+    if (delay.isEqual(Duration.ZERO)) {
+      return createTask(path, method, service, params);
+    }
+    checkArgument(delay.isLongerThan(Duration.ZERO), "Negative duration is not supported.");
+    Instant scheduleTime = Instant.ofEpochMilli(clock.nowUtc().getMillis() + delay.getMillis());
     return Task.newBuilder(createTask(path, method, service, params))
         .setScheduleTime(
             Timestamp.newBuilder()
@@ -210,6 +248,18 @@ public class CloudTasksUtils implements Serializable {
     return createTask(path, HttpMethod.GET, service, params, clock, jitterSeconds);
   }
 
+  /** Create a {@link Task} via HTTP.POST that will be delayed for {@code delay}. */
+  public static Task createPostTask(
+      String path, String service, Multimap<String, String> params, Clock clock, Duration delay) {
+    return createTask(path, HttpMethod.POST, service, params, clock, delay);
+  }
+
+  /** Create a {@link Task} via HTTP.GET that will be delayed for {@code delay}. */
+  public static Task createGetTask(
+      String path, String service, Multimap<String, String> params, Clock clock, Duration delay) {
+    return createTask(path, HttpMethod.GET, service, params, clock, delay);
+  }
+
   public abstract static class SerializableCloudTasksClient implements Serializable {
     public abstract Task enqueue(String projectId, String locationId, String queueName, Task task);
   }

util/src/main/java/google/registry/util/CollectionUtils.java

@@ -49,6 +49,11 @@ public class CollectionUtils {
     return potentiallyNull == null || potentiallyNull.isEmpty();
   }
 
+  /** Checks if a Multimap is null or empty. */
+  public static boolean isNullOrEmpty(@Nullable Multimap<?, ?> potentiallyNull) {
+    return potentiallyNull == null || potentiallyNull.isEmpty();
+  }
+
   /** Turns a null set into an empty set. JAXB leaves lots of null sets lying around. */
   public static <T> Set<T> nullToEmpty(@Nullable Set<T> potentiallyNull) {
     return firstNonNull(potentiallyNull, ImmutableSet.of());

util/src/test/java/google/registry/util/CloudTasksUtilsTest.java

@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;
 import com.google.cloud.tasks.v2.HttpMethod;
 import com.google.cloud.tasks.v2.Task;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.LinkedListMultimap;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
@@ -33,6 +34,7 @@ import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.util.Optional;
 import org.joda.time.DateTime;
+import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -80,6 +82,48 @@ public class CloudTasksUtilsTest {
     assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
   }
 
+  @Test
+  void testSuccess_createGetTasks_withNullParams() {
+    Task task = CloudTasksUtils.createGetTask("/the/path", "myservice", null);
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
+  @Test
+  void testSuccess_createPostTasks_withNullParams() {
+    Task task = CloudTasksUtils.createPostTask("/the/path", "myservice", null);
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8)).isEmpty();
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
+  @Test
+  void testSuccess_createGetTasks_withEmptyParams() {
+    Task task = CloudTasksUtils.createGetTask("/the/path", "myservice", ImmutableMultimap.of());
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
+  @Test
+  void testSuccess_createPostTasks_withEmptyParams() {
+    Task task = CloudTasksUtils.createPostTask("/the/path", "myservice", ImmutableMultimap.of());
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8)).isEmpty();
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
   @SuppressWarnings("ProtoTimestampGetSecondsGetNano")
   @Test
   void testSuccess_createGetTasks_withJitterSeconds() {
@@ -176,6 +220,87 @@ public class CloudTasksUtilsTest {
     assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
   }
 
+  @Test
+  void testSuccess_createGetTasks_withDelay() {
+    Task task =
+        CloudTasksUtils.createGetTask(
+            "/the/path", "myservice", params, clock, Duration.standardMinutes(10));
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri())
+        .isEqualTo("/the/path?key1=val1&key2=val2&key1=val3");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(Instant.ofEpochSecond(task.getScheduleTime().getSeconds()))
+        .isEqualTo(Instant.ofEpochMilli(clock.nowUtc().plusMinutes(10).getMillis()));
+  }
+
+  @Test
+  void testSuccess_createPostTasks_withDelay() {
+    Task task =
+        CloudTasksUtils.createPostTask(
+            "/the/path", "myservice", params, clock, Duration.standardMinutes(10));
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getAppEngineHttpRequest().getHeadersMap().get("Content-Type"))
+        .isEqualTo("application/x-www-form-urlencoded");
+    assertThat(task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8))
+        .isEqualTo("key1=val1&key2=val2&key1=val3");
+    assertThat(task.getScheduleTime().getSeconds()).isNotEqualTo(0);
+    assertThat(Instant.ofEpochSecond(task.getScheduleTime().getSeconds()))
+        .isEqualTo(Instant.ofEpochMilli(clock.nowUtc().plusMinutes(10).getMillis()));
+  }
+
+  @Test
+  void testFailure_createGetTasks_withNegativeDelay() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                CloudTasksUtils.createGetTask(
+                    "/the/path", "myservice", params, clock, Duration.standardMinutes(-10)));
+    assertThat(thrown).hasMessageThat().isEqualTo("Negative duration is not supported.");
+  }
+
+  @Test
+  void testFailure_createPostTasks_withNegativeDelay() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                CloudTasksUtils.createGetTask(
+                    "/the/path", "myservice", params, clock, Duration.standardMinutes(-10)));
+    assertThat(thrown).hasMessageThat().isEqualTo("Negative duration is not supported.");
+  }
+
+  @Test
+  void testSuccess_createPostTasks_withZeroDelay() {
+    Task task =
+        CloudTasksUtils.createPostTask("/the/path", "myservice", params, clock, Duration.ZERO);
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.POST);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri()).isEqualTo("/the/path");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getAppEngineHttpRequest().getHeadersMap().get("Content-Type"))
+        .isEqualTo("application/x-www-form-urlencoded");
+    assertThat(task.getAppEngineHttpRequest().getBody().toString(StandardCharsets.UTF_8))
+        .isEqualTo("key1=val1&key2=val2&key1=val3");
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
+  @Test
+  void testSuccess_createGetTasks_withZeroDelay() {
+    Task task =
+        CloudTasksUtils.createGetTask("/the/path", "myservice", params, clock, Duration.ZERO);
+    assertThat(task.getAppEngineHttpRequest().getHttpMethod()).isEqualTo(HttpMethod.GET);
+    assertThat(task.getAppEngineHttpRequest().getRelativeUri())
+        .isEqualTo("/the/path?key1=val1&key2=val2&key1=val3");
+    assertThat(task.getAppEngineHttpRequest().getAppEngineRouting().getService())
+        .isEqualTo("myservice");
+    assertThat(task.getScheduleTime().getSeconds()).isEqualTo(0);
+  }
+
   @Test
   void testFailure_illegalPath() {
     assertThrows(

util/src/test/java/google/registry/util/CollectionUtilsTest.java

@@ -22,6 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Multimap;
 import java.util.Map;
 import org.junit.jupiter.api.Test;
 
@@ -43,6 +45,12 @@ class CollectionUtilsTest {
     assertThat(convertedMap).isEmpty();
   }
 
+  @Test
+  void testNullOrEmptyMultimap() {
+    assertThat(CollectionUtils.isNullOrEmpty((Multimap<?, ?>) null)).isTrue();
+    assertThat(CollectionUtils.isNullOrEmpty(ImmutableMultimap.of())).isTrue();
+  }
+
   @Test
   void testPartitionMap() {
     Map<String, String> map = ImmutableMap.of("ka", "va", "kb", "vb", "kc", "vc");