Fix weird flake (#1394)

Relevant error log message: https://pantheon.corp.google.com/logs/viewer?project=domain-registry&minLogLevel=0&expandAll=false&timestamp=2021-10-11T15:28:01.047783000Z&customFacets=&limitCustomFacetWidth=true&dateRangeEnd=2021-10-11T20:51:40.591Z&interval=PT1H&resource=gae_app&logName=projects%2Fdomain-registry%2Flogs%2Fappengine.googleapis.com%252Frequest_log&scrollTimestamp=2021-10-11T15:10:23.174336000Z&filters=text:icannReportingUpload&dateRangeUnbound=backwardInTime&advancedFilter=resource.type%3D%22gae_app%22%0AlogName%3D%22projects%2Fdomain-registry%2Flogs%2Fappengine.googleapis.com%252Frequest_log%22%0A%22icannReportingUpload%22%0Aoperation.id%3D%22616453df00ff02a873d26cedb40001737e646f6d61696e2d726567697374727900016261636b656e643a6e6f6d756c75732d76303233000100%22

note the "invalid handle" bit

From https://cloud.google.com/datastore/docs/concepts/transactions:
"Transactions expire after 270 seconds or if idle for 60 seconds."

From b/202309933: "There is a 60 second timeout on Datastore operations
after which they will automatically rollback and the handles become
invalid."

From the logs we can see that the action is lasting significantly longer
than 270 seconds -- roughly 480 seconds in the linked log (more or
less). My running theory is that ICANN is, for some reason, now being
significantly more slow to respond than they used to be. Some uploads in
the log linked above are taking upwards of 10 seconds, especially when
they have to retry. Because we have >=45 TLDs, it's not surprising that
the action is taking >400 seconds to run.

The fix here is to perform each per-TLD operation in its own
transaction. The only reason why we need the transactions is for the
cursors anyway, and we can just grab and store those at the beginning of
the transaction.

core/build.gradle

@@ -705,7 +705,11 @@ createToolTask(
 
 
 createToolTask(
-        'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+    'jpaDemoPipeline', 'google.registry.beam.common.JpaDemoPipeline')
+
+createToolTask(
+    'createSyntheticHistoryEntries',
+    'google.registry.tools.javascrap.CreateSyntheticHistoryEntriesPipeline')
 
 project.tasks.create('initSqlPipeline', JavaExec) {
   main = 'google.registry.beam.initsql.InitSqlPipeline'

core/src/main/java/google/registry/backup/CommitLogImports.java

@@ -55,7 +55,7 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
+  static ImmutableList<ImmutableList<VersionedEntity>> loadEntitiesByTransaction(
       InputStream inputStream) {
     try (InputStream input = new BufferedInputStream(inputStream)) {
       Iterator<ImmutableObject> commitLogs = createDeserializingIterator(input, false);
@@ -104,7 +104,7 @@ public final class CommitLogImports {
    * represents the changes in one transaction. The {@code CommitLogManifest} contains deleted
    * entity keys, whereas each {@code CommitLogMutation} contains one whole entity.
    */
-  public static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
+  static ImmutableList<VersionedEntity> loadEntities(InputStream inputStream) {
     return loadEntitiesByTransaction(inputStream).stream()
         .flatMap(ImmutableList::stream)
         .collect(toImmutableList());

core/src/main/java/google/registry/backup/VersionedEntity.java

@@ -105,29 +105,29 @@ public abstract class VersionedEntity implements Serializable {
    * VersionedEntity VersionedEntities}. See {@link CommitLogImports#loadEntities} for more
    * information.
    */
-  public static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
+  static Stream<VersionedEntity> fromManifest(CommitLogManifest manifest) {
     long commitTimeMillis = manifest.getCommitTime().getMillis();
     return manifest.getDeletions().stream()
         .map(com.googlecode.objectify.Key::getRaw)
-        .map(key -> builder().commitTimeMills(commitTimeMillis).key(key).build());
+        .map(key -> newBuilder().commitTimeMills(commitTimeMillis).key(key).build());
   }
 
   /* Converts a {@link CommitLogMutation} to a {@link VersionedEntity}. */
-  public static VersionedEntity fromMutation(CommitLogMutation mutation) {
+  static VersionedEntity fromMutation(CommitLogMutation mutation) {
     return from(
         com.googlecode.objectify.Key.create(mutation).getParent().getId(),
         mutation.getEntityProtoBytes());
   }
 
   public static VersionedEntity from(long commitTimeMillis, byte[] entityProtoBytes) {
-    return builder()
+    return newBuilder()
         .entityProtoBytes(entityProtoBytes)
         .key(EntityTranslator.createFromPbBytes(entityProtoBytes).getKey())
         .commitTimeMills(commitTimeMillis)
         .build();
   }
 
-  static Builder builder() {
+  private static Builder newBuilder() {
     return new AutoValue_VersionedEntity.Builder();
   }
 
@@ -142,7 +142,7 @@ public abstract class VersionedEntity implements Serializable {
 
     public abstract VersionedEntity build();
 
-    public Builder entityProtoBytes(byte[] bytes) {
+    Builder entityProtoBytes(byte[] bytes) {
       return entityProtoBytes(new ImmutableBytes(bytes));
     }
   }

core/src/main/java/google/registry/batch/SendExpiringCertificateNotificationEmailAction.java

@@ -55,6 +55,7 @@ import org.joda.time.format.DateTimeFormatter;
     path = SendExpiringCertificateNotificationEmailAction.PATH,
     auth = Auth.AUTH_INTERNAL_OR_ADMIN)
 public class SendExpiringCertificateNotificationEmailAction implements Runnable {
+
   public static final String PATH = "/_dr/task/sendExpiringCertificateNotificationEmail";
   /**
    * Used as an offset when storing the last notification email sent date.
@@ -96,8 +97,13 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
   public void run() {
     response.setContentType(MediaType.PLAIN_TEXT_UTF_8);
     try {
-      sendNotificationEmails();
+      int numEmailsSent = sendNotificationEmails();
+      String message =
+          String.format(
+              "Done. Sent %d expiring certificate notification emails in total.", numEmailsSent);
+      logger.atInfo().log(message);
       response.setStatus(SC_OK);
+      response.setPayload(message);
     } catch (Exception e) {
       logger.atWarning().withCause(e).log(
           "Exception thrown when sending expiring certificate notification emails.");
@@ -263,8 +269,6 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
         emailsSent++;
       }
     }
-    logger.atInfo().log(
-        "Attempted to send %d expiring certificate notification emails.", emailsSent);
     return emailsSent;
   }
 
@@ -327,6 +331,7 @@ public class SendExpiringCertificateNotificationEmailAction implements Runnable
 
   @AutoValue
   public abstract static class RegistrarInfo {
+
     static RegistrarInfo create(
         Registrar registrar, boolean isCertExpiring, boolean isFailOverCertExpiring) {
       return new AutoValue_SendExpiringCertificateNotificationEmailAction_RegistrarInfo(

core/src/main/java/google/registry/batch/WipeOutContactHistoryPiiAction.java

@@ -84,8 +84,12 @@ public class WipeOutContactHistoryPiiAction implements Runnable {
                             getNextContactHistoryEntitiesWithPiiBatch(wipeOutTime)));
         totalNumOfWipedEntities += numOfWipedEntities;
       } while (numOfWipedEntities > 0);
-      logger.atInfo().log(
-          "Wiped out PII of %d ContactHistory entities in total.", totalNumOfWipedEntities);
+      String msg =
+          String.format(
+              "Done. Wiped out PII of %d ContactHistory entities in total.",
+              totalNumOfWipedEntities);
+      logger.atInfo().log(msg);
+      response.setPayload(msg);
       response.setStatus(SC_OK);
 
     } catch (Exception e) {

core/src/main/java/google/registry/batch/WipeoutDatastoreAction.java

@@ -92,7 +92,12 @@ public class WipeoutDatastoreAction implements Runnable {
               .setJobName(createJobName("bulk-delete-datastore-", clock))
               .setContainerSpecGcsPath(
                   String.format("%s/%s_metadata.json", stagingBucketUrl, PIPELINE_NAME))
-              .setParameters(ImmutableMap.of("kindsToDelete", "*"));
+              .setParameters(
+                  ImmutableMap.of(
+                      "kindsToDelete",
+                      "*",
+                      "registryEnvironment",
+                      RegistryEnvironment.get().name()));
       LaunchFlexTemplateResponse launchResponse =
           dataflow
               .projects()

core/src/main/java/google/registry/beam/common/RegistryPipelineOptions.java

@@ -34,7 +34,6 @@ import org.apache.beam.sdk.options.Description;
 public interface RegistryPipelineOptions extends GcpOptions {
 
   @Description("The Registry environment.")
-  @Nullable
   RegistryEnvironment getRegistryEnvironment();
 
   void setRegistryEnvironment(RegistryEnvironment environment);

core/src/main/java/google/registry/beam/datastore/BulkDeleteDatastorePipeline.java

@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.flogger.FluentLogger;
 import com.google.datastore.v1.Entity;
+import google.registry.config.RegistryEnvironment;
 import java.util.Iterator;
 import java.util.Map;
 import org.apache.beam.sdk.Pipeline;
@@ -308,6 +309,11 @@ public class BulkDeleteDatastorePipeline {
 
   public interface BulkDeletePipelineOptions extends GcpOptions {
 
+    @Description("The Registry environment.")
+    RegistryEnvironment getRegistryEnvironment();
+
+    void setRegistryEnvironment(RegistryEnvironment environment);
+
     @Description(
         "The Datastore KINDs to be deleted. The format may be:\n"
             + "\t- The list of kinds to be deleted as a comma-separated string, or\n"

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -22,6 +22,7 @@ import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
 import static java.util.Comparator.comparing;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -277,7 +278,7 @@ public final class Transforms {
 
   // Prober contacts referencing phantom registrars. They and their associated history entries can
   // be safely ignored.
-  private static final ImmutableSet IGNORED_CONTACTS =
+  private static final ImmutableSet<String> IGNORED_CONTACTS =
       ImmutableSet.of(
           "1_WJ0TEST-GOOGLE", "1_WJ1TEST-GOOGLE", "1_WJ2TEST-GOOGLE", "1_WJ3TEST-GOOGLE");
 
@@ -320,7 +321,8 @@ public final class Transforms {
     return true;
   }
 
-  private static Entity repairBadData(Entity entity) {
+  @VisibleForTesting
+  static Entity repairBadData(Entity entity) {
     if (entity.getKind().equals("Cancellation")
         && Objects.equals(entity.getProperty("reason"), "AUTO_RENEW")) {
       // AUTO_RENEW has been moved from 'reason' to flags. Change reason to RENEW and add the
@@ -328,6 +330,15 @@ public final class Transforms {
       // instead of append. See b/185954992.
       entity.setUnindexedProperty("reason", Reason.RENEW.name());
       entity.setUnindexedProperty("flags", ImmutableList.of(Flag.AUTO_RENEW.name()));
+    } else if (entity.getKind().equals("DomainBase")) {
+      // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this.
+      entity.setIndexedProperty(
+          "fullyQualifiedDomainName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedDomainName")));
+    } else if (entity.getKind().equals("HostResource")) {
+      entity.setIndexedProperty(
+          "fullyQualifiedHostName",
+          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedHostName")));
     }
     return entity;
   }
@@ -365,7 +376,8 @@ public final class Transforms {
    * Returns a {@link PTransform} that produces a {@link PCollection} containing all elements in the
    * given {@link Iterable}.
    */
-  static PTransform<PBegin, PCollection<String>> toStringPCollection(Iterable<String> strings) {
+  private static PTransform<PBegin, PCollection<String>> toStringPCollection(
+      Iterable<String> strings) {
     return Create.of(strings).withCoder(StringUtf8Coder.of());
   }
 
@@ -373,7 +385,7 @@ public final class Transforms {
    * Returns a {@link PTransform} from file {@link Metadata} to {@link VersionedEntity} using
    * caller-provided {@code transformer}.
    */
-  static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
+  private static PTransform<PCollection<Metadata>, PCollection<VersionedEntity>> processFiles(
       DoFn<ReadableFile, VersionedEntity> transformer) {
     return new PTransform<PCollection<Metadata>, PCollection<VersionedEntity>>() {
       @Override
@@ -389,7 +401,7 @@ public final class Transforms {
     private final DateTime fromTime;
     private final DateTime toTime;
 
-    public FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
+    FilterCommitLogFileByTime(DateTime fromTime, DateTime toTime) {
       checkNotNull(fromTime, "fromTime");
       checkNotNull(toTime, "toTime");
       checkArgument(

core/src/main/java/google/registry/env/production/default/WEB-INF/cron.xml

@@ -348,4 +348,14 @@
     <schedule>every 3 minutes</schedule>
     <target>backend</target>
   </cron>
+
+  <cron>
+    <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
+    <description>
+      This job runs weekly to wipe out PII fields of ContactHistory entities
+      that have been in the database for a certain period of time.
+    </description>
+    <schedule>every monday 15:00</schedule>
+    <target>backend</target>
+  </cron>
 </cronentries>

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -229,6 +229,15 @@ public final class DomainRenewFlow implements TransactionalFlow {
 
   private DomainHistory buildDomainHistory(
       DomainBase newDomain, DateTime now, Period period, Duration renewGracePeriod) {
+    Optional<MetadataExtension> metadataExtensionOpt =
+        eppInput.getSingleExtension(MetadataExtension.class);
+    if (metadataExtensionOpt.isPresent()) {
+      MetadataExtension metadataExtension = metadataExtensionOpt.get();
+      if (metadataExtension.getReason() != null) {
+        historyBuilder.setReason(metadataExtension.getReason());
+      }
+      historyBuilder.setRequestedByRegistrar(metadataExtension.getRequestedByRegistrar());
+    }
     return historyBuilder
         .setType(DOMAIN_RENEW)
         .setPeriod(period)

core/src/main/java/google/registry/model/UpdateAutoTimestamp.java

@@ -15,12 +15,8 @@
 package google.registry.model;
 
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static org.joda.time.DateTimeZone.UTC;
 
-import com.google.common.flogger.FluentLogger;
-import com.google.common.flogger.StackSize;
 import com.googlecode.objectify.annotation.Ignore;
 import com.googlecode.objectify.annotation.OnLoad;
 import google.registry.model.translators.UpdateAutoTimestampTranslatorFactory;
@@ -44,12 +40,10 @@ import org.joda.time.DateTime;
 @Embeddable
 public class UpdateAutoTimestamp extends ImmutableObject {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-
   // When set to true, database converters/translators should do the auto update.  When set to
   // false, auto update should be suspended (this exists to allow us to preserve the original value
   // during a replay).
-  private static ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
+  private static final ThreadLocal<Boolean> autoUpdateEnabled = ThreadLocal.withInitial(() -> true);
 
   @Transient DateTime timestamp;
 
@@ -63,16 +57,7 @@ public class UpdateAutoTimestamp extends ImmutableObject {
   @PrePersist
   @PreUpdate
   void setTimestamp() {
-    // On the off chance that this is called outside of a transaction, log it instead of failing
-    // with an exception from attempting to call jpaTm().getTransactionTime(), and then fall back
-    // to DateTime.now(UTC).
-    if (!jpaTm().inTransaction()) {
-      logger.atSevere().withStackTrace(StackSize.MEDIUM).log(
-          "Failed to update automatic timestamp because this wasn't called in a JPA transaction%s.",
-          ofyTm().inTransaction() ? " (but there is an open Ofy transaction)" : "");
-      timestamp = DateTime.now(UTC);
-      lastUpdateTime = DateTimeUtils.toZonedDateTime(timestamp);
-    } else if (autoUpdateEnabled() || lastUpdateTime == null) {
+    if (autoUpdateEnabled() || lastUpdateTime == null) {
       timestamp = jpaTm().getTransactionTime();
       lastUpdateTime = DateTimeUtils.toZonedDateTime(timestamp);
     }

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -865,7 +865,8 @@ public class DomainContent extends EppResource
     public B setDomainName(String domainName) {
       checkArgument(
           domainName.equals(canonicalizeDomainName(domainName)),
-          "Domain name must be in puny-coded, lower-case form");
+          "Domain name %s not in puny-coded, lower-case form",
+          domainName);
       getInstance().fullyQualifiedDomainName = domainName;
       return thisCastToDerived();
     }

core/src/main/java/google/registry/model/domain/GracePeriod.java

@@ -24,6 +24,7 @@ import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.replay.DatastoreAndSqlEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import google.registry.persistence.BillingVKey.BillingEventVKey;
 import google.registry.persistence.BillingVKey.BillingRecurrenceVKey;
 import google.registry.persistence.VKey;
@@ -202,7 +203,7 @@ public class GracePeriod extends GracePeriodBase implements DatastoreAndSqlEntit
   /** Entity class to represent a historic {@link GracePeriod}. */
   @Entity(name = "GracePeriodHistory")
   @Table(indexes = @Index(columnList = "domainRepoId"))
-  static class GracePeriodHistory extends GracePeriodBase {
+  static class GracePeriodHistory extends GracePeriodBase implements SqlOnlyEntity {
     @Id Long gracePeriodHistoryRevisionId;
 
     /** ID for the associated {@link DomainHistory} entity. */

core/src/main/java/google/registry/model/host/HostBase.java

@@ -196,7 +196,8 @@ public class HostBase extends EppResource {
     public B setHostName(String hostName) {
       checkArgument(
           hostName.equals(canonicalizeDomainName(hostName)),
-          "Host name must be in puny-coded, lower-case form");
+          "Host name %s not in puny-coded, lower-case form",
+          hostName);
       getInstance().fullyQualifiedHostName = hostName;
       return thisCastToDerived();
     }

core/src/main/java/google/registry/model/tmch/ClaimsEntry.java

@@ -15,7 +15,7 @@
 package google.registry.model.tmch;
 
 import google.registry.model.ImmutableObject;
-import google.registry.model.replay.NonReplicatedEntity;
+import google.registry.model.replay.SqlOnlyEntity;
 import java.io.Serializable;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -30,7 +30,7 @@ import javax.persistence.Id;
  * work.
  */
 @Entity(name = "ClaimsEntry")
-class ClaimsEntry extends ImmutableObject implements NonReplicatedEntity, Serializable {
+class ClaimsEntry extends ImmutableObject implements SqlOnlyEntity, Serializable {
   @Id private Long revisionId;
   @Id private String domainLabel;
 

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -132,13 +132,16 @@ public class TransactionManagerFactory {
   /**
    * Sets the return of {@link #tm()} to the given instance of {@link TransactionManager}.
    *
+   * <p>DO NOT CALL THIS DIRECTLY IF POSSIBLE. Strongly prefer the use of <code>TmOverrideExtension
+   * </code> in test code instead.
+   *
    * <p>Used when overriding the per-test transaction manager for dual-database tests. Should be
    * matched with a corresponding invocation of {@link #removeTmOverrideForTest()} either at the end
    * of the test or in an <code>@AfterEach</code> handler.
    */
   @VisibleForTesting
-  public static void setTmForTest(TransactionManager newTm) {
-    tmForTest = Optional.of(newTm);
+  public static void setTmOverrideForTest(TransactionManager newTmOverride) {
+    tmForTest = Optional.of(newTmOverride);
   }
 
   /** Resets the overridden transaction manager post-test. */
@@ -152,10 +155,10 @@ public class TransactionManagerFactory {
     }
   }
 
-  /** Thrown when a write is attempted when the DB is in read-only mode. */
+  /** Registry is currently undergoing maintenance and is in read-only mode. */
   public static class ReadOnlyModeException extends IllegalStateException {
-    public ReadOnlyModeException() {
-      super("Registry is currently in read-only mode");
+    ReadOnlyModeException() {
+      super("Registry is currently undergoing maintenance and is in read-only mode");
     }
   }
 }

core/src/main/java/google/registry/reporting/icann/IcannReportingUploadAction.java

@@ -14,7 +14,6 @@
 
 package google.registry.reporting.icann;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.request.Action.Method.POST;
@@ -46,7 +45,6 @@ import google.registry.util.SendEmailService;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Map;
-import java.util.concurrent.Callable;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import javax.inject.Inject;
@@ -78,6 +76,7 @@ public final class IcannReportingUploadAction implements Runnable {
   static final String PATH = "/_dr/task/icannReportingUpload";
 
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+  private static final String LOCK_NAME = "IcannReportingUploadAction";
 
   @Inject
   @Config("reportingBucket")
@@ -98,48 +97,33 @@ public final class IcannReportingUploadAction implements Runnable {
 
   @Override
   public void run() {
-    Runnable transactional =
-        () -> {
-          ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
-
-          ImmutableMap<Cursor, String> cursors = loadCursors();
-
-          // If cursor time is before now, upload the corresponding report
-          cursors.entrySet().stream()
-              .filter(entry -> entry.getKey().getCursorTime().isBefore(clock.nowUtc()))
-              .forEach(
-                  entry -> {
-                    DateTime cursorTime = entry.getKey().getCursorTime();
-                    uploadReport(
-                        cursorTime,
-                        entry.getKey().getType(),
-                        entry.getValue(),
-                        reportSummaryBuilder);
-                  });
-          // Send email of which reports were uploaded
-          emailUploadResults(reportSummaryBuilder.build());
-          response.setStatus(SC_OK);
-          response.setContentType(PLAIN_TEXT_UTF_8);
-        };
-
-    Callable<Void> lockRunner =
-        () -> {
-          tm().transact(transactional);
-          return null;
-        };
-
-    String lockname = "IcannReportingUploadAction";
-    if (!lockHandler.executeWithLocks(lockRunner, null, Duration.standardHours(2), lockname)) {
-      throw new ServiceUnavailableException("Lock for IcannReportingUploadAction already in use");
+    if (!lockHandler.executeWithLocks(
+        this::runWithLock, null, Duration.standardHours(2), LOCK_NAME)) {
+      throw new ServiceUnavailableException(String.format("Lock for %s already in use", LOCK_NAME));
     }
   }
 
+  private Void runWithLock() {
+    ImmutableMap.Builder<String, Boolean> reportSummaryBuilder = new ImmutableMap.Builder<>();
+
+    ImmutableMap<Cursor, String> cursors = tm().transact(this::loadCursors);
+
+    // If cursor time is before now, upload the corresponding report
+    cursors.entrySet().stream()
+        .filter(entry -> entry.getKey().getCursorTime().isBefore(clock.nowUtc()))
+        .forEach(entry -> uploadReport(entry.getKey(), entry.getValue(), reportSummaryBuilder));
+    // Send email of which reports were uploaded
+    emailUploadResults(reportSummaryBuilder.build());
+    response.setStatus(SC_OK);
+    response.setContentType(PLAIN_TEXT_UTF_8);
+    return null;
+  }
+
   /** Uploads the report and rolls forward the cursor for that report. */
   private void uploadReport(
-      DateTime cursorTime,
-      CursorType cursorType,
-      String tldStr,
-      ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) {
+      Cursor cursor, String tldStr, ImmutableMap.Builder<String, Boolean> reportSummaryBuilder) {
+    DateTime cursorTime = cursor.getCursorTime();
+    CursorType cursorType = cursor.getType();
     DateTime cursorTimeMinusMonth = cursorTime.withDayOfMonth(1).minusMonths(1);
     String reportSubdir =
         String.format(
@@ -150,17 +134,16 @@ public final class IcannReportingUploadAction implements Runnable {
         BlobId.of(reportingBucket, String.format("%s/%s", reportSubdir, filename));
     logger.atInfo().log("Reading ICANN report %s from bucket '%s'.", filename, reportingBucket);
     // Check that the report exists
-    try {
-      verifyFileExists(gcsFilename);
-    } catch (IllegalArgumentException e) {
+    if (!gcsUtils.existsAndNotEmpty(gcsFilename)) {
       String logMessage =
           String.format(
-              "Could not upload %s report for %s because file %s did not exist.",
-              cursorType, tldStr, filename);
+              "Could not upload %s report for %s because file %s (object %s in bucket %s) did not"
+                  + " exist.",
+              cursorType, tldStr, filename, gcsFilename.getName(), gcsFilename.getBucket());
       if (clock.nowUtc().dayOfMonth().get() == 1) {
-        logger.atInfo().withCause(e).log(logMessage + " This report may not have been staged yet.");
+        logger.atInfo().log(logMessage + " This report may not have been staged yet.");
       } else {
-        logger.atSevere().withCause(e).log(logMessage);
+        logger.atSevere().log(logMessage);
       }
       reportSummaryBuilder.put(filename, false);
       return;
@@ -179,7 +162,6 @@ public final class IcannReportingUploadAction implements Runnable {
     } catch (RuntimeException e) {
       logger.atWarning().withCause(e).log("Upload to %s failed.", gcsFilename);
     }
-    reportSummaryBuilder.put(filename, success);
 
     // Set cursor to first day of next month if the upload succeeded
     if (success) {
@@ -188,8 +170,24 @@ public final class IcannReportingUploadAction implements Runnable {
               cursorType,
               cursorTime.withTimeAtStartOfDay().withDayOfMonth(1).plusMonths(1),
               Registry.get(tldStr));
-      tm().put(newCursor);
+      // In order to keep the transactions short-lived, we load all of the cursors in a single
+      // transaction then later use per-cursor transactions when checking + saving the cursors. We
+      // run behind a lock so the cursors shouldn't be changed, but double check to be sure.
+      success =
+          tm().transact(
+                  () -> {
+                    Cursor fromDb = tm().transact(() -> tm().loadByEntity(cursor));
+                    if (!cursor.equals(fromDb)) {
+                      logger.atSevere().log(
+                          "Expected previously-loaded cursor %s to equal current cursor %s",
+                          cursor, fromDb);
+                      return false;
+                    }
+                    tm().put(newCursor);
+                    return true;
+                  });
     }
+    reportSummaryBuilder.put(filename, success);
   }
 
   private String getFileName(CursorType cursorType, DateTime cursorTime, String tld) {
@@ -303,13 +301,4 @@ public final class IcannReportingUploadAction implements Runnable {
       return ByteStreams.toByteArray(gcsInput);
     }
   }
-
-  private void verifyFileExists(BlobId gcsFilename) {
-    checkArgument(
-        gcsUtils.existsAndNotEmpty(gcsFilename),
-        "Object %s in bucket %s not found",
-        gcsFilename.getName(),
-        gcsFilename.getBucket());
-  }
-
 }

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
 import google.registry.tools.javascrap.BackfillSpec11ThreatMatchesCommand;
 import google.registry.tools.javascrap.DeleteContactByRoidCommand;
+import google.registry.tools.javascrap.HardDeleteHostCommand;
 import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand;
 import google.registry.tools.javascrap.RemoveIpAddressCommand;
 import google.registry.tools.javascrap.ResaveAllTldsCommand;
@@ -86,6 +87,7 @@ public final class RegistryTool {
           .put("get_sql_credential", GetSqlCredentialCommand.class)
           .put("get_tld", GetTldCommand.class)
           .put("ghostryde", GhostrydeCommand.class)
+          .put("hard_delete_host", HardDeleteHostCommand.class)
           .put("hash_certificate", HashCertificateCommand.class)
           .put("import_datastore", ImportDatastoreCommand.class)
           .put("list_cursors", ListCursorsCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -43,6 +43,7 @@ import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.tools.javascrap.BackfillRegistryLocksCommand;
 import google.registry.tools.javascrap.DeleteContactByRoidCommand;
+import google.registry.tools.javascrap.HardDeleteHostCommand;
 import google.registry.util.UtilsModule;
 import google.registry.whois.NonCachingWhoisModule;
 import javax.annotation.Nullable;
@@ -124,6 +125,8 @@ interface RegistryToolComponent {
 
   void inject(GhostrydeCommand command);
 
+  void inject(HardDeleteHostCommand command);
+
   void inject(ImportDatastoreCommand command);
 
   void inject(ListCursorsCommand command);

core/src/main/java/google/registry/tools/RenewDomainCommand.java

@@ -18,6 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.util.CollectionUtils.findDuplicates;
+import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
 
 import com.beust.jcommander.Parameter;
@@ -53,6 +54,17 @@ final class RenewDomainCommand extends MutatingEppToolCommand {
   @Parameter(description = "Names of the domains to renew.", required = true)
   private List<String> mainParameters;
 
+  @Parameter(
+      names = {"--reason"},
+      description = "Reason for the change.")
+  String reason;
+
+  @Parameter(
+      names = {"--registrar_request"},
+      description = "Whether the change was requested by a registrar.",
+      arity = 1)
+  Boolean requestedByRegistrar;
+
   @Inject
   Clock clock;
 
@@ -70,12 +82,23 @@ final class RenewDomainCommand extends MutatingEppToolCommand {
       checkArgumentPresent(domainOptional, "Domain '%s' does not exist or is deleted", domainName);
       setSoyTemplate(DomainRenewSoyInfo.getInstance(), DomainRenewSoyInfo.RENEWDOMAIN);
       DomainBase domain = domainOptional.get();
-      addSoyRecord(
-          isNullOrEmpty(clientId) ? domain.getCurrentSponsorRegistrarId() : clientId,
+
+      SoyMapData soyMapData =
           new SoyMapData(
               "domainName", domain.getDomainName(),
               "expirationDate", domain.getRegistrationExpirationTime().toString(DATE_FORMATTER),
-              "period", String.valueOf(period)));
+              "period", String.valueOf(period));
+
+      if (requestedByRegistrar != null) {
+        soyMapData.put("requestedByRegistrar", requestedByRegistrar.toString());
+      }
+      if (reason != null) {
+        checkArgumentNotNull(
+            requestedByRegistrar, "--registrar_request is required when --reason is specified");
+        soyMapData.put("reason", reason);
+      }
+      addSoyRecord(
+          isNullOrEmpty(clientId) ? domain.getCurrentSponsorRegistrarId() : clientId, soyMapData);
     }
   }
 }

core/src/main/java/google/registry/tools/UniformRapidSuspensionCommand.java

@@ -163,7 +163,12 @@ final class UniformRapidSuspensionCommand extends MutatingEppToolCommand {
                   .toString(DateTimeFormat.forPattern("YYYY-MM-dd")),
               // period is the number of years to renew the registration for
               "period",
-              String.valueOf(1)));
+              String.valueOf(1),
+              // use the same values for reason and requestedByRegistrar from update flow
+              "reason",
+              (undo ? "Undo " : "") + "Uniform Rapid Suspension",
+              "requestedByRegistrar",
+              Boolean.toString(false)));
     }
 
     // trigger update flow

core/src/main/java/google/registry/tools/javascrap/CreateSyntheticHistoryEntriesAction.java

@@ -83,12 +83,12 @@ public class CreateSyntheticHistoryEntriesAction implements Runnable {
   }
 
   /**
-   * The number of shards to run the map-only mapreduce on.
+   * The default number of shards to run the map-only mapreduce on.
    *
-   * <p>This is much lower than the default of 100, or even 10, because we can afford it being slow
-   * and we want to avoid overloading SQL.
+   * <p>This is much lower than the default of 100 because we can afford it being slow and we want
+   * to avoid overloading SQL.
    */
-  private static final int NUM_SHARDS = 3;
+  private static final int NUM_SHARDS = 10;
 
   @Override
   public void run() {

core/src/main/java/google/registry/tools/javascrap/CreateSyntheticHistoryEntriesPipeline.java

@@ -0,0 +1,135 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+
+import com.google.common.collect.ImmutableList;
+import dagger.Component;
+import google.registry.beam.common.RegistryJpaIO;
+import google.registry.beam.common.RegistryPipelineOptions;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.model.EppResource;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.host.HostResource;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
+import google.registry.persistence.VKey;
+import java.io.Serializable;
+import javax.inject.Singleton;
+import javax.persistence.Entity;
+import org.apache.beam.sdk.Pipeline;
+import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.transforms.MapElements;
+import org.apache.beam.sdk.values.TypeDescriptor;
+
+/**
+ * Pipeline that creates a synthetic history entry for every {@link EppResource} in SQL at the
+ * current time.
+ *
+ * <p>The history entries in Datastore does not have the EPP resource embedded in them. Therefore
+ * after {@link google.registry.beam.initsql.InitSqlPipeline} runs, these fields will all be empty.
+ * This pipeline loads all EPP resources and for each of them creates a synthetic history entry that
+ * contains the resource and saves them back to SQL, so that they can be used in the RDE pipeline.
+ *
+ * <p>Note that this pipeline should only be run in a test environment right after the init SQL
+ * pipeline finishes, and no EPP update is being made to the system, otherwise there is no garuantee
+ * that the latest history entry for a given EPP resource does not already have the resource
+ * embedded within it.
+ *
+ * <p>To run the pipeline:
+ *
+ * <p><code>
+ * $ ./nom_build :core:cSHE --args="--region=us-central1
+ *   --runner=DataflowRunner
+ *   --registryEnvironment=CRASH
+ *   --project={project-id}
+ *   --workerMachineType=n2-standard-4"
+ * </code>
+ *
+ * @see google.registry.tools.javascrap.CreateSyntheticHistoryEntriesAction
+ */
+public class CreateSyntheticHistoryEntriesPipeline implements Serializable {
+
+  private static final ImmutableList<Class<? extends EppResource>> EPP_RESOURCE_CLASSES =
+      ImmutableList.of(DomainBase.class, ContactResource.class, HostResource.class);
+
+  private static final String HISTORY_REASON =
+      "Backfill EppResource history objects after initial backup to SQL";
+
+  static void setup(Pipeline pipeline, String registryAdminRegistrarId) {
+    for (Class<? extends EppResource> clazz : EPP_RESOURCE_CLASSES) {
+      pipeline
+          .apply(
+              String.format("Read all %s", clazz.getSimpleName()),
+              RegistryJpaIO.read(
+                  "SELECT id FROM %entity%"
+                      .replace("%entity%", clazz.getAnnotation(Entity.class).name()),
+                  String.class,
+                  repoId -> VKey.createSql(clazz, repoId)))
+          .apply(
+              String.format("Save a synthetic HistoryEntry for each %s", clazz),
+              MapElements.into(TypeDescriptor.of(Void.class))
+                  .via(
+                      (VKey<? extends EppResource> key) -> {
+                        jpaTm()
+                            .transact(
+                                () -> {
+                                  EppResource eppResource = jpaTm().loadByKey(key);
+                                  try (DisableAutoUpdateResource disable =
+                                      UpdateAutoTimestamp.disableAutoUpdate()) {
+                                    jpaTm()
+                                        .put(
+                                            HistoryEntry.createBuilderForResource(eppResource)
+                                                .setRegistrarId(registryAdminRegistrarId)
+                                                .setBySuperuser(true)
+                                                .setRequestedByRegistrar(false)
+                                                .setModificationTime(jpaTm().getTransactionTime())
+                                                .setReason(HISTORY_REASON)
+                                                .setType(HistoryEntry.Type.SYNTHETIC)
+                                                .build());
+                                  }
+                                });
+                        return null;
+                      }));
+    }
+  }
+
+  public static void main(String[] args) {
+    RegistryPipelineOptions options =
+        PipelineOptionsFactory.fromArgs(args).withValidation().as(RegistryPipelineOptions.class);
+    RegistryPipelineOptions.validateRegistryPipelineOptions(options);
+    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
+    String registryAdminRegistrarId =
+        DaggerCreateSyntheticHistoryEntriesPipeline_ConfigComponent.create()
+            .getRegistryAdminRegistrarId();
+
+    Pipeline pipeline = Pipeline.create(options);
+    setup(pipeline, registryAdminRegistrarId);
+    pipeline.run();
+  }
+
+  @Singleton
+  @Component(modules = ConfigModule.class)
+  interface ConfigComponent {
+
+    @Config("registryAdminClientId")
+    String getRegistryAdminRegistrarId();
+  }
+}

core/src/main/java/google/registry/tools/javascrap/HardDeleteHostCommand.java

@@ -0,0 +1,99 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.base.Verify.verify;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.collect.ImmutableList;
+import com.googlecode.objectify.Key;
+import google.registry.model.host.HostResource;
+import google.registry.model.index.EppResourceIndex;
+import google.registry.model.index.ForeignKeyIndex;
+import google.registry.tools.CommandWithRemoteApi;
+import google.registry.tools.ConfirmingCommand;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Deletes a {@link HostResource} by its ROID.
+ *
+ * <p>This deletes the host itself, everything in the same entity group including all {@link
+ * google.registry.model.reporting.HistoryEntry}s and {@link
+ * google.registry.model.poll.PollMessage}s, the {@link EppResourceIndex}, and the {@link
+ * ForeignKeyIndex} (if it exists).
+ *
+ * <p>DO NOT use this to hard-delete a host that is still in use on a domain. Bad things will
+ * happen.
+ */
+@Parameters(separators = " =", commandDescription = "Delete a host by its ROID.")
+public class HardDeleteHostCommand extends ConfirmingCommand implements CommandWithRemoteApi {
+
+  @Parameter(names = "--roid", description = "The ROID of the host to be deleted.")
+  String roid;
+
+  @Parameter(names = "--hostname", description = "The hostname, for verification.")
+  String hostname;
+
+  private ImmutableList<Key<Object>> toDelete;
+
+  @Override
+  protected void init() {
+    ofyTm()
+        .transact(
+            () -> {
+              Key<HostResource> targetKey = Key.create(HostResource.class, roid);
+              HostResource host = auditedOfy().load().key(targetKey).now();
+              verify(Objects.equals(host.getHostName(), hostname), "Hostname does not match");
+
+              List<Key<Object>> objectsInEntityGroup =
+                  auditedOfy().load().ancestor(host).keys().list();
+
+              Optional<ForeignKeyIndex<HostResource>> fki =
+                  Optional.ofNullable(
+                      auditedOfy().load().key(ForeignKeyIndex.createKey(host)).now());
+              if (!fki.isPresent()) {
+                System.out.println(
+                    "No ForeignKeyIndex exists, likely because resource is soft-deleted."
+                        + " Continuing.");
+              }
+
+              EppResourceIndex eppResourceIndex =
+                  auditedOfy().load().entity(EppResourceIndex.create(targetKey)).now();
+              verify(eppResourceIndex.getKey().equals(targetKey), "Wrong EppResource Index loaded");
+
+              ImmutableList.Builder<Key<Object>> toDeleteBuilder =
+                  new ImmutableList.Builder<Key<Object>>()
+                      .addAll(objectsInEntityGroup)
+                      .add(Key.create(eppResourceIndex));
+              fki.ifPresent(f -> toDeleteBuilder.add(Key.create(f)));
+              toDelete = toDeleteBuilder.build();
+
+              System.out.printf("\n\nAbout to delete %d entities with keys:\n", toDelete.size());
+              toDelete.forEach(System.out::println);
+            });
+  }
+
+  @Override
+  protected String execute() {
+    tm().transact(() -> auditedOfy().delete().keys(toDelete).now());
+    return "Done.";
+  }
+}

core/src/main/resources/google/registry/beam/bulk_delete_datastore_pipeline_metadata.json

@@ -2,6 +2,15 @@
   "name": "Bulk Delete Cloud Datastore",
   "description": "An Apache Beam batch pipeline that deletes Cloud Datastore in bulk. This is easier to use than the GCP-provided template.",
   "parameters": [
+    {
+      "name": "registryEnvironment",
+      "label": "The Registry environment.",
+      "helpText": "The Registry environment, required only because the worker initializer demands it.",
+      "is_optional": false,
+      "regexes": [
+        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
+      ]
+    },
     {
       "name": "kindsToDelete",
       "label": "The data KINDs to delete.",

core/src/main/resources/google/registry/beam/init_sql_pipeline_metadata.json

@@ -5,10 +5,10 @@
     {
       "name": "registryEnvironment",
       "label": "The Registry environment.",
-      "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.",
-      "is_optional": true,
+      "helpText": "The Registry environment.",
+      "is_optional": false,
       "regexes": [
-        "^[0-9A-Z_]+$"
+        "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"
       ]
     },
     {

core/src/main/resources/google/registry/beam/invoicing_pipeline_metadata.json

@@ -5,7 +5,7 @@
     {
       "name": "registryEnvironment",
       "label": "The Registry environment.",
-      "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.",
+      "helpText": "The Registry environment.",
       "is_optional": false,
       "regexes": [
         "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"

core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json

@@ -5,7 +5,7 @@
     {
       "name": "registryEnvironment",
       "label": "The Registry environment.",
-      "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.",
+      "helpText": "The Registry environment.",
       "is_optional": false,
       "regexes": [
         "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"

core/src/main/resources/google/registry/beam/spec11_pipeline_metadata.json

@@ -5,7 +5,7 @@
     {
       "name": "registryEnvironment",
       "label": "The Registry environment.",
-      "helpText": "The Registry environment, required if environment-specific initialization (such as JPA) is needed on worker VMs.",
+      "helpText": "The Registry environment.",
       "is_optional": false,
       "regexes": [
         "^PRODUCTION|SANDBOX|CRASH|QA|ALPHA$"

core/src/main/resources/google/registry/tools/soy/DomainRenew.soy

@@ -21,6 +21,8 @@
 {@param domainName: string}
 {@param expirationDate: string}
 {@param period: string}
+{@param? reason: string}
+{@param? requestedByRegistrar: string}
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
   <command>
@@ -32,6 +34,18 @@
         <domain:period unit="y">{$period}</domain:period>
       </domain:renew>
     </renew>
+    {if $reason or $requestedByRegistrar}
+      <extension>
+          <metadata:metadata xmlns:metadata="urn:google:params:xml:ns:metadata-1.0">
+            {if $reason}
+              <metadata:reason>{$reason}</metadata:reason>
+            {/if}
+            {if $requestedByRegistrar}
+              <metadata:requestedByRegistrar>{$requestedByRegistrar}</metadata:requestedByRegistrar>
+            {/if}
+          </metadata:metadata>
+      </extension>
+    {/if}
     <clTRID>RegistryTool</clTRID>
   </command>
 </epp>

core/src/test/java/google/registry/batch/SendExpiringCertificateNotificationEmailActionTest.java

@@ -639,9 +639,32 @@ class SendExpiringCertificateNotificationEmailActionTest {
   }
 
   @TestOfyAndSql
-  void run_responseStatusIs200() {
+  void run_sentZeroEmail_responseStatusIs200() {
     action.run();
     assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(response.getPayload())
+        .isEqualTo("Done. Sent 0 expiring certificate notification emails in total.");
+  }
+
+  @TestOfyAndSql
+  void run_sentEmails_responseStatusIs200() throws Exception {
+    for (int i = 1; i <= 5; i++) {
+      persistResource(
+          createRegistrar(
+                  "id_" + i,
+                  "name" + i,
+                  SelfSignedCaCertificate.create(
+                          "www.example.tld",
+                          DateTime.parse("2020-09-02T00:00:00Z"),
+                          DateTime.parse("2021-06-01T00:00:00Z"))
+                      .cert(),
+                  null)
+              .build());
+    }
+    action.run();
+    assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(response.getPayload())
+        .isEqualTo("Done. Sent 5 expiring certificate notification emails in total.");
   }
 
   /** Returns a sample registrar with a customized registrar name, client id and certificate* */

core/src/test/java/google/registry/batch/WipeOutContactHistoryPiiActionTest.java

@@ -23,7 +23,6 @@ import static org.apache.http.HttpStatus.SC_OK;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.truth.Truth8;
 import google.registry.model.contact.ContactAddress;
 import google.registry.model.contact.ContactAuthInfo;
 import google.registry.model.contact.ContactBase;
@@ -50,8 +49,8 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 @DualDatabaseTest
 class WipeOutContactHistoryPiiActionTest {
 
+  private static final int TEST_BATCH_SIZE = 20;
   private static final int MIN_MONTHS_BEFORE_WIPE_OUT = 18;
-  private static final int BATCH_SIZE = 500;
   private static final ContactResource defaultContactResource =
       new ContactResource.Builder()
           .setContactId("sh8013")
@@ -115,7 +114,8 @@ class WipeOutContactHistoryPiiActionTest {
   void beforeEach() {
     response = new FakeResponse();
     action =
-        new WipeOutContactHistoryPiiAction(clock, MIN_MONTHS_BEFORE_WIPE_OUT, BATCH_SIZE, response);
+        new WipeOutContactHistoryPiiAction(
+            clock, MIN_MONTHS_BEFORE_WIPE_OUT, TEST_BATCH_SIZE, response);
   }
 
   @TestSqlOnly
@@ -133,10 +133,10 @@ class WipeOutContactHistoryPiiActionTest {
   }
 
   @TestSqlOnly
-  void getAllHistoryEntitiesOlderThan_returnsOnlyPartOfThePersistedEntities() {
+  void getAllHistoryEntitiesOlderThan_returnsOnlyOldEnoughPersistedEntities() {
     ImmutableList<ContactHistory> expectedToBeWipedOut =
         persistLotsOfContactHistoryEntities(
-            40, MIN_MONTHS_BEFORE_WIPE_OUT + 2, 0, defaultContactResource);
+            19, MIN_MONTHS_BEFORE_WIPE_OUT + 2, 0, defaultContactResource);
 
     // persisted entities that should not be part of the actual result
     persistLotsOfContactHistoryEntities(
@@ -145,7 +145,7 @@ class WipeOutContactHistoryPiiActionTest {
     jpaTm()
         .transact(
             () ->
-                Truth8.assertThat(
+                assertThat(
                         action.getNextContactHistoryEntitiesWithPiiBatch(
                             clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT)))
                     .containsExactlyElementsIn(expectedToBeWipedOut));
@@ -175,6 +175,8 @@ class WipeOutContactHistoryPiiActionTest {
         .isEqualTo(0);
 
     assertThat(response.getStatus()).isEqualTo(SC_OK);
+    assertThat(response.getPayload())
+        .isEqualTo("Done. Wiped out PII of 0 ContactHistory entities in total.");
   }
 
   @TestSqlOnly
@@ -197,6 +199,8 @@ class WipeOutContactHistoryPiiActionTest {
     assertAllEntitiesContainPii(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut));
 
     action.run();
+    assertThat(response.getPayload())
+        .isEqualTo("Done. Wiped out PII of 20 ContactHistory entities in total.");
 
     // The query should return an empty stream after the wipe out action.
     assertThat(
@@ -216,7 +220,7 @@ class WipeOutContactHistoryPiiActionTest {
   void run_withMultipleBatches_numOfEntitiesAsNonMultipleOfBatchSize_success() {
     int numOfMonthsFromNow = MIN_MONTHS_BEFORE_WIPE_OUT + 2;
     ImmutableList<ContactHistory> expectedToBeWipedOut =
-        persistLotsOfContactHistoryEntities(1234, numOfMonthsFromNow, 0, defaultContactResource);
+        persistLotsOfContactHistoryEntities(56, numOfMonthsFromNow, 0, defaultContactResource);
 
     // The query should return a subset of all persisted data.
     assertThat(
@@ -227,10 +231,12 @@ class WipeOutContactHistoryPiiActionTest {
                             .getNextContactHistoryEntitiesWithPiiBatch(
                                 clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT))
                             .count()))
-        .isEqualTo(BATCH_SIZE);
+        .isEqualTo(TEST_BATCH_SIZE);
 
     assertAllEntitiesContainPii(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut));
     action.run();
+    assertThat(response.getPayload())
+        .isEqualTo("Done. Wiped out PII of 56 ContactHistory entities in total.");
 
     // The query should return an empty stream after the wipe out action.
     assertThat(
@@ -250,7 +256,8 @@ class WipeOutContactHistoryPiiActionTest {
   void run_withMultipleBatches_numOfEntitiesAsMultiplesOfBatchSize_success() {
     int numOfMonthsFromNow = MIN_MONTHS_BEFORE_WIPE_OUT + 2;
     ImmutableList<ContactHistory> expectedToBeWipedOut =
-        persistLotsOfContactHistoryEntities(2000, numOfMonthsFromNow, 0, defaultContactResource);
+        persistLotsOfContactHistoryEntities(
+            TEST_BATCH_SIZE * 2, numOfMonthsFromNow, 0, defaultContactResource);
 
     // The query should return a subset of all persisted data.
     assertThat(
@@ -261,10 +268,12 @@ class WipeOutContactHistoryPiiActionTest {
                             .getNextContactHistoryEntitiesWithPiiBatch(
                                 clock.nowUtc().minusMonths(MIN_MONTHS_BEFORE_WIPE_OUT))
                             .count()))
-        .isEqualTo(BATCH_SIZE);
+        .isEqualTo(TEST_BATCH_SIZE);
 
     assertAllEntitiesContainPii(DatabaseHelper.loadByEntitiesIfPresent(expectedToBeWipedOut));
     action.run();
+    assertThat(response.getPayload())
+        .isEqualTo("Done. Wiped out PII of 40 ContactHistory entities in total.");
 
     // The query should return an empty stream after the wipe out action.
     assertThat(

core/src/test/java/google/registry/beam/initsql/TransformsTest.java

@@ -0,0 +1,64 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.beam.initsql;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.beam.initsql.Transforms.repairBadData;
+import static google.registry.model.ofy.ObjectifyService.auditedOfy;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.newHostResource;
+
+import com.google.appengine.api.datastore.Entity;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.host.HostResource;
+import google.registry.testing.AppEngineExtension;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit tests for {@link Transforms}. */
+public class TransformsTest {
+
+  @RegisterExtension
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder().withDatastoreAndCloudSql().build();
+
+  @BeforeEach
+  void beforeEach() {
+    createTld("tld");
+  }
+
+  @Test
+  void testRepairBadData_canonicalizesDomainName() {
+    DomainBase domain = newDomainBase("foobar.tld");
+    Entity entity = ofyTm().transact(() -> auditedOfy().toEntity(domain));
+    entity.setIndexedProperty("fullyQualifiedDomainName", "FOOBäR.TLD");
+    assertThat(((DomainBase) auditedOfy().toPojo(repairBadData(entity))).getDomainName())
+        .isEqualTo("xn--foobr-jra.tld");
+  }
+
+  @Test
+  void testRepairBadData_canonicalizesHostName() {
+    HostResource host = newHostResource("baz.foobar.tld");
+    Entity entity = ofyTm().transact(() -> auditedOfy().toEntity(host));
+    entity.setIndexedProperty(
+        "fullyQualifiedHostName", "b̴̹͔͓̣̭̫͇͕̻̬̱͇͗͌́̆̋͒a̶̬̖͚̋̈́̽̇͝͠z̵͠.FOOBäR.TLD");
+    assertThat(((HostResource) auditedOfy().toPojo(repairBadData(entity))).getHostName())
+        .isEqualTo(
+            "xn--baz-kdcb2ajgzb4jtg6doej4e6b9am7c7b6c5nd4k7gpa2a9a7dufyewec.xn--foobr-jra.tld");
+  }
+}

core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java

@@ -17,9 +17,6 @@ package google.registry.beam.invoicing;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.tld.Registry.TldState.GENERAL_AVAILABILITY;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.removeTmOverrideForTest;
-import static google.registry.persistence.transaction.TransactionManagerFactory.setTmForTest;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.newRegistry;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
@@ -48,6 +45,7 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT
 import google.registry.testing.DatastoreEntityExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.TestDataHelper;
+import google.registry.testing.TmOverrideExtension;
 import google.registry.util.ResourceUtils;
 import java.io.File;
 import java.nio.file.Files;
@@ -77,6 +75,25 @@ import org.junit.jupiter.api.io.TempDir;
 /** Unit tests for {@link InvoicingPipeline}. */
 class InvoicingPipelineTest {
 
+  @RegisterExtension
+  @Order(Order.DEFAULT - 1)
+  final transient DatastoreEntityExtension datastore =
+      new DatastoreEntityExtension().allThreads(true);
+
+  @RegisterExtension
+  final TestPipelineExtension pipeline =
+      TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);
+
+  @RegisterExtension
+  final JpaIntegrationTestExtension database =
+      new JpaTestExtensions.Builder().withClock(new FakeClock()).buildIntegrationTestExtension();
+
+  @RegisterExtension
+  @Order(Order.DEFAULT + 1)
+  TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa();
+
+  @TempDir Path tmpDir;
+
   private static final String BILLING_BUCKET_URL = "billing_bucket";
   private static final String YEAR_MONTH = "2017-10";
   private static final String INVOICE_FILE_PREFIX = "REG-INV";
@@ -225,21 +242,6 @@ class InvoicingPipelineTest {
           "2017-10-01,2018-09-30,456,20.50,USD,10125,1,PURCHASE,bestdomains - test,1,"
               + "RENEW | TLD: test | TERM: 1-year,20.50,USD,116688");
 
-  @RegisterExtension
-  @Order(Order.DEFAULT - 1)
-  final transient DatastoreEntityExtension datastore =
-      new DatastoreEntityExtension().allThreads(true);
-
-  @RegisterExtension
-  final TestPipelineExtension pipeline =
-      TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);
-
-  @RegisterExtension
-  final JpaIntegrationTestExtension database =
-      new JpaTestExtensions.Builder().withClock(new FakeClock()).buildIntegrationTestExtension();
-
-  @TempDir Path tmpDir;
-
   private final InvoicingPipelineOptions options =
       PipelineOptionsFactory.create().as(InvoicingPipelineOptions.class);
 
@@ -261,13 +263,12 @@ class InvoicingPipelineTest {
     String query = InvoicingPipeline.makeQuery("2017-10", "my-project-id");
     assertThat(query)
         .isEqualTo(TestDataHelper.loadFile(this.getClass(), "billing_events_test.sql"));
-    // This is necessary because the TestPipelineExtension verifies that the pipelien is run.
+    // This is necessary because the TestPipelineExtension verifies that the pipeline is run.
     pipeline.run();
   }
 
   @Test
   void testSuccess_fullSqlPipeline() throws Exception {
-    setTmForTest(jpaTm());
     setupCloudSql();
     options.setDatabase("CLOUD_SQL");
     InvoicingPipeline invoicingPipeline = new InvoicingPipeline(options);
@@ -282,18 +283,15 @@ class InvoicingPipelineTest {
                 + "UnitPriceCurrency,PONumber");
     assertThat(overallInvoice.subList(1, overallInvoice.size()))
         .containsExactlyElementsIn(EXPECTED_INVOICE_OUTPUT);
-    removeTmOverrideForTest();
   }
 
   @Test
   void testSuccess_readFromCloudSql() throws Exception {
-    setTmForTest(jpaTm());
     setupCloudSql();
     PCollection<BillingEvent> billingEvents = InvoicingPipeline.readFromCloudSql(options, pipeline);
     billingEvents = billingEvents.apply(new ChangeDomainRepo());
     PAssert.that(billingEvents).containsInAnyOrder(INPUT_EVENTS);
     pipeline.run().waitUntilFinish();
-    removeTmOverrideForTest();
   }
 
   @Test

core/src/test/java/google/registry/beam/rde/RdePipelineTest.java

@@ -22,7 +22,6 @@ import static google.registry.beam.rde.RdePipeline.encodePendings;
 import static google.registry.model.common.Cursor.CursorType.RDE_STAGING;
 import static google.registry.model.rde.RdeMode.FULL;
 import static google.registry.model.rde.RdeMode.THIN;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.rde.RdeResourceType.CONTACT;
 import static google.registry.rde.RdeResourceType.DOMAIN;
@@ -64,7 +63,6 @@ import google.registry.model.tld.Registry;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
-import google.registry.persistence.transaction.TransactionManagerFactory;
 import google.registry.rde.DepositFragment;
 import google.registry.rde.Ghostryde;
 import google.registry.rde.PendingDeposit;
@@ -74,6 +72,7 @@ import google.registry.testing.CloudTasksHelper.TaskMatcher;
 import google.registry.testing.DatastoreEntityExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeKeyringModule;
+import google.registry.testing.TmOverrideExtension;
 import java.io.IOException;
 import java.util.function.Function;
 import java.util.regex.Matcher;
@@ -88,7 +87,6 @@ import org.bouncycastle.openpgp.PGPPrivateKey;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
@@ -156,6 +154,10 @@ public class RdePipelineTest {
   final JpaIntegrationTestExtension database =
       new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension();
 
+  @RegisterExtension
+  @Order(Order.DEFAULT + 1)
+  TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa();
+
   @RegisterExtension
   final TestPipelineExtension pipeline =
       TestPipelineExtension.fromOptions(options).enableAbandonedNodeEnforcement(true);
@@ -164,7 +166,6 @@ public class RdePipelineTest {
 
   @BeforeEach
   void beforeEach() throws Exception {
-    TransactionManagerFactory.setTmForTest(jpaTm());
     loadInitialData();
 
     // Two real registrars have been created by loadInitialData(), named "New Registrar" and "The
@@ -221,11 +222,6 @@ public class RdePipelineTest {
     rdePipeline = new RdePipeline(options, gcsUtils, cloudTasksHelper.getTestCloudTasksUtils());
   }
 
-  @AfterEach
-  void afterEach() {
-    TransactionManagerFactory.removeTmOverrideForTest();
-  }
-
   @Test
   void testSuccess_encodeAndDecodePendingsMap() throws Exception {
     String encodedString = encodePendings(pendings);

core/src/test/java/google/registry/beam/spec11/Spec11PipelineTest.java

@@ -18,8 +18,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.ImmutableObjectSubject.immutableObjectCorrespondence;
 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.removeTmOverrideForTest;
-import static google.registry.persistence.transaction.TransactionManagerFactory.setTmForTest;
 import static google.registry.testing.AppEngineExtension.makeRegistrar1;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistActiveContact;
@@ -52,6 +50,7 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT
 import google.registry.testing.DatastoreEntityExtension;
 import google.registry.testing.FakeClock;
 import google.registry.testing.FakeSleeper;
+import google.registry.testing.TmOverrideExtension;
 import google.registry.util.ResourceUtils;
 import google.registry.util.Retrier;
 import java.io.File;
@@ -129,6 +128,10 @@ class Spec11PipelineTest {
   final JpaIntegrationTestExtension database =
       new JpaTestExtensions.Builder().withClock(new FakeClock()).buildIntegrationTestExtension();
 
+  @RegisterExtension
+  @Order(Order.DEFAULT + 1)
+  TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa();
+
   private final Spec11PipelineOptions options =
       PipelineOptionsFactory.create().as(Spec11PipelineOptions.class);
 
@@ -233,7 +236,6 @@ class Spec11PipelineTest {
   }
 
   private void setupCloudSql() {
-    setTmForTest(jpaTm());
     persistNewRegistrar("TheRegistrar");
     persistNewRegistrar("NewRegistrar");
     Registrar registrar1 =
@@ -273,7 +275,6 @@ class Spec11PipelineTest {
     persistResource(createDomain("no-email.com", "2A4BA9BBC-COM", registrar2, contact2));
     persistResource(
         createDomain("anti-anti-anti-virus.dev", "555666888-DEV", registrar3, contact3));
-    removeTmOverrideForTest();
   }
 
   private void verifySaveToGcs() throws Exception {

core/src/test/java/google/registry/flows/domain/DomainRenewFlowTest.java

@@ -42,6 +42,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.flows.EppException;
+import google.registry.flows.EppRequestSource;
 import google.registry.flows.FlowUtils.UnknownCurrencyEppException;
 import google.registry.flows.ResourceFlowTestCase;
 import google.registry.flows.ResourceFlowUtils.ResourceDoesNotExistException;
@@ -533,6 +534,55 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, DomainBa
             .build());
   }
 
+  @TestOfyAndSql
+  void testSuccess_metaData_withReasonAndRequestedByRegistrar() throws Exception {
+    eppRequestSource = EppRequestSource.TOOL;
+    setEppInput(
+        "domain_renew_metadata_with_reason_and_requestedByRegistrar.xml",
+        ImmutableMap.of(
+            "DOMAIN",
+            "example.tld",
+            "EXPDATE",
+            "2000-04-03",
+            "YEARS",
+            "1",
+            "REASON",
+            "domain-renew-test",
+            "REQUESTED",
+            "false"));
+    persistDomain();
+    runFlow();
+    DomainBase domain = reloadResourceByForeignKey();
+    assertAboutDomains()
+        .that(domain)
+        .hasOneHistoryEntryEachOfTypes(
+            HistoryEntry.Type.DOMAIN_CREATE, HistoryEntry.Type.DOMAIN_RENEW);
+    assertAboutHistoryEntries()
+        .that(getOnlyHistoryEntryOfType(domain, HistoryEntry.Type.DOMAIN_RENEW))
+        .hasMetadataReason("domain-renew-test")
+        .and()
+        .hasMetadataRequestedByRegistrar(false);
+  }
+
+  @TestOfyAndSql
+  void testSuccess_metaData_withRequestedByRegistrarOnly() throws Exception {
+    eppRequestSource = EppRequestSource.TOOL;
+    setEppInput("domain_renew_metadata_with_requestedByRegistrar_only.xml");
+
+    persistDomain();
+    runFlow();
+    DomainBase domain1 = reloadResourceByForeignKey();
+    assertAboutDomains()
+        .that(domain1)
+        .hasOneHistoryEntryEachOfTypes(
+            HistoryEntry.Type.DOMAIN_CREATE, HistoryEntry.Type.DOMAIN_RENEW);
+    assertAboutHistoryEntries()
+        .that(getOnlyHistoryEntryOfType(domain1, HistoryEntry.Type.DOMAIN_RENEW))
+        .hasMetadataReason(null)
+        .and()
+        .hasMetadataRequestedByRegistrar(true);
+  }
+
   @TestOfyAndSql
   void testFailure_neverExisted() throws Exception {
     ResourceDoesNotExistException thrown =

core/src/test/java/google/registry/model/domain/DomainBaseTest.java

@@ -696,7 +696,7 @@ public class DomainBaseTest extends EntityTestCase {
             IllegalArgumentException.class, () -> domain.asBuilder().setDomainName("AAA.BBB"));
     assertThat(thrown)
         .hasMessageThat()
-        .contains("Domain name must be in puny-coded, lower-case form");
+        .isEqualTo("Domain name AAA.BBB not in puny-coded, lower-case form");
   }
 
   @Test
@@ -706,7 +706,7 @@ public class DomainBaseTest extends EntityTestCase {
             IllegalArgumentException.class, () -> domain.asBuilder().setDomainName("みんな.みんな"));
     assertThat(thrown)
         .hasMessageThat()
-        .contains("Domain name must be in puny-coded, lower-case form");
+        .isEqualTo("Domain name みんな.みんな not in puny-coded, lower-case form");
   }
 
   @Test

core/src/test/java/google/registry/model/host/HostResourceTest.java

@@ -200,7 +200,7 @@ class HostResourceTest extends EntityTestCase {
             IllegalArgumentException.class, () -> host.asBuilder().setHostName("AAA.BBB.CCC"));
     assertThat(thrown)
         .hasMessageThat()
-        .contains("Host name must be in puny-coded, lower-case form");
+        .isEqualTo("Host name AAA.BBB.CCC not in puny-coded, lower-case form");
   }
 
   @TestOfyAndSql
@@ -210,7 +210,7 @@ class HostResourceTest extends EntityTestCase {
             IllegalArgumentException.class, () -> host.asBuilder().setHostName("みんな.みんな.みんな"));
     assertThat(thrown)
         .hasMessageThat()
-        .contains("Host name must be in puny-coded, lower-case form");
+        .isEqualTo("Host name みんな.みんな.みんな not in puny-coded, lower-case form");
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/model/replay/EntityTest.java

@@ -51,8 +51,7 @@ public class EntityTest {
 
   @Test
   void testSqlEntityPersistence() {
-    try (ScanResult scanResult =
-        new ClassGraph().enableAnnotationInfo().whitelistPackages("google.registry").scan()) {
+    try (ScanResult scanResult = scanForClasses()) {
       // All javax.persistence entities must implement SqlEntity and vice versa
       ImmutableSet<String> javaxPersistenceClasses =
           getAllClassesWithAnnotation(scanResult, javax.persistence.Entity.class.getName());
@@ -75,8 +74,7 @@ public class EntityTest {
     // For replication, we need to be able to convert from Key -> VKey for the relevant classes.
     // This means that the relevant classes must have non-composite Objectify keys or must have a
     // createVKey method
-    try (ScanResult scanResult =
-        new ClassGraph().enableAnnotationInfo().whitelistPackages("google.registry").scan()) {
+    try (ScanResult scanResult = scanForClasses()) {
       ImmutableSet<Class<?>> datastoreEntityClasses =
           getClasses(scanResult.getClassesImplementing(DatastoreEntity.class.getName()));
       // some classes aren't converted so they aren't relevant
@@ -126,9 +124,12 @@ public class EntityTest {
     return classInfoList.stream()
         .filter(ClassInfo::isStandardClass)
         .map(ClassInfo::loadClass)
-        .filter(clazz -> !clazz.isAnnotationPresent(EntityForTesting.class))
-        .filter(clazz -> !clazz.isAnnotationPresent(Embed.class))
-        .filter(clazz -> !NON_CONVERTED_CLASSES.contains(clazz))
+        .filter(
+            clazz ->
+                !clazz.isAnnotationPresent(EntityForTesting.class)
+                    && !clazz.isAnnotationPresent(Embed.class)
+                    && !NON_CONVERTED_CLASSES.contains(clazz)
+                    && !clazz.getName().contains("Test"))
         .collect(toImmutableSet());
   }
 
@@ -136,6 +137,14 @@ public class EntityTest {
     return getClasses(classInfoList).stream().map(Class::getName).collect(toImmutableSet());
   }
 
+  private ScanResult scanForClasses() {
+    return new ClassGraph()
+        .enableAnnotationInfo()
+        .ignoreClassVisibility()
+        .acceptPackages("google.registry")
+        .scan();
+  }
+
   /** Entities that are solely used for testing, to avoid scanning them in {@link EntityTest}. */
   @Target(ElementType.TYPE)
   @Retention(RetentionPolicy.RUNTIME)

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -49,8 +49,7 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.junitpioneer.jupiter.RetryingTest;
@@ -74,7 +73,8 @@ public class ReplicateToDatastoreActionTest {
   private ReplicateToDatastoreAction action;
   private FakeResponse response;
 
-  @BeforeEach
+  // TODO(b/197534789): fix these tests and re-add the @BeforeEach
+  // @BeforeEach
   void setUp() {
     resetAction();
     injectExtension.setStaticField(Ofy.class, "clock", fakeClock);
@@ -88,13 +88,15 @@ public class ReplicateToDatastoreActionTest {
     TestObject.beforeDatastoreSaveCallCount = 0;
   }
 
-  @AfterEach
+  // TODO(b/197534789): fix these tests and re-add the @AfterEach
+  // @AfterEach
   void tearDown() {
     DatabaseHelper.removeDatabaseMigrationSchedule();
     fakeClock.disableAutoIncrement();
   }
 
   @RetryingTest(4)
+  @Disabled("b/197534789")
   void testReplication() {
     TestObject foo = TestObject.create("foo");
     TestObject bar = TestObject.create("bar");
@@ -120,6 +122,7 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @RetryingTest(4)
+  @Disabled("b/197534789")
   void testReplayFromLastTxn() {
     TestObject foo = TestObject.create("foo");
     TestObject bar = TestObject.create("bar");
@@ -142,6 +145,7 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @RetryingTest(4)
+  @Disabled("b/197534789")
   void testUnintentionalConcurrency() {
     TestObject foo = TestObject.create("foo");
     TestObject bar = TestObject.create("bar");
@@ -177,6 +181,7 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @RetryingTest(4)
+  @Disabled("b/197534789")
   void testMissingTransactions() {
     // Write a transaction (should have a transaction id of 1).
     TestObject foo = TestObject.create("foo");
@@ -194,6 +199,7 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @Test
+  @Disabled("b/197534789")
   void testMissingTransactions_fullTask() {
     // Write a transaction (should have a transaction id of 1).
     TestObject foo = TestObject.create("foo");
@@ -212,6 +218,7 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @Test
+  @Disabled("b/197534789")
   void testBeforeDatastoreSaveCallback() {
     TestObject testObject = TestObject.create("foo");
     insertInDb(testObject);
@@ -221,6 +228,7 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @Test
+  @Disabled("b/197534789")
   void testNotInMigrationState_doesNothing() {
     // set a schedule that backtracks the current status to DATASTORE_PRIMARY
     DateTime now = fakeClock.nowUtc();
@@ -257,6 +265,7 @@ public class ReplicateToDatastoreActionTest {
   }
 
   @Test
+  @Disabled("b/197534789")
   void testFailure_cannotAcquireLock() {
     RequestStatusChecker requestStatusChecker = mock(RequestStatusChecker.class);
     when(requestStatusChecker.getLogId()).thenReturn("logId");

core/src/test/java/google/registry/model/smd/SignedMarkRevocationListTest.java

@@ -30,7 +30,8 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 public class SignedMarkRevocationListTest {
 
   @RegisterExtension
-  public final AppEngineExtension appEngine = AppEngineExtension.builder().withCloudSql().build();
+  public final AppEngineExtension appEngine =
+      AppEngineExtension.builder().withDatastoreAndCloudSql().build();
 
   private final FakeClock clock = new FakeClock(DateTime.parse("2013-01-01T00:00:00Z"));
 

core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java

@@ -36,6 +36,7 @@ import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaUnitTestExtension;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.FakeClock;
+import google.registry.testing.TmOverrideExtension;
 import java.io.Serializable;
 import java.math.BigInteger;
 import java.sql.SQLException;
@@ -48,8 +49,7 @@ import javax.persistence.IdClass;
 import javax.persistence.OptimisticLockException;
 import javax.persistence.RollbackException;
 import org.hibernate.exception.JDBCConnectionException;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -84,15 +84,9 @@ class JpaTransactionManagerImplTest {
               TestEntity.class, TestCompoundIdEntity.class, TestNamedCompoundIdEntity.class)
           .buildUnitTestExtension();
 
-  @BeforeEach
-  void beforeEach() {
-    TransactionManagerFactory.setTmForTest(jpaTm());
-  }
-
-  @AfterEach
-  void afterEach() {
-    TransactionManagerFactory.removeTmOverrideForTest();
-  }
+  @RegisterExtension
+  @Order(Order.DEFAULT + 1)
+  TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa();
 
   @Test
   void transact_succeeds() {

core/src/test/java/google/registry/rdap/RdapDomainActionTest.java

@@ -45,7 +45,12 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
-/** Unit tests for {@link RdapDomainAction}. */
+/**
+ * Unit tests for {@link RdapDomainAction}.
+ *
+ * <p>TODO(b/26872828): The next time we do any work on RDAP, consider adding the APNIC RDAP
+ * conformance checker to the unit test suite.
+ */
 class RdapDomainActionTest extends RdapActionBaseTestCase<RdapDomainAction> {
 
   RdapDomainActionTest() {

core/src/test/java/google/registry/reporting/icann/IcannReportingUploadActionTest.java

@@ -294,8 +294,9 @@ class IcannReportingUploadActionTest {
         .that(logHandler)
         .hasLogAtLevelWithMessage(
             Level.SEVERE,
-            "Could not upload ICANN_UPLOAD_ACTIVITY report for tld because file"
-                + " tld-activity-200512.csv did not exist");
+            "Could not upload ICANN_UPLOAD_ACTIVITY report for tld because file "
+                + "tld-activity-200512.csv (object icann/monthly/2005-12/tld-activity-200512.csv in"
+                + " bucket basin) did not exist.");
   }
 
   @TestOfyAndSql
@@ -310,9 +311,9 @@ class IcannReportingUploadActionTest {
         .that(logHandler)
         .hasLogAtLevelWithMessage(
             Level.INFO,
-            "Could not upload ICANN_UPLOAD_ACTIVITY report for foo because file"
-                + " foo-activity-200607.csv did not exist. This report may not have been staged"
-                + " yet.");
+            "Could not upload ICANN_UPLOAD_ACTIVITY report for foo because file "
+                + "foo-activity-200607.csv (object icann/monthly/2006-07/foo-activity-200607.csv in"
+                + " bucket basin) did not exist. This report may not have been staged yet.");
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/schema/registrar/RegistrarContactTest.java

@@ -16,8 +16,6 @@ package google.registry.schema.registrar;
 
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.registrar.RegistrarContact.Type.WHOIS;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
-import static google.registry.persistence.transaction.TransactionManagerFactory.setTmForTest;
 import static google.registry.testing.DatabaseHelper.insertInDb;
 import static google.registry.testing.DatabaseHelper.loadByEntity;
 import static google.registry.testing.SqlHelper.saveRegistrar;
@@ -28,6 +26,7 @@ import google.registry.model.registrar.RegistrarContact;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
 import google.registry.testing.DatastoreEntityExtension;
+import google.registry.testing.TmOverrideExtension;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
@@ -44,13 +43,16 @@ class RegistrarContactTest {
   JpaIntegrationWithCoverageExtension jpa =
       new JpaTestExtensions.Builder().buildIntegrationWithCoverageExtension();
 
+  @RegisterExtension
+  @Order(Order.DEFAULT + 1)
+  TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa();
+
   private Registrar testRegistrar;
 
   private RegistrarContact testRegistrarPoc;
 
   @BeforeEach
   public void beforeEach() {
-    setTmForTest(jpaTm());
     testRegistrar = saveRegistrar("registrarId");
     testRegistrarPoc =
         new RegistrarContact.Builder()

core/src/test/java/google/registry/schema/registrar/RegistrarDaoTest.java

@@ -15,7 +15,6 @@
 package google.registry.schema.registrar;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
 import static google.registry.testing.DatabaseHelper.existsInDb;
 import static google.registry.testing.DatabaseHelper.insertInDb;
 import static google.registry.testing.DatabaseHelper.loadByKey;
@@ -29,11 +28,10 @@ import google.registry.model.registrar.RegistrarAddress;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
-import google.registry.persistence.transaction.TransactionManagerFactory;
 import google.registry.testing.DatastoreEntityExtension;
 import google.registry.testing.FakeClock;
+import google.registry.testing.TmOverrideExtension;
 import org.joda.time.DateTime;
-import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
@@ -52,13 +50,16 @@ public class RegistrarDaoTest {
   JpaIntegrationWithCoverageExtension jpa =
       new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension();
 
+  @RegisterExtension
+  @Order(Order.DEFAULT + 1)
+  TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa();
+
   private final VKey<Registrar> registrarKey = VKey.createSql(Registrar.class, "registrarId");
 
   private Registrar testRegistrar;
 
   @BeforeEach
   void beforeEach() {
-    TransactionManagerFactory.setTmForTest(jpaTm());
     testRegistrar =
         new Registrar.Builder()
             .setType(Registrar.Type.TEST)
@@ -75,11 +76,6 @@ public class RegistrarDaoTest {
             .build();
   }
 
-  @AfterEach
-  void afterEach() {
-    TransactionManagerFactory.removeTmOverrideForTest();
-  }
-
   @Test
   void saveNew_worksSuccessfully() {
     assertThat(existsInDb(testRegistrar)).isFalse();

core/src/test/java/google/registry/schema/replay/SqlEntityTest.java

@@ -21,10 +21,9 @@ import google.registry.model.registrar.Registrar;
 import google.registry.model.registrar.RegistrarContact;
 import google.registry.model.registrar.RegistrarContact.RegistrarPocId;
 import google.registry.persistence.VKey;
-import google.registry.persistence.transaction.TransactionManagerFactory;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DatastoreEntityExtension;
-import org.junit.jupiter.api.AfterEach;
+import google.registry.testing.TmOverrideExtension;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
@@ -41,17 +40,15 @@ public class SqlEntityTest {
   final AppEngineExtension database =
       new AppEngineExtension.Builder().withCloudSql().withoutCannedData().build();
 
+  @RegisterExtension
+  @Order(Order.DEFAULT + 1)
+  TmOverrideExtension tmOverrideExtension = TmOverrideExtension.withJpa();
+
   @BeforeEach
   void setup() throws Exception {
-    TransactionManagerFactory.setTmForTest(TransactionManagerFactory.jpaTm());
     AppEngineExtension.loadInitialData();
   }
 
-  @AfterEach
-  void teardown() {
-    TransactionManagerFactory.removeTmOverrideForTest();
-  }
-
   @Test
   void getPrimaryKeyString_oneIdColumn() {
     // AppEngineExtension canned data: Registrar1

core/src/test/java/google/registry/testing/AppEngineExtension.java

@@ -487,20 +487,22 @@ public final class AppEngineExtension implements BeforeEachCallback, AfterEachCa
       if (replayer != null) {
         replayer.replay();
       }
-
-      if (withCloudSql) {
-        if (enableJpaEntityCoverageCheck) {
-          jpaIntegrationWithCoverageExtension.afterEach(context);
-        } else if (withJpaUnitTest) {
-          jpaUnitTestExtension.afterEach(context);
-        } else {
-          jpaIntegrationTestExtension.afterEach(context);
-        }
-      }
-      tearDown();
     } finally {
-      if (isWithDatastoreAndCloudSql()) {
-        restoreTmAfterDualDatabaseTest(context);
+      try {
+        if (withCloudSql) {
+          if (enableJpaEntityCoverageCheck) {
+            jpaIntegrationWithCoverageExtension.afterEach(context);
+          } else if (withJpaUnitTest) {
+            jpaUnitTestExtension.afterEach(context);
+          } else {
+            jpaIntegrationTestExtension.afterEach(context);
+          }
+        }
+        tearDown();
+      } finally {
+        if (isWithDatastoreAndCloudSql()) {
+          restoreTmAfterDualDatabaseTest(context);
+        }
       }
     }
   }

core/src/test/java/google/registry/testing/DualDatabaseTestInvocationContextProvider.java

@@ -154,7 +154,7 @@ class DualDatabaseTestInvocationContextProvider implements TestTemplateInvocatio
       context.getStore(NAMESPACE).put(ORIGINAL_TM_KEY, tm());
       DatabaseType databaseType =
           (DatabaseType) context.getStore(NAMESPACE).get(INJECTED_TM_SUPPLIER_KEY);
-      TransactionManagerFactory.setTmForTest(databaseType.getTm());
+      TransactionManagerFactory.setTmOverrideForTest(databaseType.getTm());
     }
   }
 

core/src/test/java/google/registry/testing/ReplayExtension.java

@@ -38,7 +38,6 @@ import google.registry.persistence.transaction.Transaction.Delete;
 import google.registry.persistence.transaction.Transaction.Mutation;
 import google.registry.persistence.transaction.Transaction.Update;
 import google.registry.persistence.transaction.TransactionEntity;
-import google.registry.util.RequestStatusChecker;
 import java.io.IOException;
 import java.util.List;
 import java.util.Optional;
@@ -46,7 +45,6 @@ import javax.annotation.Nullable;
 import org.junit.jupiter.api.extension.AfterEachCallback;
 import org.junit.jupiter.api.extension.BeforeEachCallback;
 import org.junit.jupiter.api.extension.ExtensionContext;
-import org.mockito.Mockito;
 
 /**
  * A JUnit extension that replays datastore transactions against postgresql.
@@ -83,11 +81,13 @@ public class ReplayExtension implements BeforeEachCallback, AfterEachCallback {
    * Create a replay extension that replays from SQL to cloud datastore when running in SQL mode.
    */
   public static ReplayExtension createWithDoubleReplay(FakeClock clock) {
-    return new ReplayExtension(
-        clock,
-        true,
-        new ReplicateToDatastoreAction(
-            clock, Mockito.mock(RequestStatusChecker.class), new FakeResponse()));
+    // TODO: use the proper double-replay extension when the tests are not flaky
+    // return new ReplayExtension(
+    //     clock,
+    //     true,
+    //     new ReplicateToDatastoreAction(
+    //         clock, Mockito.mock(RequestStatusChecker.class), new FakeResponse()));
+    return createWithCompare(clock);
   }
 
   @Override

core/src/test/java/google/registry/testing/TestLogHandlerUtils.java

@@ -43,6 +43,15 @@ public final class TestLogHandlerUtils {
         handler.getStoredLogRecords(), logRecord -> logRecord.getMessage().startsWith(prefix));
   }
 
+  /** Assert that the specified log message is <em>not</em> found. */
+  public static void assertNoLogMessage(CapturingLogHandler handler, Level level, String message) {
+    for (LogRecord logRecord : handler.getRecords()) {
+      if (logRecord.getLevel().equals(level) && logRecord.getMessage().contains(message)) {
+        assertWithMessage("Log message \"%s\" found: %s", message, logRecord.getMessage()).fail();
+      }
+    }
+  }
+
   public static void assertLogMessage(CapturingLogHandler handler, Level level, String message) {
     for (LogRecord logRecord : handler.getRecords()) {
       if (logRecord.getLevel().equals(level) && logRecord.getMessage().contains(message)) {

core/src/test/java/google/registry/testing/TmOverrideExtension.java

@@ -0,0 +1,73 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.testing;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
+
+import google.registry.persistence.transaction.TransactionManager;
+import google.registry.persistence.transaction.TransactionManagerFactory;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+/**
+ * JUnit extension for overriding the {@link TransactionManager} in tests.
+ *
+ * <p>You will typically want to run this at <code>@Order(Order.DEFAULT + 1)</code> alongside a
+ * {@link google.registry.persistence.transaction.JpaTransactionManagerExtension} or {@link
+ * DatastoreEntityExtension} with default {@link org.junit.jupiter.api.Order}. The transaction
+ * manager extension needs to run first so that when this override is called it's not trying to use
+ * the default dummy one.
+ *
+ * <p>This extension is incompatible with {@link DualDatabaseTest}. Use either that or this, but not
+ * both.
+ */
+public final class TmOverrideExtension implements BeforeEachCallback, AfterEachCallback {
+
+  private static enum TmOverride {
+    OFY,
+    JPA;
+  }
+
+  private final TmOverride tmOverride;
+
+  private TmOverrideExtension(TmOverride tmOverride) {
+    this.tmOverride = tmOverride;
+  }
+
+  /** Use the {@link google.registry.model.ofy.DatastoreTransactionManager} for all tests. */
+  public static TmOverrideExtension withOfy() {
+    return new TmOverrideExtension(TmOverride.OFY);
+  }
+
+  /**
+   * Use the {@link google.registry.persistence.transaction.JpaTransactionManager} for all tests.
+   */
+  public static TmOverrideExtension withJpa() {
+    return new TmOverrideExtension(TmOverride.JPA);
+  }
+
+  @Override
+  public void beforeEach(ExtensionContext context) {
+    TransactionManagerFactory.setTmOverrideForTest(
+        tmOverride == TmOverride.OFY ? ofyTm() : jpaTm());
+  }
+
+  @Override
+  public void afterEach(ExtensionContext context) {
+    TransactionManagerFactory.removeTmOverrideForTest();
+  }
+}

core/src/test/java/google/registry/tools/CommandTestCase.java

@@ -42,7 +42,6 @@ import java.io.OutputStream;
 import java.io.PrintStream;
 import java.nio.file.Path;
 import java.util.List;
-import java.util.concurrent.locks.ReentrantLock;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -59,10 +58,6 @@ import org.mockito.junit.jupiter.MockitoExtension;
 @ExtendWith(MockitoExtension.class)
 public abstract class CommandTestCase<C extends Command> {
 
-  // Lock for stdout/stderr.  Note that this is static: since we're dealing with globals, we need
-  // to lock for the entire JVM.
-  private static final ReentrantLock streamsLock = new ReentrantLock();
-
   private final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
   private final ByteArrayOutputStream stderr = new ByteArrayOutputStream();
   private PrintStream oldStdout, oldStderr;
@@ -90,10 +85,7 @@ public abstract class CommandTestCase<C extends Command> {
     RegistryToolEnvironment.UNITTEST.setup(systemPropertyExtension);
     command = newCommandInstance();
 
-    // Capture standard output/error. This is problematic because gradle tests run in parallel in
-    // the same JVM.  So first lock out any other tests in this JVM that are trying to do this
-    // trick.
-    streamsLock.lock();
+    // Capture standard output/error.
     oldStdout = System.out;
     System.setOut(new PrintStream(new OutputSplitter(System.out, stdout)));
     oldStderr = System.err;
@@ -104,7 +96,6 @@ public abstract class CommandTestCase<C extends Command> {
   public final void afterEachCommandTestCase() {
     System.setOut(oldStdout);
     System.setErr(oldStderr);
-    streamsLock.unlock();
   }
 
   void runCommandInEnvironment(RegistryToolEnvironment env, String... args) throws Exception {

core/src/test/java/google/registry/tools/RenewDomainCommandTest.java

@@ -132,6 +132,55 @@ public class RenewDomainCommandTest extends EppToolCommandTestCase<RenewDomainCo
         .verifyNoMoreSent();
   }
 
+  @Test
+  void testSuccess_withReasonAndRegistrarRequest() throws Exception {
+    persistActiveDomain(
+        "domain.tld",
+        DateTime.parse("2014-09-05T05:05:05Z"),
+        DateTime.parse("2015-09-05T05:05:05Z"));
+    runCommandForced(
+        "domain.tld", "--period=1", "--reason=Renewing test domain", "--registrar_request=true");
+
+    eppVerifier
+        .expectRegistrarId("TheRegistrar")
+        .verifySent(
+            "domain_renew_via_urs.xml",
+            ImmutableMap.of(
+                "DOMAIN",
+                "domain.tld",
+                "EXPDATE",
+                "2015-09-05",
+                "YEARS",
+                "1",
+                "REASON",
+                "Renewing test domain",
+                "REQUESTED",
+                "true"));
+  }
+
+  @Test
+  void testSuccess_withReqistrarRequestOnly() throws Exception {
+    persistActiveDomain(
+        "domain.tld",
+        DateTime.parse("2014-09-05T05:05:05Z"),
+        DateTime.parse("2015-09-05T05:05:05Z"));
+    runCommandForced("domain.tld", "--period=1", "--registrar_request=true");
+
+    eppVerifier
+        .expectRegistrarId("TheRegistrar")
+        .verifySent(
+            "domain_renew_with_metadata_requestedByRegistrar_only.xml",
+            ImmutableMap.of(
+                "DOMAIN",
+                "domain.tld",
+                "EXPDATE",
+                "2015-09-05",
+                "YEARS",
+                "1",
+                "REQUESTED",
+                "true"));
+  }
+
   @Test
   void testFailure_domainDoesntExist() {
     IllegalArgumentException e =
@@ -173,4 +222,19 @@ public class RenewDomainCommandTest extends EppToolCommandTestCase<RenewDomainCo
   void testFailure_missingDomainNames() {
     assertThrows(ParameterException.class, () -> runCommand("--period 4"));
   }
+
+  @Test
+  void testFailure_registrarRequestIsRequiredWhenReasonIsPresent() {
+    persistActiveDomain(
+        "domain.tld",
+        DateTime.parse("2014-09-05T05:05:05Z"),
+        DateTime.parse("2015-09-05T05:05:05Z"));
+    IllegalArgumentException e =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> runCommandForced("domain.tld", "--period=1", "--reason=testing_only"));
+    assertThat(e)
+        .hasMessageThat()
+        .isEqualTo("--registrar_request is required when --reason is specified");
+  }
 }

core/src/test/java/google/registry/tools/UniformRapidSuspensionCommandTest.java

@@ -242,13 +242,44 @@ class UniformRapidSuspensionCommandTest
   }
 
   @Test
-  void testRenewOneYear_renewFlowIsTriggered() throws Exception {
-    // this test case was written based on an existing test case,
-    // testUndo_removesLocksReplacesHostsAndDsData() but two things were modified to test the
-    // renew workflow, which were:
-    // 1) a different domain that contains creation time and expiration time
-    // 2) renew_one_year is set to true
+  void testRenewOneYearWithoutUndo_verifyReasonWithoutUndo() throws Exception {
+    persistDomainWithHosts(
+        newDomainBase("evil.tld")
+            .asBuilder()
+            .setCreationTimeForTest(DateTime.parse("2021-10-01T05:01:11Z"))
+            .setRegistrationExpirationTime(DateTime.parse("2022-10-01T05:01:11Z"))
+            .setPersistedCurrentSponsorRegistrarId("CharlestonRoad")
+            .build(),
+        defaultDsData,
+        urs1,
+        urs2);
 
+    runCommandForced(
+        "--domain_name=evil.tld",
+        "--hosts=ns1.example.com,ns2.example.com",
+        "--renew_one_year=true");
+
+    eppVerifier
+        .expectRegistrarId("CharlestonRoad")
+        .expectSuperuser()
+        .verifySent(
+            "domain_renew_via_urs.xml",
+            ImmutableMap.of(
+                "DOMAIN",
+                "evil.tld",
+                "EXPDATE",
+                "2022-10-01",
+                "YEARS",
+                "1",
+                "REASON",
+                "Uniform Rapid Suspension",
+                "REQUESTED",
+                "false"))
+        .verifySentAny();
+  }
+
+  @Test
+  void testRenewOneYearWithUndo_verifyReasonWithUndo() throws Exception {
     persistDomainWithHosts(
         newDomainBase("evil.tld")
             .asBuilder()
@@ -266,21 +297,65 @@ class UniformRapidSuspensionCommandTest
         "--hosts=ns1.example.com,ns2.example.com",
         "--renew_one_year=true");
 
-    // verify if renew flow is triggered
     eppVerifier
         .expectRegistrarId("CharlestonRoad")
         .expectSuperuser()
         .verifySent(
-            "domain_renew.xml",
-            ImmutableMap.of("DOMAIN", "evil.tld", "EXPDATE", "2022-10-01", "YEARS", "1"));
+            "domain_renew_via_urs.xml",
+            ImmutableMap.of(
+                "DOMAIN",
+                "evil.tld",
+                "EXPDATE",
+                "2022-10-01",
+                "YEARS",
+                "1",
+                "REASON",
+                "Undo Uniform Rapid Suspension",
+                "REQUESTED",
+                "false"))
+        .verifySentAny();
+  }
+
+  @Test
+  void testRenewOneYear_verifyBothRenewAndUpdateFlowsAreTriggered() throws Exception {
+    persistDomainWithHosts(
+        newDomainBase("evil.tld")
+            .asBuilder()
+            .setCreationTimeForTest(DateTime.parse("2021-10-01T05:01:11Z"))
+            .setRegistrationExpirationTime(DateTime.parse("2022-10-01T05:01:11Z"))
+            .setPersistedCurrentSponsorRegistrarId("CharlestonRoad")
+            .build(),
+        defaultDsData,
+        urs1,
+        urs2);
+
+    runCommandForced(
+        "--domain_name=evil.tld",
+        "--undo",
+        "--hosts=ns1.example.com,ns2.example.com",
+        "--renew_one_year=true");
 
-    // verify if update flow is triggered
     eppVerifier
         .expectRegistrarId("CharlestonRoad")
         .expectSuperuser()
-        .verifySent("uniform_rapid_suspension_undo.xml")
-        .verifyNoMoreSent();
-    assertNotInStdout("--undo"); // Undo shouldn't print a new undo command.
+        .verifySent(
+            "domain_renew_via_urs.xml",
+            ImmutableMap.of(
+                "DOMAIN",
+                "evil.tld",
+                "EXPDATE",
+                "2022-10-01",
+                "YEARS",
+                "1",
+                "REASON",
+                "Undo Uniform Rapid Suspension",
+                "REQUESTED",
+                "false"));
+
+    eppVerifier
+        .expectRegistrarId("CharlestonRoad")
+        .expectSuperuser()
+        .verifySent("uniform_rapid_suspension_undo.xml");
 
     // verify that no other flows are triggered after the renew and update flows
     eppVerifier.verifyNoMoreSent();

core/src/test/java/google/registry/tools/UpdateDomainCommandTest.java

@@ -25,12 +25,15 @@ import static google.registry.testing.DatabaseHelper.newDomainBase;
 import static google.registry.testing.DatabaseHelper.persistActiveDomain;
 import static google.registry.testing.DatabaseHelper.persistActiveHost;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.TestLogHandlerUtils.assertLogMessage;
+import static google.registry.testing.TestLogHandlerUtils.assertNoLogMessage;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.beust.jcommander.ParameterException;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.flogger.LoggerConfig;
 import google.registry.model.billing.BillingEvent;
 import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
@@ -46,6 +49,9 @@ import google.registry.persistence.VKey;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.InjectExtension;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.util.CapturingLogHandler;
+import java.util.logging.Level;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -53,6 +59,8 @@ import org.junit.jupiter.api.extension.RegisterExtension;
 @DualDatabaseTest
 class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand> {
 
+  private final CapturingLogHandler logHandler = new CapturingLogHandler();
+
   private DomainBase domain;
 
   @RegisterExtension public final InjectExtension inject = new InjectExtension();
@@ -62,6 +70,12 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
     inject.setStaticField(Ofy.class, "clock", fakeClock);
     command.clock = fakeClock;
     domain = persistActiveDomain("example.tld");
+    LoggerConfig.getConfig(UpdateDomainCommand.class).addHandler(logHandler);
+  }
+
+  @AfterEach
+  void afterEach() {
+    LoggerConfig.getConfig(UpdateDomainCommand.class).removeHandler(logHandler);
   }
 
   @TestOfyAndSql
@@ -299,7 +313,7 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
     runCommandForced("--client=NewRegistrar", "--autorenews=false", "example.tld");
     eppVerifier.verifySent(
         "domain_update_set_autorenew.xml", ImmutableMap.of("AUTORENEWS", "false"));
-    assertThat(getStderrAsString()).doesNotContain("autorenew grace period");
+    assertNoLogMessage(logHandler, Level.WARNING, "autorenew grace period");
   }
 
   @TestOfyAndSql
@@ -340,9 +354,9 @@ class UpdateDomainCommandTest extends EppToolCommandTestCase<UpdateDomainCommand
     runCommandForced("--client=NewRegistrar", "--autorenews=false", "example.tld");
     eppVerifier.verifySent(
         "domain_update_set_autorenew.xml", ImmutableMap.of("AUTORENEWS", "false"));
-    String stdErr = getStderrAsString();
-    assertThat(stdErr).contains("The following domains are in autorenew grace periods.");
-    assertThat(stdErr).contains("example.tld");
+    assertLogMessage(
+        logHandler, Level.WARNING, "The following domains are in autorenew grace periods.");
+    assertLogMessage(logHandler, Level.WARNING, "example.tld");
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/tools/javascrap/CreateSyntheticHistoryEntriesPipelineTest.java

@@ -0,0 +1,115 @@
+// Copyright 2021 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools.javascrap;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableObjects;
+import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
+import static google.registry.persistence.transaction.TransactionManagerFactory.removeTmOverrideForTest;
+import static google.registry.persistence.transaction.TransactionManagerFactory.setTmOverrideForTest;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistActiveHost;
+import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
+import static google.registry.testing.DatabaseHelper.persistSimpleResource;
+
+import com.google.common.collect.ImmutableList;
+import google.registry.beam.TestPipelineExtension;
+import google.registry.model.EppResource;
+import google.registry.model.contact.ContactHistory;
+import google.registry.model.contact.ContactResource;
+import google.registry.model.domain.DomainBase;
+import google.registry.model.domain.DomainHistory;
+import google.registry.model.host.HostHistory;
+import google.registry.model.host.HostResource;
+import google.registry.model.reporting.HistoryEntry;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
+import google.registry.testing.DatastoreEntityExtension;
+import google.registry.testing.FakeClock;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+/** Unit tests for {@link CreateSyntheticHistoryEntriesPipeline}. */
+public class CreateSyntheticHistoryEntriesPipelineTest {
+
+  FakeClock clock = new FakeClock();
+
+  @RegisterExtension
+  JpaIntegrationTestExtension jpaEextension =
+      new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension();
+
+  @RegisterExtension
+  DatastoreEntityExtension datastoreEntityExtension =
+      new DatastoreEntityExtension().allThreads(true);
+
+  @RegisterExtension TestPipelineExtension pipeline = TestPipelineExtension.create();
+
+  DomainBase domain;
+  ContactResource contact;
+  HostResource host;
+
+  @BeforeEach
+  void beforeEach() {
+    setTmOverrideForTest(jpaTm());
+    persistNewRegistrar("TheRegistrar");
+    persistNewRegistrar("NewRegistrar");
+    createTld("tld");
+    host = persistActiveHost("external.com");
+    domain =
+        persistSimpleResource(
+            newDomainBase("example.tld").asBuilder().setNameservers(host.createVKey()).build());
+    contact = jpaTm().transact(() -> jpaTm().loadByKey(domain.getRegistrant()));
+    clock.advanceOneMilli();
+  }
+
+  @AfterEach
+  void afterEach() {
+    removeTmOverrideForTest();
+  }
+
+  @Test
+  void testSuccess() {
+    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(DomainHistory.class))).isEmpty();
+    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(ContactHistory.class))).isEmpty();
+    assertThat(jpaTm().transact(() -> jpaTm().loadAllOf(HostHistory.class))).isEmpty();
+    CreateSyntheticHistoryEntriesPipeline.setup(pipeline, "NewRegistrar");
+    pipeline.run().waitUntilFinish();
+    validateHistoryEntry(DomainHistory.class, domain);
+    validateHistoryEntry(ContactHistory.class, contact);
+    validateHistoryEntry(HostHistory.class, host);
+  }
+
+  private static <T extends EppResource> void validateHistoryEntry(
+      Class<? extends HistoryEntry> historyClazz, T resource) {
+    ImmutableList<? extends HistoryEntry> historyEntries =
+        jpaTm().transact(() -> jpaTm().loadAllOf(historyClazz));
+    assertThat(historyEntries.size()).isEqualTo(1);
+    HistoryEntry historyEntry = historyEntries.get(0);
+    assertThat(historyEntry.getType()).isEqualTo(HistoryEntry.Type.SYNTHETIC);
+    assertThat(historyEntry.getRegistrarId()).isEqualTo("NewRegistrar");
+    EppResource embeddedResource;
+    if (historyEntry instanceof DomainHistory) {
+      embeddedResource = ((DomainHistory) historyEntry).getDomainContent().get();
+    } else if (historyEntry instanceof ContactHistory) {
+      embeddedResource = ((ContactHistory) historyEntry).getContactBase().get();
+    } else {
+      embeddedResource = ((HostHistory) historyEntry).getHostBase().get();
+    }
+    assertAboutImmutableObjects().that(embeddedResource).hasFieldsEqualTo(resource);
+  }
+}

core/src/test/resources/google/registry/flows/domain/domain_renew_metadata_with_reason_and_requestedByRegistrar.xml

@@ -0,0 +1,19 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <renew>
+      <domain:renew
+          xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>%DOMAIN%</domain:name>
+        <domain:curExpDate>%EXPDATE%</domain:curExpDate>
+        <domain:period unit="y">%YEARS%</domain:period>
+      </domain:renew>
+    </renew>
+    <extension>
+      <metadata:metadata xmlns:metadata="urn:google:params:xml:ns:metadata-1.0">
+        <metadata:reason>%REASON%</metadata:reason>
+        <metadata:requestedByRegistrar>%REQUESTED%</metadata:requestedByRegistrar>
+      </metadata:metadata>
+    </extension>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

core/src/test/resources/google/registry/flows/domain/domain_renew_metadata_with_requestedByRegistrar_only.xml

@@ -0,0 +1,18 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <renew>
+      <domain:renew
+          xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>example.tld</domain:name>
+        <domain:curExpDate>2000-04-03</domain:curExpDate>
+        <domain:period unit="y">1</domain:period>
+      </domain:renew>
+    </renew>
+    <extension>
+      <metadata:metadata xmlns:metadata="urn:google:params:xml:ns:metadata-1.0">
+        <metadata:requestedByRegistrar>true</metadata:requestedByRegistrar>
+      </metadata:metadata>
+    </extension>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

core/src/test/resources/google/registry/tools/server/domain_renew_via_urs.xml

@@ -0,0 +1,19 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <renew>
+      <domain:renew
+          xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>%DOMAIN%</domain:name>
+        <domain:curExpDate>%EXPDATE%</domain:curExpDate>
+        <domain:period unit="y">%YEARS%</domain:period>
+      </domain:renew>
+    </renew>
+    <extension>
+      <metadata:metadata xmlns:metadata="urn:google:params:xml:ns:metadata-1.0">
+        <metadata:reason>%REASON%</metadata:reason>
+        <metadata:requestedByRegistrar>%REQUESTED%</metadata:requestedByRegistrar>
+      </metadata:metadata>
+    </extension>
+    <clTRID>RegistryTool</clTRID>
+  </command>
+</epp>

core/src/test/resources/google/registry/tools/server/domain_renew_with_metadata_requestedByRegistrar_only.xml

@@ -0,0 +1,18 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <renew>
+      <domain:renew
+          xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>%DOMAIN%</domain:name>
+        <domain:curExpDate>%EXPDATE%</domain:curExpDate>
+        <domain:period unit="y">%YEARS%</domain:period>
+      </domain:renew>
+    </renew>
+    <extension>
+      <metadata:metadata xmlns:metadata="urn:google:params:xml:ns:metadata-1.0">
+        <metadata:requestedByRegistrar>%REQUESTED%</metadata:requestedByRegistrar>
+      </metadata:metadata>
+    </extension>
+    <clTRID>RegistryTool</clTRID>
+  </command>
+</epp>

util/build.gradle

@@ -43,12 +43,16 @@ dependencies {
   testCompile deps['com.google.appengine:appengine-api-stubs']
   testCompile deps['com.google.guava:guava-testlib']
   testCompile deps['com.google.truth:truth']
+  testCompile deps['junit:junit']
   testCompile deps['org.junit.jupiter:junit-jupiter-api']
   testCompile deps['org.junit.jupiter:junit-jupiter-engine']
+  testCompile deps['org.junit.platform:junit-platform-runner']
+  testCompile deps['org.junit.platform:junit-platform-suite-api']
   testCompile deps['org.hamcrest:hamcrest']
   testCompile deps['org.hamcrest:hamcrest-core']
   testCompile deps['org.mockito:mockito-core']
   testCompile deps['org.mockito:mockito-junit-jupiter']
+  testCompile deps['org.testcontainers:junit-jupiter']
   testCompile files("${rootDir}/third_party/objectify/v4_1/objectify-4.1.3.jar")
   testCompile project(path: ':common', configuration: 'testing')
   testRuntime deps['com.google.flogger:flogger-system-backend']
@@ -57,3 +61,7 @@ dependencies {
   testAnnotationProcessor deps['com.google.auto.value:auto-value']
   testAnnotationProcessor deps['com.google.dagger:dagger-compiler']
 }
+
+test {
+  useJUnitPlatform()
+}

util/gradle/dependency-locks/testCompile.lockfile

@@ -1,6 +1,10 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.fasterxml.jackson.core:jackson-annotations:2.10.3
+com.github.docker-java:docker-java-api:3.2.7
+com.github.docker-java:docker-java-transport-zerodep:3.2.7
+com.github.docker-java:docker-java-transport:3.2.7
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client:1.31.3
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
@@ -61,6 +65,8 @@ joda-time:joda-time:2.9.2
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
+net.java.dev.jna:jna:5.5.0
+org.apache.commons:commons-compress:1.20
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.apiguardian:apiguardian-api:1.1.0
@@ -76,11 +82,19 @@ org.junit.jupiter:junit-jupiter-api:5.7.0
 org.junit.jupiter:junit-jupiter-engine:5.7.0
 org.junit.platform:junit-platform-commons:1.7.0
 org.junit.platform:junit-platform-engine:1.7.0
+org.junit.platform:junit-platform-launcher:1.7.0
+org.junit.platform:junit-platform-runner:1.7.0
+org.junit.platform:junit-platform-suite-api:1.7.0
 org.junit:junit-bom:5.7.0
 org.mockito:mockito-core:3.7.7
 org.mockito:mockito-junit-jupiter:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm:9.0
+org.rnorth.duct-tape:duct-tape:1.0.8
+org.rnorth.visible-assertions:visible-assertions:2.1.2
+org.slf4j:slf4j-api:1.7.30
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.yaml:snakeyaml:1.17

util/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -1,6 +1,10 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.fasterxml.jackson.core:jackson-annotations:2.10.3
+com.github.docker-java:docker-java-api:3.2.7
+com.github.docker-java:docker-java-transport-zerodep:3.2.7
+com.github.docker-java:docker-java-transport:3.2.7
 com.google.api-client:google-api-client:1.31.3
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
 com.google.api.grpc:proto-google-cloud-tasks-v2beta2:0.89.2
@@ -58,6 +62,8 @@ joda-time:joda-time:2.9.2
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
+net.java.dev.jna:jna:5.5.0
+org.apache.commons:commons-compress:1.20
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.apiguardian:apiguardian-api:1.1.0
@@ -68,15 +74,23 @@ org.checkerframework:checker-qual:3.9.1
 org.conscrypt:conscrypt-openjdk-uber:2.5.1
 org.hamcrest:hamcrest-core:2.2
 org.hamcrest:hamcrest:2.2
-org.junit.jupiter:junit-jupiter-api:5.6.2
-org.junit.jupiter:junit-jupiter-engine:5.6.2
-org.junit.platform:junit-platform-commons:1.6.2
-org.junit.platform:junit-platform-engine:1.6.2
-org.junit:junit-bom:5.6.2
+org.junit.jupiter:junit-jupiter-api:5.7.0
+org.junit.jupiter:junit-jupiter-engine:5.7.0
+org.junit.platform:junit-platform-commons:1.7.0
+org.junit.platform:junit-platform-engine:1.7.0
+org.junit.platform:junit-platform-launcher:1.7.0
+org.junit.platform:junit-platform-runner:1.7.0
+org.junit.platform:junit-platform-suite-api:1.7.0
+org.junit:junit-bom:5.7.0
 org.mockito:mockito-core:3.7.7
 org.mockito:mockito-junit-jupiter:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm:9.0
+org.rnorth.duct-tape:duct-tape:1.0.8
+org.rnorth.visible-assertions:visible-assertions:2.1.2
+org.slf4j:slf4j-api:1.7.30
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.yaml:snakeyaml:1.17

util/gradle/dependency-locks/testRuntime.lockfile

@@ -1,6 +1,10 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.fasterxml.jackson.core:jackson-annotations:2.10.3
+com.github.docker-java:docker-java-api:3.2.7
+com.github.docker-java:docker-java-transport-zerodep:3.2.7
+com.github.docker-java:docker-java-transport:3.2.7
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client:1.31.3
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
@@ -63,6 +67,8 @@ joda-time:joda-time:2.9.2
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
+net.java.dev.jna:jna:5.5.0
+org.apache.commons:commons-compress:1.20
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.apiguardian:apiguardian-api:1.1.0
@@ -78,11 +84,19 @@ org.junit.jupiter:junit-jupiter-api:5.7.0
 org.junit.jupiter:junit-jupiter-engine:5.7.0
 org.junit.platform:junit-platform-commons:1.7.0
 org.junit.platform:junit-platform-engine:1.7.0
+org.junit.platform:junit-platform-launcher:1.7.0
+org.junit.platform:junit-platform-runner:1.7.0
+org.junit.platform:junit-platform-suite-api:1.7.0
 org.junit:junit-bom:5.7.0
 org.mockito:mockito-core:3.7.7
 org.mockito:mockito-junit-jupiter:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm:9.0
+org.rnorth.duct-tape:duct-tape:1.0.8
+org.rnorth.visible-assertions:visible-assertions:2.1.2
+org.slf4j:slf4j-api:1.7.30
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.yaml:snakeyaml:1.17

util/gradle/dependency-locks/testRuntimeClasspath.lockfile

@@ -1,6 +1,10 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
+com.fasterxml.jackson.core:jackson-annotations:2.10.3
+com.github.docker-java:docker-java-api:3.2.7
+com.github.docker-java:docker-java-transport-zerodep:3.2.7
+com.github.docker-java:docker-java-transport:3.2.7
 com.google.android:annotations:4.1.1.4
 com.google.api-client:google-api-client:1.31.3
 com.google.api.grpc:proto-google-cloud-tasks-v2:1.33.2
@@ -63,6 +67,8 @@ joda-time:joda-time:2.9.2
 junit:junit:4.13.2
 net.bytebuddy:byte-buddy-agent:1.10.19
 net.bytebuddy:byte-buddy:1.10.19
+net.java.dev.jna:jna:5.5.0
+org.apache.commons:commons-compress:1.20
 org.apache.httpcomponents:httpclient:4.5.13
 org.apache.httpcomponents:httpcore:4.4.14
 org.apiguardian:apiguardian-api:1.1.0
@@ -78,11 +84,19 @@ org.junit.jupiter:junit-jupiter-api:5.7.0
 org.junit.jupiter:junit-jupiter-engine:5.7.0
 org.junit.platform:junit-platform-commons:1.7.0
 org.junit.platform:junit-platform-engine:1.7.0
+org.junit.platform:junit-platform-launcher:1.7.0
+org.junit.platform:junit-platform-runner:1.7.0
+org.junit.platform:junit-platform-suite-api:1.7.0
 org.junit:junit-bom:5.7.0
 org.mockito:mockito-core:3.7.7
 org.mockito:mockito-junit-jupiter:3.7.7
 org.objenesis:objenesis:3.1
 org.opentest4j:opentest4j:1.2.0
 org.ow2.asm:asm:9.0
+org.rnorth.duct-tape:duct-tape:1.0.8
+org.rnorth.visible-assertions:visible-assertions:2.1.2
+org.slf4j:slf4j-api:1.7.30
+org.testcontainers:junit-jupiter:1.15.2
+org.testcontainers:testcontainers:1.15.2
 org.threeten:threetenbp:1.5.1
 org.yaml:snakeyaml:1.17

util/src/test/java/google/registry/util/DomainNameUtilsTest.java

@@ -25,7 +25,7 @@ import org.junit.jupiter.api.Test;
 class DomainNameUtilsTest {
 
   @Test
-  void testCanonicalizeDomainName() {
+  void testCanonicalizeDomainName_succeeds() {
     assertThat(canonicalizeDomainName("foo")).isEqualTo("foo");
     assertThat(canonicalizeDomainName("FOO")).isEqualTo("foo");
     assertThat(canonicalizeDomainName("foo.tld")).isEqualTo("foo.tld");
@@ -38,6 +38,21 @@ class DomainNameUtilsTest {
     assertThat(canonicalizeDomainName("ħ")).isEqualTo("xn--1ea");
   }
 
+  @Test
+  void testCanonicalizeDomainName_allowsRdnsNames() {
+    assertThat(canonicalizeDomainName("119.63.227.45-ns1.jhz-tt.uk"))
+        .isEqualTo("119.63.227.45-ns1.jhz-tt.uk");
+  }
+
+  @Test
+  void testCanonicalizeDomainName_throwsOn34HyphenRule() {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> canonicalizeDomainName("119.63.227.45--ns1.jhz-tt.uk"));
+    assertThat(thrown).hasCauseThat().hasMessageThat().contains("HYPHEN_3_4");
+  }
+
   @Test
   void testCanonicalizeDomainName_acePrefixUnicodeChars() {
     assertThrows(IllegalArgumentException.class, () -> canonicalizeDomainName("xn--みんな"));