Remove Optional.isEmpty() in code (#1585)

* Remove Optional.isEmpty() in code

core/src/main/java/google/registry/beam/common/RegistryJpaIO.java

@@ -23,6 +23,7 @@ import com.google.common.collect.Streams;
 import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
 import google.registry.model.UpdateAutoTimestamp;
 import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
+import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.persistence.transaction.TransactionManagerFactory;
@@ -234,6 +235,10 @@ public final class RegistryJpaIO {
 
       @ProcessElement
       public void processElement(OutputReceiver<T> outputReceiver) {
+        // Preload the migration schedule into cache, otherwise the cache loading query may happen
+        // before the setDatabaseSnapshot call in the transaction below, causing it to fail.
+        DatabaseMigrationStateSchedule.get();
+
         jpaTm()
             .transactNoRetry(
                 () -> {

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -22,7 +22,7 @@ import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static java.util.Comparator.comparing;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -343,11 +343,11 @@ public final class Transforms {
       // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this.
       entity.setIndexedProperty(
           "fullyQualifiedDomainName",
-          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedDomainName")));
+          canonicalizeHostname((String) entity.getProperty("fullyQualifiedDomainName")));
     } else if (entity.getKind().equals("HostResource")) {
       entity.setIndexedProperty(
           "fullyQualifiedHostName",
-          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedHostName")));
+          canonicalizeHostname((String) entity.getProperty("fullyQualifiedHostName")));
     }
     return entity;
   }

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -14,6 +14,7 @@
 
 package google.registry.beam.invoicing;
 
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -53,6 +54,7 @@ import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.joda.money.CurrencyUnit;
 
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
@@ -122,12 +124,19 @@ public class InvoicingPipeline implements Serializable {
     google.registry.model.billing.BillingEvent.OneTime oneTime =
         (google.registry.model.billing.BillingEvent.OneTime) row[0];
     Registrar registrar = (Registrar) row[1];
+    CurrencyUnit currency = oneTime.getCost().getCurrencyUnit();
+    checkState(
+        registrar.getBillingAccountMap().containsKey(currency),
+        "Registrar %s does not have a product account key for the currency unit: %s",
+        registrar.getRegistrarId(),
+        currency);
+
     return BillingEvent.create(
         oneTime.getId(),
         DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
         DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
         registrar.getRegistrarId(),
-        registrar.getBillingIdentifier().toString(),
+        registrar.getBillingAccountMap().get(currency),
         registrar.getPoNumber().orElse(""),
         DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
         oneTime.getReason().toString(),

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -576,6 +576,19 @@ public final class RegistryConfig {
       return config.beam.highPerformanceMachineType;
     }
 
+    /**
+     * Returns initial number of workers used for a Beam pipeline. Autoscaling can still in effect.
+     *
+     * @see <a
+     *     href=https://cloud.google.com/dataflow/docs/guides/deploying-a-pipeline#horizontal-autoscaling>
+     *     Horizontal Autoscaling </a>
+     */
+    @Provides
+    @Config("initialWorkerCount")
+    public static int provideInitialWorkerCount(RegistryConfigSettings config) {
+      return config.beam.initialWorkerCount;
+    }
+
     /**
      * Returns the default job region to run Apache Beam (Cloud Dataflow) jobs in.
      *

core/src/main/java/google/registry/config/RegistryConfigSettings.java

@@ -137,6 +137,7 @@ public class RegistryConfigSettings {
   public static class Beam {
     public String defaultJobRegion;
     public String highPerformanceMachineType;
+    public int initialWorkerCount;
     public String stagingBucketUrl;
   }
 

core/src/main/java/google/registry/config/files/default-config.yaml

@@ -442,6 +442,11 @@ beam:
   # core count per machine may be preferable in order to preserve IP addresses.
   # See: https://cloud.google.com/compute/quotas#cpu_quota
   highPerformanceMachineType: n2-standard-4
+  # The initial number of workers requested. This can help speed up the pipeline
+  # which otherwise would take some time to spin up the necessary number of
+  # works. Autoscaling is still in effect to reduce the number of workers if
+  # not in use.
+  initialWorkerCount: 24
   stagingBucketUrl: gcs-bucket-with-staged-templates
 
 keyring:

core/src/main/java/google/registry/flows/CheckApiAction.java

@@ -31,7 +31,7 @@ import static google.registry.monitoring.whitebox.CheckApiMetric.Status.UNKNOWN_
 import static google.registry.monitoring.whitebox.CheckApiMetric.Tier.PREMIUM;
 import static google.registry.monitoring.whitebox.CheckApiMetric.Tier.STANDARD;
 import static google.registry.pricing.PricingEngineProxy.isDomainPremium;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static org.json.simple.JSONValue.toJSONString;
 
 import com.google.common.collect.ImmutableList;
@@ -105,7 +105,7 @@ public class CheckApiAction implements Runnable {
     String domainString;
     InternetDomainName domainName;
     try {
-      domainString = canonicalizeDomainName(nullToEmpty(domain));
+      domainString = canonicalizeHostname(nullToEmpty(domain));
       domainName = validateDomainName(domainString);
     } catch (IllegalArgumentException | EppException e) {
       metricBuilder.status(INVALID_NAME);

core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java

@@ -25,6 +25,7 @@ import static google.registry.model.ResourceTransferUtils.handlePendingTransferO
 import static google.registry.model.eppoutput.Result.Code.SUCCESS;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
 import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED;
+import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
@@ -94,6 +95,7 @@ public final class ContactDeleteFlow implements TransactionalFlow {
     extensionManager.register(MetadataExtension.class);
     validateRegistrarIsLoggedIn(registrarId);
     extensionManager.validate();
+    assertAsyncActionsAreAllowed();
     DateTime now = tm().getTransactionTime();
     checkLinkedDomains(targetId, now, ContactResource.class, DomainBase::getReferencedContacts);
     ContactResource existingContact = loadAndVerifyExistence(ContactResource.class, targetId, now);

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -22,6 +22,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyResourceOwnership;
 import static google.registry.flows.host.HostFlowUtils.validateHostName;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
+import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
@@ -96,6 +97,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
     extensionManager.register(MetadataExtension.class);
     validateRegistrarIsLoggedIn(registrarId);
     extensionManager.validate();
+    assertAsyncActionsAreAllowed();
     DateTime now = tm().getTransactionTime();
     validateHostName(targetId);
     checkLinkedDomains(targetId, now, HostResource.class, DomainBase::getNameservers);

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -28,6 +28,7 @@ import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomain
 import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomainOwnership;
 import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
 import static google.registry.model.reporting.HistoryEntry.Type.HOST_UPDATE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
 
@@ -136,6 +137,9 @@ public final class HostUpdateFlow implements TransactionalFlow {
     validateHostName(targetId);
     HostResource existingHost = loadAndVerifyExistence(HostResource.class, targetId, now);
     boolean isHostRename = suppliedNewHostName != null;
+    if (isHostRename) {
+      assertAsyncActionsAreAllowed();
+    }
     String oldHostName = targetId;
     String newHostName = firstNonNull(suppliedNewHostName, oldHostName);
     DomainBase oldSuperordinateDomain =

core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java

@@ -30,6 +30,7 @@ import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.model.replay.SqlOnlyEntity;
 import java.time.Duration;
+import java.util.Arrays;
 import javax.persistence.Entity;
 import javax.persistence.PersistenceException;
 import org.joda.time.DateTime;
@@ -62,11 +63,28 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
    * not the phase is read-only.
    */
   public enum MigrationState {
+    /** Datastore is the only DB being used. */
     DATASTORE_ONLY(PrimaryDatabase.DATASTORE, false, ReplayDirection.NO_REPLAY),
+
+    /** Datastore is the primary DB, with changes replicated to Cloud SQL. */
     DATASTORE_PRIMARY(PrimaryDatabase.DATASTORE, false, ReplayDirection.DATASTORE_TO_SQL),
+
+    /** Datastore is the primary DB, with replication, and async actions are disallowed. */
+    DATASTORE_PRIMARY_NO_ASYNC(PrimaryDatabase.DATASTORE, false, ReplayDirection.DATASTORE_TO_SQL),
+
+    /** Datastore is the primary DB, with replication, and all mutating actions are disallowed. */
     DATASTORE_PRIMARY_READ_ONLY(PrimaryDatabase.DATASTORE, true, ReplayDirection.DATASTORE_TO_SQL),
+
+    /**
+     * Cloud SQL is the primary DB, with replication back to Datastore, and all mutating actions are
+     * disallowed.
+     */
     SQL_PRIMARY_READ_ONLY(PrimaryDatabase.CLOUD_SQL, true, ReplayDirection.SQL_TO_DATASTORE),
+
+    /** Cloud SQL is the primary DB, with changes replicated to Datastore. */
     SQL_PRIMARY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.SQL_TO_DATASTORE),
+
+    /** Cloud SQL is the only DB being used. */
     SQL_ONLY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.NO_REPLAY);
 
     private final PrimaryDatabase primaryDatabase;
@@ -146,11 +164,17 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
             .putAll(
                 MigrationState.DATASTORE_PRIMARY,
                 MigrationState.DATASTORE_ONLY,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
+            .putAll(
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
+                MigrationState.DATASTORE_ONLY,
+                MigrationState.DATASTORE_PRIMARY,
                 MigrationState.DATASTORE_PRIMARY_READ_ONLY)
             .putAll(
                 MigrationState.DATASTORE_PRIMARY_READ_ONLY,
                 MigrationState.DATASTORE_ONLY,
                 MigrationState.DATASTORE_PRIMARY,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                 MigrationState.SQL_PRIMARY_READ_ONLY,
                 MigrationState.SQL_PRIMARY)
             .putAll(
@@ -165,10 +189,9 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
                 MigrationState.SQL_ONLY,
                 MigrationState.SQL_PRIMARY_READ_ONLY,
                 MigrationState.SQL_PRIMARY);
+
     // In addition, we can always transition from a state to itself (useful when updating the map).
-    for (MigrationState migrationState : MigrationState.values()) {
-      builder.put(migrationState, migrationState);
-    }
+    Arrays.stream(MigrationState.values()).forEach(state -> builder.put(state, state));
     return builder.build();
   }
 
@@ -246,7 +269,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
    * A provided map of transitions may be valid by itself (i.e. it shifts states properly, doesn't
    * skip states, and doesn't backtrack incorrectly) while still being invalid. In addition to the
    * transitions in the map being valid, the single transition from the current map at the current
-   * time to the new map at the current time time must also be valid.
+   * time to the new map at the current time must also be valid.
    */
   private static void validateTransitionAtCurrentTime(
       TimedTransitionProperty<MigrationState, MigrationStateTransition> newTransitions) {

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -175,7 +175,7 @@ public class DomainBase extends DomainContent
 
   @Override
   public void beforeSqlSaveOnReplay() {
-    fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+    fullyQualifiedDomainName = DomainNameUtils.canonicalizeHostname(fullyQualifiedDomainName);
     dsData =
         dsData.stream()
             .filter(datum -> datum.getDigest() != null && datum.getDigest().length > 0)

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -34,7 +34,7 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.earliestOf;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.leapSafeAddYears;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -890,7 +890,7 @@ public class DomainContent extends EppResource
 
     public B setDomainName(String domainName) {
       checkArgument(
-          domainName.equals(canonicalizeDomainName(domainName)),
+          domainName.equals(canonicalizeHostname(domainName)),
           "Domain name %s not in puny-coded, lower-case form",
           domainName);
       getInstance().fullyQualifiedDomainName = domainName;

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -309,7 +309,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     if (domainContent == null) {
       domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
       domainContent.fullyQualifiedDomainName =
-          DomainNameUtils.canonicalizeDomainName(domainContent.fullyQualifiedDomainName);
+          DomainNameUtils.canonicalizeHostname(domainContent.fullyQualifiedDomainName);
       fillAuxiliaryFieldsFromDomain(this);
     }
   }

core/src/main/java/google/registry/model/host/HostBase.java

@@ -19,7 +19,7 @@ import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.annotation.IgnoreSave;
@@ -195,7 +195,7 @@ public class HostBase extends EppResource {
 
     public B setHostName(String hostName) {
       checkArgument(
-          hostName.equals(canonicalizeDomainName(hostName)),
+          hostName.equals(canonicalizeHostname(hostName)),
           "Host name %s not in puny-coded, lower-case form",
           hostName);
       getInstance().fullyQualifiedHostName = hostName;

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -143,7 +143,7 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial
     if (hostBase == null) {
       hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
       hostBase.fullyQualifiedHostName =
-          DomainNameUtils.canonicalizeDomainName(hostBase.fullyQualifiedHostName);
+          DomainNameUtils.canonicalizeHostname(hostBase.fullyQualifiedHostName);
     }
   }
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -73,7 +73,7 @@ public class HostResource extends HostBase
 
   @Override
   public void beforeSqlSaveOnReplay() {
-    fullyQualifiedHostName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedHostName);
+    fullyQualifiedHostName = DomainNameUtils.canonicalizeHostname(fullyQualifiedHostName);
   }
 
   @Override

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -69,7 +69,7 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
    * converted to a raw Datastore Entity, serialized to bytes, and stored within the mutation.
    */
   public static CommitLogMutation create(Key<CommitLogManifest> parent, Object entity) {
-    return createFromRaw(parent, auditedOfy().save().toEntity(entity));
+    return createFromRaw(parent, auditedOfy().saveIgnoringReadOnlyWithBackup().toEntity(entity));
   }
 
   /**

core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java

@@ -156,7 +156,7 @@ public class CommitLoggedWork<R> implements Runnable {
             .map(entity -> (ImmutableObject) CommitLogMutation.create(manifestKey, entity))
             .collect(toImmutableSet());
     auditedOfy()
-        .saveWithoutBackup()
+        .saveIgnoringReadOnlyWithoutBackup()
         .entities(
             new ImmutableSet.Builder<>()
                 .add(manifest)

core/src/main/java/google/registry/model/tld/label/DomainLabelEntry.java

@@ -16,7 +16,7 @@ package google.registry.model.tld.label;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Strings.emptyToNull;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.net.InternetDomainName;
@@ -77,7 +77,7 @@ public abstract class DomainLabelEntry<T extends Comparable<?>, D extends Domain
     public T build() {
       checkArgumentNotNull(emptyToNull(getInstance().domainLabel), "Label must be specified");
       checkArgument(
-          getInstance().domainLabel.equals(canonicalizeDomainName(getInstance().domainLabel)),
+          getInstance().domainLabel.equals(canonicalizeHostname(getInstance().domainLabel)),
           "Label '%s' must be in puny-coded, lower-case form",
           getInstance().domainLabel);
       checkArgumentNotNull(getInstance().getValue(), "Value must be specified");

core/src/main/java/google/registry/model/tmch/ClaimsList.java

@@ -70,8 +70,7 @@ public class ClaimsList extends ImmutableObject implements SqlOnlyEntity {
    *
    * <p>Note that the value of this field is parsed from the claims list file(See this <a
    * href="https://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.1">RFC</>), it is
-   * the DNL List creation datetime from the rfc. Since this field has been used by Datastore, we
-   * cannot change its name until we finish the migration.
+   * the DNL List creation datetime from the rfc.
    *
    * <p>TODO(b/177567432): Rename this field to tmdbGenerationTime.
    */

core/src/main/java/google/registry/model/tmch/TmchCrl.java

@@ -47,7 +47,7 @@ public final class TmchCrl extends CrossTldSingleton implements SqlOnlyEntity {
    * Change the singleton to a new ASCII-armored X.509 CRL.
    *
    * <p>Please do not call this function unless your CRL is properly formatted, signed by the root,
-   * and actually newer than the one currently in Datastore.
+   * and actually newer than the one currently in the database.
    */
   public static void set(final String crl, final String url) {
     jpaTm()

core/src/main/java/google/registry/module/backend/BackendComponent.java

@@ -44,6 +44,7 @@ import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.NetHttpTransportModule;
 import google.registry.request.Modules.UrlConnectionServiceModule;
+import google.registry.request.Modules.UrlFetchServiceModule;
 import google.registry.request.Modules.UrlFetchTransportModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.request.auth.AuthModule;
@@ -81,6 +82,7 @@ import javax.inject.Singleton;
       SheetsServiceModule.class,
       StackdriverModule.class,
       UrlConnectionServiceModule.class,
+      UrlFetchServiceModule.class,
       UrlFetchTransportModule.class,
       UserServiceModule.class,
       VoidDnsWriterModule.class,

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -15,23 +15,25 @@
 package google.registry.persistence.transaction;
 
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
-import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.appengine.api.utils.SystemProperty;
 import com.google.appengine.api.utils.SystemProperty.Environment.Value;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Suppliers;
 import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
 import google.registry.model.ofy.DatastoreTransactionManager;
 import google.registry.persistence.DaggerPersistenceComponent;
 import google.registry.tools.RegistryToolEnvironment;
+import google.registry.util.Clock;
 import google.registry.util.NonFinalForTesting;
+import google.registry.util.SystemClock;
 import java.util.Optional;
 import java.util.function.Supplier;
-import org.joda.time.DateTime;
 
 /** Factory class to create {@link TransactionManager} instance. */
 // TODO: Rename this to PersistenceFactory and move to persistence package.
@@ -42,6 +44,9 @@ public final class TransactionManagerFactory {
   /** Optional override to manually set the transaction manager per-test. */
   private static Optional<TransactionManager> tmForTest = Optional.empty();
 
+  /** The current clock (defined as a variable so we can override it in tests) */
+  private static Clock clock = new SystemClock();
+
   /** Supplier for jpaTm so that it is initialized only once, upon first usage. */
   @NonFinalForTesting
   private static Supplier<JpaTransactionManager> jpaTm =
@@ -103,7 +108,7 @@ public final class TransactionManagerFactory {
     if (onBeam) {
       return jpaTm();
     }
-    return DatabaseMigrationStateSchedule.getValueAtTime(DateTime.now(UTC))
+    return DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc())
             .getPrimaryDatabase()
             .equals(PrimaryDatabase.DATASTORE)
         ? ofyTm()
@@ -193,11 +198,33 @@ public final class TransactionManagerFactory {
   }
 
   public static void assertNotReadOnlyMode() {
-    if (DatabaseMigrationStateSchedule.getValueAtTime(DateTime.now(UTC)).isReadOnly()) {
+    if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()).isReadOnly()) {
       throw new ReadOnlyModeException();
     }
   }
 
+  /**
+   * Asserts that async actions (contact/host deletes and host renames) are allowed.
+   *
+   * <p>These are allowed at all times except during the {@link
+   * DatabaseMigrationStateSchedule.MigrationState#DATASTORE_PRIMARY_NO_ASYNC} stage. Note that
+   * {@link ReadOnlyModeException} may well be thrown during other read-only stages inside the
+   * transaction manager; this method specifically checks only async actions.
+   */
+  @DeleteAfterMigration
+  public static void assertAsyncActionsAreAllowed() {
+    if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc())
+        .equals(DATASTORE_PRIMARY_NO_ASYNC)) {
+      throw new ReadOnlyModeException();
+    }
+  }
+
+  /** Allows us to set the clock used by the factory in unit tests. */
+  @VisibleForTesting
+  public static void setClockForTesting(Clock clock) {
+    TransactionManagerFactory.clock = clock;
+  }
+
   /** Registry is currently undergoing maintenance and is in read-only mode. */
   public static class ReadOnlyModeException extends IllegalStateException {
     public ReadOnlyModeException() {

core/src/main/java/google/registry/rdap/RdapActionBase.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Charsets.UTF_8;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN;
 import static google.registry.request.Actions.getPathForAction;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -245,7 +245,7 @@ public abstract class RdapActionBase implements Runnable {
   }
 
   String canonicalizeName(String name) {
-    name = canonicalizeDomainName(name);
+    name = canonicalizeHostname(name);
     if (name.endsWith(".")) {
       name = name.substring(0, name.length() - 1);
     }

core/src/main/java/google/registry/rde/RdeReporter.java

@@ -14,24 +14,25 @@
 
 package google.registry.rde;
 
-import static google.registry.request.UrlConnectionUtils.getResponseBytes;
-import static google.registry.request.UrlConnectionUtils.setBasicAuth;
-import static google.registry.request.UrlConnectionUtils.setPayload;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate;
+import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
+import static com.google.common.io.BaseEncoding.base64;
+import static com.google.common.net.HttpHeaders.AUTHORIZATION;
+import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
 
-import com.google.api.client.http.HttpMethods;
+import com.google.appengine.api.urlfetch.HTTPHeader;
+import com.google.appengine.api.urlfetch.HTTPRequest;
 import com.google.appengine.api.urlfetch.HTTPResponse;
+import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.common.flogger.FluentLogger;
-import com.google.common.net.MediaType;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.keyring.api.KeyModule.Key;
 import google.registry.request.HttpException.InternalServerErrorException;
-import google.registry.request.UrlConnectionService;
 import google.registry.util.Retrier;
-import google.registry.util.UrlConnectionException;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.iirdea.XjcIirdeaResponseElement;
 import google.registry.xjc.iirdea.XjcIirdeaResult;
@@ -39,7 +40,6 @@ import google.registry.xjc.rdeheader.XjcRdeHeader;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
-import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.SocketTimeoutException;
 import java.net.URL;
@@ -58,10 +58,10 @@ public class RdeReporter {
    * @see <a href="http://tools.ietf.org/html/draft-lozano-icann-registry-interfaces-05#section-4">
    *     ICANN Registry Interfaces - Interface details</a>
    */
-  private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8;
+  private static final String REPORT_MIME = "text/xml";
 
   @Inject Retrier retrier;
-  @Inject UrlConnectionService urlConnectionService;
+  @Inject URLFetchService urlFetchService;
 
   @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix;
   @Inject @Key("icannReportingPassword") String password;
@@ -76,24 +76,29 @@ public class RdeReporter {
     // Send a PUT request to ICANN's HTTPS server.
     URL url = makeReportUrl(header.getTld(), report.getId());
     String username = header.getTld() + "_ry";
+    String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8));
+    final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d));
+    req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME));
+    req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token));
+    req.setPayload(reportBytes);
     logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8));
-    byte[] responseBytes =
+    HTTPResponse rsp =
         retrier.callWithRetry(
             () -> {
-              HttpURLConnection connection = urlConnectionService.createConnection(url);
-              connection.setRequestMethod(HttpMethods.PUT);
-              setBasicAuth(connection, username, password);
-              setPayload(connection, reportBytes, MEDIA_TYPE.toString());
-              int responseCode = connection.getResponseCode();
-              if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) {
-                return getResponseBytes(connection);
+              HTTPResponse rsp1 = urlFetchService.fetch(req);
+              switch (rsp1.getResponseCode()) {
+                case SC_OK:
+                case SC_BAD_REQUEST:
+                  break;
+                default:
+                  throw new RuntimeException("PUT failed");
               }
-              throw new UrlConnectionException("PUT failed", connection);
+              return rsp1;
             },
             SocketTimeoutException.class);
 
     // Ensure the XML response is valid.
-    XjcIirdeaResult result = parseResult(responseBytes);
+    XjcIirdeaResult result = parseResult(rsp.getContent());
     if (result.getCode().getValue() != 1000) {
       logger.atWarning().log(
           "PUT rejected: %d %s\n%s",
@@ -118,7 +123,7 @@ public class RdeReporter {
 
   private URL makeReportUrl(String tld, String id) {
     try {
-      return new URL(String.format("%s/%s/%s", reportUrlPrefix, canonicalizeDomainName(tld), id));
+      return new URL(String.format("%s/%s/%s", reportUrlPrefix, canonicalizeHostname(tld), id));
     } catch (MalformedURLException e) {
       throw new RuntimeException(e);
     }

core/src/main/java/google/registry/rde/RdeStagingAction.java

@@ -261,6 +261,10 @@ public final class RdeStagingAction implements Runnable {
   @Config("highPerformanceMachineType")
   String machineType;
 
+  @Inject
+  @Config("initialWorkerCount")
+  int numWorkers;
+
   @Inject @Config("transactionCooldown") Duration transactionCooldown;
   @Inject @Config("beamStagingBucketUrl") String stagingBucketUrl;
   @Inject @Config("rdeBucket") String rdeBucket;
@@ -341,6 +345,7 @@ public final class RdeStagingAction implements Runnable {
                                           .encode(stagingKeyBytes))
                                   .put("registryEnvironment", RegistryEnvironment.get().name())
                                   .put("workerMachineType", machineType)
+                                  .put("numWorkers", String.valueOf(numWorkers))
                                   .put(
                                       "jpaTransactionManagerType",
                                       JpaTransactionManagerType.READ_ONLY_REPLICA.toString())

core/src/main/java/google/registry/rde/RdeUploadAction.java

@@ -32,6 +32,7 @@ import static java.util.Arrays.asList;
 import com.google.cloud.storage.BlobId;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Ordering;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.io.ByteStreams;
 import com.jcraft.jsch.JSch;
@@ -122,7 +123,6 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
   @Inject @Key("rdeSigningKey") PGPKeyPair signingKey;
   @Inject @Key("rdeStagingDecryptionKey") PGPPrivateKey stagingDecryptionKey;
   @Inject RdeUploadAction() {}
-
   @Override
   public void run() {
     logger.atInfo().log("Attempting to acquire RDE upload lock for TLD '%s'.", tld);
@@ -140,6 +140,27 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
 
   @Override
   public void runWithLock(final DateTime watermark) throws Exception {
+    // If a prefix is not provided, but we are in SQL mode, try to determine the prefix. This should
+    // only happen when the RDE upload cron job runs to catch up any un-retried (i. e. expected)
+    // RDE failures.
+    if (!prefix.isPresent() && !tm().isOfy()) {
+      // The prefix is always in the format of: rde-2022-02-21t00-00-00z-2022-02-21t00-07-33z, where
+      // the first datetime is the watermark and the second one is the time when the RDE beam job
+      // launched. We search for the latest folder that starts with "rde-[watermark]".
+      String partialPrefix =
+          String.format("rde-%s", watermark.toString("yyyy-MM-dd't'HH-mm-ss'z'"));
+      String latestFilenameSuffix =
+          gcsUtils.listFolderObjects(bucket, partialPrefix).stream()
+              .max(Ordering.natural())
+              .orElse(null);
+      if (latestFilenameSuffix == null) {
+        throw new NoContentException(
+            String.format("RDE deposit for TLD %s on %s does not exist", tld, watermark));
+      }
+      int firstSlashPosition = latestFilenameSuffix.indexOf('/');
+      prefix =
+          Optional.of(partialPrefix + latestFilenameSuffix.substring(0, firstSlashPosition + 1));
+    }
     logger.atInfo().log("Verifying readiness to upload the RDE deposit.");
     Optional<Cursor> cursor =
         transactIfJpaTm(() -> tm().loadByKeyIfPresent(Cursor.createVKey(RDE_STAGING, tld)));
@@ -241,9 +262,9 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
                     .setSignatureOutput(sigOut, signingKey)
                     .setFileMetadata(nameWithoutPrefix, xmlLength, watermark)
                     .build()) {
-            long bytesCopied = ByteStreams.copy(ghostrydeDecoder, rydeEncoder);
+          long bytesCopied = ByteStreams.copy(ghostrydeDecoder, rydeEncoder);
           logger.atInfo().log("Uploaded %,d bytes to path '%s'.", bytesCopied, rydeFilename);
-          }
+        }
         String sigFilename = nameWithoutPrefix + ".sig";
         BlobId sigGcsFilename = BlobId.of(bucket, name + ".sig");
         byte[] signature = sigOut.toByteArray();

core/src/main/java/google/registry/request/Modules.java

@@ -23,6 +23,8 @@ import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.JsonFactory;
 import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.appengine.api.datastore.DatastoreService;
+import com.google.appengine.api.urlfetch.URLFetchService;
+import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
 import com.google.appengine.api.users.UserService;
 import com.google.appengine.api.users.UserServiceFactory;
 import dagger.Module;
@@ -53,6 +55,17 @@ public final class Modules {
     }
   }
 
+  /** Dagger module for {@link URLFetchService}. */
+  @Module
+  public static final class UrlFetchServiceModule {
+    private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService();
+
+    @Provides
+    static URLFetchService provideUrlFetchService() {
+      return fetchService;
+    }
+  }
+
   /** Dagger module for {@link UserService}. */
   @Module
   public static final class UserServiceModule {

core/src/main/java/google/registry/request/UrlConnectionService.java

@@ -17,9 +17,10 @@ package google.registry.request;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.security.GeneralSecurityException;
 
 /** Functional interface for opening a connection from a URL, injectable for testing. */
 public interface UrlConnectionService {
 
-  HttpURLConnection createConnection(URL url) throws IOException;
+  HttpURLConnection createConnection(URL url) throws IOException, GeneralSecurityException;
 }

core/src/main/java/google/registry/tmch/Marksdb.java

@@ -32,6 +32,7 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.security.GeneralSecurityException;
 import java.security.Security;
 import java.security.SignatureException;
 import java.util.Arrays;
@@ -110,7 +111,8 @@ public final class Marksdb {
     }
   }
 
-  byte[] fetch(URL url, Optional<String> loginAndPassword) throws IOException {
+  byte[] fetch(URL url, Optional<String> loginAndPassword)
+      throws IOException, GeneralSecurityException {
     HttpURLConnection connection = urlConnectionService.createConnection(url);
     loginAndPassword.ifPresent(auth -> setBasicAuth(connection, auth));
     try {
@@ -124,7 +126,7 @@ public final class Marksdb {
   }
 
   List<String> fetchSignedCsv(Optional<String> loginAndPassword, String csvPath, String sigPath)
-      throws IOException, SignatureException, PGPException {
+      throws IOException, GeneralSecurityException, PGPException {
     checkArgument(
         loginAndPassword.isPresent(), "Cannot fetch from MarksDB without login credentials");
 

core/src/main/java/google/registry/tmch/NordnUploadAction.java

@@ -54,6 +54,7 @@ import google.registry.util.UrlConnectionException;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 import java.util.List;
 import java.util.Random;
@@ -116,7 +117,7 @@ public final class NordnUploadAction implements Runnable {
   public void run() {
     try {
       processLordnTasks();
-    } catch (IOException e) {
+    } catch (IOException | GeneralSecurityException e) {
       throw new RuntimeException(e);
     }
   }
@@ -161,7 +162,7 @@ public final class NordnUploadAction implements Runnable {
     }
   }
 
-  private void processLordnTasks() throws IOException {
+  private void processLordnTasks() throws IOException, GeneralSecurityException {
     checkArgument(phase.equals(PARAM_LORDN_PHASE_SUNRISE)
         || phase.equals(PARAM_LORDN_PHASE_CLAIMS),
         "Invalid phase specified to Nordn servlet: %s.", phase);
@@ -194,7 +195,8 @@ public final class NordnUploadAction implements Runnable {
    * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3">TMCH
    *     functional specifications - LORDN File</a>
    */
-  private void uploadCsvToLordn(String urlPath, String csvData) throws IOException {
+  private void uploadCsvToLordn(String urlPath, String csvData)
+      throws IOException, GeneralSecurityException {
     String url = tmchMarksdbUrl + urlPath;
     logger.atInfo().log(
         "LORDN upload task %s: Sending to URL: %s ; data: %s", actionLogId, url, csvData);

core/src/main/java/google/registry/tmch/NordnVerifyAction.java

@@ -34,6 +34,7 @@ import google.registry.util.UrlConnectionException;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.security.GeneralSecurityException;
 import java.util.Map.Entry;
 import javax.inject.Inject;
 
@@ -77,7 +78,7 @@ public final class NordnVerifyAction implements Runnable {
   public void run() {
     try {
       verify();
-    } catch (IOException e) {
+    } catch (IOException | GeneralSecurityException e) {
       throw new RuntimeException(e);
     }
   }
@@ -89,11 +90,11 @@ public final class NordnVerifyAction implements Runnable {
    * available.
    *
    * @throws ConflictException if MarksDB has not yet finished processing the LORDN upload
-   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3.1">
-   *     TMCH functional specifications LORDN Log File</a>
+   * @see <a href="http://tools.ietf.org/html/draft-lozano-tmch-func-spec-08#section-6.3.1">TMCH
+   *     functional specifications LORDN Log File</a>
    */
   @VisibleForTesting
-  LordnLog verify() throws IOException {
+  LordnLog verify() throws IOException, GeneralSecurityException {
     logger.atInfo().log("LORDN verify task %s: Sending request to URL %s", actionLogId, url);
     HttpURLConnection connection = urlConnectionService.createConnection(url);
     lordnRequestInitializer.initialize(connection, tld);

core/src/main/java/google/registry/tmch/TmchCertificateAuthority.java

@@ -41,8 +41,8 @@ import javax.inject.Inject;
  * Helper methods for accessing ICANN's TMCH root certificate and revocation list.
  *
  * <p>There are two CRLs, a real one for the production environment and a pilot one for
- * non-production environments. The Datastore singleton {@link TmchCrl} entity is used to cache this
- * CRL once loaded and will always contain the proper one corresponding to the environment.
+ * non-production environments. The singleton {@link TmchCrl} entity is used to cache this CRL once
+ * loaded and will always contain the proper one corresponding to the environment.
  *
  * <p>The CRTs do not change and are included as files in the codebase that are not refreshed. They
  * were downloaded from https://ca.icann.org/tmch.crt and https://ca.icann.org/tmch_pilot.crt
@@ -66,7 +66,7 @@ public final class TmchCertificateAuthority {
   }
 
   /**
-   * A cached supplier that loads the CRL from Datastore or chooses a default value.
+   * A cached supplier that loads the CRL from the database or chooses a default value.
    *
    * <p>We keep the cache here rather than caching TmchCrl in the model, because loading the CRL
    * string into an X509CRL instance is expensive and should itself be cached.
@@ -132,7 +132,7 @@ public final class TmchCertificateAuthority {
   }
 
   /**
-   * Update to the latest TMCH X.509 certificate revocation list and save it to Datastore.
+   * Update to the latest TMCH X.509 certificate revocation list and save it to the database.
    *
    * <p>Your ASCII-armored CRL must be signed by the current ICANN root certificate.
    *

core/src/main/java/google/registry/tmch/TmchCrlAction.java

@@ -40,7 +40,7 @@ public final class TmchCrlAction implements Runnable {
   @Inject TmchCertificateAuthority tmchCertificateAuthority;
   @Inject TmchCrlAction() {}
 
-  /** Synchronously fetches latest ICANN TMCH CRL and saves it to Datastore. */
+  /** Synchronously fetches latest ICANN TMCH CRL and saves it to the database. */
   @Override
   public void run() {
     try {

core/src/main/java/google/registry/tmch/TmchDnlAction.java

@@ -23,7 +23,7 @@ import google.registry.model.tmch.ClaimsListDao;
 import google.registry.request.Action;
 import google.registry.request.auth.Auth;
 import java.io.IOException;
-import java.security.SignatureException;
+import java.security.GeneralSecurityException;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -52,7 +52,7 @@ public final class TmchDnlAction implements Runnable {
     List<String> lines;
     try {
       lines = marksdb.fetchSignedCsv(marksdbDnlLoginAndPassword, DNL_CSV_PATH, DNL_SIG_PATH);
-    } catch (SignatureException | IOException | PGPException e) {
+    } catch (GeneralSecurityException | IOException | PGPException e) {
       throw new RuntimeException(e);
     }
     ClaimsList claims = ClaimsListParser.parse(lines);

core/src/main/java/google/registry/tmch/TmchSmdrlAction.java

@@ -22,7 +22,7 @@ import google.registry.model.smd.SignedMarkRevocationList;
 import google.registry.request.Action;
 import google.registry.request.auth.Auth;
 import java.io.IOException;
-import java.security.SignatureException;
+import java.security.GeneralSecurityException;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -51,7 +51,7 @@ public final class TmchSmdrlAction implements Runnable {
     List<String> lines;
     try {
       lines = marksdb.fetchSignedCsv(marksdbSmdrlLoginAndPassword, SMDRL_CSV_PATH, SMDRL_SIG_PATH);
-    } catch (SignatureException | IOException | PGPException e) {
+    } catch (GeneralSecurityException | IOException | PGPException e) {
       throw new RuntimeException(e);
     }
     SignedMarkRevocationList smdrl = SmdrlCsvParser.parse(lines);

core/src/main/java/google/registry/tools/CanonicalizeLabelsCommand.java

@@ -14,7 +14,7 @@
 
 package google.registry.tools;
 
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.beust.jcommander.Parameter;
@@ -80,7 +80,7 @@ final class CanonicalizeLabelsCommand implements Command {
 
   private String canonicalize(String rawLabel) {
     try {
-      return canonicalizeDomainName(rawLabel.replaceAll(" ", ""));
+      return canonicalizeHostname(rawLabel.replaceAll(" ", ""));
     } catch (Exception e) {
       System.err.printf("Error canonicalizing %s: %s\n", rawLabel, e.getMessage());
       return "";

core/src/main/java/google/registry/tools/ConvertIdnCommand.java

@@ -15,7 +15,7 @@
 package google.registry.tools;
 
 import static google.registry.util.DomainNameUtils.ACE_PREFIX;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -38,7 +38,7 @@ final class ConvertIdnCommand implements Command {
       if (label.startsWith(ACE_PREFIX)) {
         System.out.println(Idn.toUnicode(Ascii.toLowerCase(label)));
       } else {
-        System.out.println(canonicalizeDomainName(label));
+        System.out.println(canonicalizeHostname(label));
       }
     }
   }

core/src/main/java/google/registry/tools/CreateHostCommand.java

@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.net.InetAddresses;
 import com.google.template.soy.data.SoyMapData;
 import google.registry.tools.soy.HostCreateSoyInfo;
+import google.registry.util.DomainNameUtils;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
@@ -68,7 +69,7 @@ final class CreateHostCommand extends MutatingEppToolCommand {
     addSoyRecord(
         clientId,
         new SoyMapData(
-            "hostname", hostName,
+            "hostname", DomainNameUtils.canonicalizeHostname(hostName),
             "ipv4addresses", ipv4Addresses.build(),
             "ipv6addresses", ipv6Addresses.build()));
   }

core/src/main/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Predicates.isNull;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.RegistrarUtils.normalizeRegistrarName;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static org.joda.time.DateTimeZone.UTC;
@@ -319,7 +319,7 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
             addAllowedTlds.isEmpty(), "Can't specify both --allowedTlds and --addAllowedTlds");
         ImmutableSet.Builder<String> allowedTldsBuilder = new ImmutableSet.Builder<>();
         for (String allowedTld : allowedTlds) {
-          allowedTldsBuilder.add(canonicalizeDomainName(allowedTld));
+          allowedTldsBuilder.add(canonicalizeHostname(allowedTld));
         }
         builder.setAllowedTlds(allowedTldsBuilder.build());
       }
@@ -329,7 +329,7 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
           allowedTldsBuilder.addAll(oldRegistrar.getAllowedTlds());
         }
         for (String allowedTld : addAllowedTlds) {
-          allowedTldsBuilder.add(canonicalizeDomainName(allowedTld));
+          allowedTldsBuilder.add(canonicalizeHostname(allowedTld));
         }
         builder.setAllowedTlds(allowedTldsBuilder.build());
       }

core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java

@@ -16,7 +16,7 @@ package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.util.CollectionUtils.findDuplicates;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.beust.jcommander.Parameter;
 import com.google.common.base.Joiner;
@@ -263,10 +263,10 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
         "Can't update roid suffixes on multiple TLDs simultaneously");
     for (String tld : tlds) {
       checkArgument(
-          tld.equals(canonicalizeDomainName(tld)),
+          tld.equals(canonicalizeHostname(tld)),
           "TLD '%s' should be given in the canonical form '%s'",
           tld,
-          canonicalizeDomainName(tld));
+          canonicalizeHostname(tld));
       checkArgument(
           !Character.isDigit(tld.charAt(0)),
           "TLDs cannot begin with a number");

core/src/main/java/google/registry/tools/DeleteHostCommand.java

@@ -18,6 +18,7 @@ import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.template.soy.data.SoyMapData;
 import google.registry.tools.soy.HostDeleteSoyInfo;
+import google.registry.util.DomainNameUtils;
 
 /** A command to delete a host via EPP. */
 @Parameters(separators = " =", commandDescription = "Delete host")
@@ -50,9 +51,11 @@ final class DeleteHostCommand extends MutatingEppToolCommand {
   @Override
   protected void initMutatingEppToolCommand() {
     setSoyTemplate(HostDeleteSoyInfo.getInstance(), HostDeleteSoyInfo.DELETEHOST);
-    addSoyRecord(clientId, new SoyMapData(
-        "hostName", hostName,
-        "reason", reason,
-        "requestedByRegistrar", requestedByRegistrar));
+    addSoyRecord(
+        clientId,
+        new SoyMapData(
+            "hostName", DomainNameUtils.canonicalizeHostname(hostName),
+            "reason", reason,
+            "requestedByRegistrar", requestedByRegistrar));
   }
 }

core/src/main/java/google/registry/tools/EncryptEscrowDepositCommand.java

@@ -14,7 +14,7 @@
 
 package google.registry.tools;
 
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -61,6 +61,6 @@ class EncryptEscrowDepositCommand implements CommandWithRemoteApi {
 
   @Override
   public final void run() throws Exception {
-    encryptor.encrypt(mode, canonicalizeDomainName(tld), revision, input, outdir);
+    encryptor.encrypt(mode, canonicalizeHostname(tld), revision, input, outdir);
   }
 }

core/src/main/java/google/registry/tools/EnqueuePollMessageCommand.java

@@ -30,6 +30,7 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.util.DomainNameUtils;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -79,6 +80,7 @@ class EnqueuePollMessageCommand extends MutatingCommand {
 
   @Override
   protected final void init() {
+    domainName = DomainNameUtils.canonicalizeHostname(domainName);
     checkArgument(
         !sendToAll || isNullOrEmpty(clientIds), "Cannot specify both --all and --clients");
     tm().transact(

core/src/main/java/google/registry/tools/EppToolCommand.java

@@ -20,6 +20,7 @@ import static com.google.common.collect.Maps.filterValues;
 import static com.google.common.io.Resources.getResource;
 import static google.registry.model.tld.Registries.findTldForNameOrThrow;
 import static google.registry.tools.CommandUtilities.addHeader;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
 import static google.registry.xml.XmlTransformer.prettyPrint;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -83,8 +84,9 @@ abstract class EppToolCommand extends ConfirmingCommand
   protected static Multimap<String, String> validateAndGroupDomainNamesByTld(List<String> names) {
     ImmutableMultimap.Builder<String, String> builder = new ImmutableMultimap.Builder<>();
     for (String name : names) {
-      InternetDomainName tld = findTldForNameOrThrow(InternetDomainName.from(name));
-      builder.put(tld.toString(), name);
+      String canonicalDomain = canonicalizeHostname(name);
+      InternetDomainName tld = findTldForNameOrThrow(InternetDomainName.from(canonicalDomain));
+      builder.put(tld.toString(), canonicalDomain);
     }
     return builder.build();
   }

core/src/main/java/google/registry/tools/GenerateAllocationTokensCommand.java

@@ -16,7 +16,6 @@ package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.collect.Queues.newArrayDeque;
 import static com.google.common.collect.Sets.difference;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
@@ -44,15 +43,18 @@ import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.persistence.VKey;
 import google.registry.tools.params.TransitionListParameter.TokenStatusTransitions;
 import google.registry.util.CollectionUtils;
+import google.registry.util.DomainNameUtils;
 import google.registry.util.NonFinalForTesting;
 import google.registry.util.Retrier;
 import google.registry.util.StringGenerator;
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayDeque;
 import java.util.Collection;
 import java.util.Deque;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -164,11 +166,12 @@ class GenerateAllocationTokensCommand implements CommandWithRemoteApi {
       domainNames = null;
     } else {
       domainNames =
-          newArrayDeque(
-              Splitter.on('\n')
-                  .omitEmptyStrings()
-                  .trimResults()
-                  .split(Files.asCharSource(new File(domainNamesFile), UTF_8).read()));
+          Splitter.on('\n')
+              .omitEmptyStrings()
+              .trimResults()
+              .splitToStream(Files.asCharSource(new File(domainNamesFile), UTF_8).read())
+              .map(DomainNameUtils::canonicalizeHostname)
+              .collect(Collectors.toCollection(ArrayDeque::new));
       numTokens = domainNames.size();
     }
 

core/src/main/java/google/registry/tools/GetDomainCommand.java

@@ -19,6 +19,7 @@ import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import google.registry.model.domain.DomainBase;
+import google.registry.util.DomainNameUtils;
 import java.util.List;
 
 /** Command to show a domain resource. */
@@ -33,8 +34,11 @@ final class GetDomainCommand extends GetEppResourceCommand {
   @Override
   public void runAndPrint() {
     for (String domainName : mainParameters) {
+      String canonicalDomain = DomainNameUtils.canonicalizeHostname(domainName);
       printResource(
-          "Domain", domainName, loadByForeignKey(DomainBase.class, domainName, readTimestamp));
+          "Domain",
+          canonicalDomain,
+          loadByForeignKey(DomainBase.class, canonicalDomain, readTimestamp));
     }
   }
 }

core/src/main/java/google/registry/tools/GetHostCommand.java

@@ -19,6 +19,7 @@ import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import google.registry.model.host.HostResource;
+import google.registry.util.DomainNameUtils;
 import java.util.List;
 
 /** Command to show one or more host resources. */
@@ -32,7 +33,9 @@ final class GetHostCommand extends GetEppResourceCommand {
 
   @Override
   public void runAndPrint() {
-    mainParameters.forEach(
-        h -> printResource("Host", h, loadByForeignKey(HostResource.class, h, readTimestamp)));
+    mainParameters.stream()
+        .map(DomainNameUtils::canonicalizeHostname)
+        .forEach(
+            h -> printResource("Host", h, loadByForeignKey(HostResource.class, h, readTimestamp)));
   }
 }

core/src/main/java/google/registry/tools/ListTxnsCommand.java

@@ -0,0 +1,116 @@
+// Copyright 2022 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.tools;
+
+import static google.registry.persistence.transaction.TransactionManagerFactory.replicaJpaTm;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.common.base.Splitter;
+import com.google.common.io.BaseEncoding;
+import google.registry.model.common.Cursor;
+import google.registry.persistence.transaction.Transaction;
+import google.registry.persistence.transaction.TransactionEntity;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+
+/** Lists {@link Cursor} timestamps used by locking rolling cursor tasks, like in RDE. */
+@Parameters(separators = " =", commandDescription = "Lists the contents of the Transaction table.")
+final class ListTxnsCommand implements CommandWithRemoteApi {
+
+  @Parameter(names = "--infile", description = "Parse an input file instead of reading from db.")
+  private String infile;
+
+  @Parameter(
+      names = "--full_dump",
+      description =
+          "Do a full dump of the contents of the transaction.  Without this, "
+              + "just write transactions as CSV lines suitable for ingestion via --infile.")
+  private boolean fullDump = false;
+
+  @Override
+  public void run() {
+    if (infile == null) {
+      fetchFromDb();
+    } else {
+      parseCsvFile();
+    }
+  }
+
+  private void parseCsvFile() {
+    try {
+      BufferedReader src = Files.newBufferedReader(Paths.get(infile), UTF_8);
+      String line;
+      while ((line = src.readLine()) != null) {
+        List<String> cols = Splitter.on(",").splitToList(line);
+        writeRecord(Integer.parseInt(cols.get(0)), BaseEncoding.base64().decode(cols.get(1)));
+      }
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private void fetchFromDb() {
+    long lastTransactionId = 0;
+    List<TransactionEntity> results;
+
+    do {
+      final long txnId = lastTransactionId; // For use in the lambda.
+      results =
+          replicaJpaTm()
+              .transact(
+                  () ->
+                      replicaJpaTm()
+                          .query(
+                              "select t from TransactionEntity t where id > :lastTransactionId",
+                              TransactionEntity.class)
+                          .setParameter("lastTransactionId", txnId)
+                          .setMaxResults(1000)
+                          .getResultList());
+
+      for (TransactionEntity txn : results) {
+        writeRecord(txn.getId(), txn.getContents());
+        lastTransactionId = txn.getId();
+      }
+    } while (results.size() > 0);
+  }
+
+  private void writeRecord(long id, byte[] contents) {
+    if (fullDump) {
+      Transaction txn;
+      try {
+        txn = Transaction.deserialize(contents);
+      } catch (IOException ex) {
+        System.err.printf("Error deserializing transaction %s\n", id);
+        return;
+      }
+      System.out.printf("transaction %s <<<\n", id);
+      for (Transaction.Mutation mut : txn.getMutations()) {
+        if (mut instanceof Transaction.Update) {
+          System.out.println("updating: " + ((Transaction.Update) mut).getEntity());
+        } else {
+          System.out.println("deleting: " + ((Transaction.Delete) mut).getKey());
+        }
+      }
+      System.out.println(">>>");
+    } else {
+      System.out.printf("%s,%s\n", id, BaseEncoding.base64().encode(contents));
+    }
+  }
+}

core/src/main/java/google/registry/tools/LockDomainCommand.java

@@ -14,6 +14,8 @@
 
 package google.registry.tools;
 
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
+
 import com.beust.jcommander.Parameters;
 
 /**
@@ -26,6 +28,6 @@ public class LockDomainCommand extends LockOrUnlockDomainCommand {
 
   @Override
   protected void createAndApplyRequest(String domain) {
-    domainLockUtils.administrativelyApplyLock(domain, clientId, null, true);
+    domainLockUtils.administrativelyApplyLock(canonicalizeHostname(domain), clientId, null, true);
   }
 }

core/src/main/java/google/registry/tools/RegistryCli.java

@@ -38,7 +38,9 @@ import java.io.ByteArrayInputStream;
 import java.net.URL;
 import java.security.Security;
 import java.util.Map;
+import java.util.Optional;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.postgresql.util.PSQLException;
 
 /** Container class to create and run remote commands against a Datastore instance. */
 @Parameters(separators = " =", commandDescription = "Command-line interface to the registry")
@@ -178,14 +180,30 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
 
     try {
       runCommand(command);
-    } catch (RuntimeException ex) {
-      if (Throwables.getRootCause(ex) instanceof LoginRequiredException) {
+    } catch (RuntimeException e) {
+      if (Throwables.getRootCause(e) instanceof LoginRequiredException) {
         System.err.println("===================================================================");
         System.err.println("You must login using 'nomulus login' prior to running this command.");
         System.err.println("===================================================================");
         System.exit(1);
       } else {
-        throw ex;
+        // See if this looks like the error we get when there's another instance of nomulus tool
+        // running against SQL and give the user some additional guidance if so.
+        Optional<Throwable> psqlException =
+            Throwables.getCausalChain(e).stream()
+                .filter(x -> x instanceof PSQLException)
+                .findFirst();
+        if (psqlException.isPresent() && psqlException.get().getMessage().contains("google:5432")) {
+          e.printStackTrace();
+          System.err.println("===================================================================");
+          System.err.println(
+              "This error is likely the result of having another instance of\n"
+                  + "nomulus running at the same time.  Check your system, shut down\n"
+                  + "the other instance, and try again.");
+          System.err.println("===================================================================");
+        } else {
+          throw e;
+        }
       }
     }
   }
@@ -250,6 +268,8 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
       // Cloud SQL after the database migration. Note that the DB password is stored in Datastore
       // and it is already initialized above.
       TransactionManagerFactory.setJpaTm(() -> component.nomulusToolJpaTransactionManager().get());
+      TransactionManagerFactory.setReplicaJpaTm(
+          () -> component.nomulusToolReplicaJpaTransactionManager().get());
     }
 
     command.run();

core/src/main/java/google/registry/tools/RegistryTool.java

@@ -92,6 +92,7 @@ public final class RegistryTool {
           .put("list_registrars", ListRegistrarsCommand.class)
           .put("list_reserved_lists", ListReservedListsCommand.class)
           .put("list_tlds", ListTldsCommand.class)
+          .put("list_txns", ListTxnsCommand.class)
           .put("load_snapshot", LoadSnapshotCommand.class)
           .put("load_test", LoadTestCommand.class)
           .put("lock_domain", LockDomainCommand.class)

core/src/main/java/google/registry/tools/RegistryToolComponent.java

@@ -33,12 +33,14 @@ import google.registry.keyring.api.KeyModule;
 import google.registry.keyring.kms.KmsModule;
 import google.registry.persistence.PersistenceModule;
 import google.registry.persistence.PersistenceModule.NomulusToolJpaTm;
+import google.registry.persistence.PersistenceModule.ReadOnlyReplicaJpaTm;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.privileges.secretmanager.SecretManagerModule;
 import google.registry.rde.RdeModule;
 import google.registry.request.Modules.DatastoreServiceModule;
 import google.registry.request.Modules.Jackson2Module;
 import google.registry.request.Modules.UrlConnectionServiceModule;
+import google.registry.request.Modules.UrlFetchServiceModule;
 import google.registry.request.Modules.UserServiceModule;
 import google.registry.tools.AuthModule.LocalCredentialModule;
 import google.registry.tools.javascrap.CompareEscrowDepositsCommand;
@@ -79,6 +81,7 @@ import javax.inject.Singleton;
       RequestFactoryModule.class,
       SecretManagerModule.class,
       UrlConnectionServiceModule.class,
+      UrlFetchServiceModule.class,
       UserServiceModule.class,
       UtilsModule.class,
       VoidDnsWriterModule.class,
@@ -187,6 +190,9 @@ interface RegistryToolComponent {
   @NomulusToolJpaTm
   Lazy<JpaTransactionManager> nomulusToolJpaTransactionManager();
 
+  @ReadOnlyReplicaJpaTm
+  Lazy<JpaTransactionManager> nomulusToolReplicaJpaTransactionManager();
+
   @Component.Builder
   interface Builder {
     @BindsInstance

core/src/main/java/google/registry/tools/UniformRapidSuspensionCommand.java

@@ -38,6 +38,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.tools.soy.DomainRenewSoyInfo;
 import google.registry.tools.soy.UniformRapidSuspensionSoyInfo;
+import google.registry.util.DomainNameUtils;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -122,12 +123,14 @@ final class UniformRapidSuspensionCommand extends MutatingEppToolCommand {
   protected void initMutatingEppToolCommand() {
     superuser = true;
     DateTime now = DateTime.now(UTC);
-    ImmutableSet<String> newHostsSet = ImmutableSet.copyOf(newHosts);
+    ImmutableList<String> newCanonicalHosts =
+        newHosts.stream().map(DomainNameUtils::canonicalizeHostname).collect(toImmutableList());
+    ImmutableSet<String> newHostsSet = ImmutableSet.copyOf(newCanonicalHosts);
     Optional<DomainBase> domainOpt = loadByForeignKey(DomainBase.class, domainName, now);
     checkArgumentPresent(domainOpt, "Domain '%s' does not exist or is deleted", domainName);
     DomainBase domain = domainOpt.get();
     Set<String> missingHosts =
-        difference(newHostsSet, checkResourcesExist(HostResource.class, newHosts, now));
+        difference(newHostsSet, checkResourcesExist(HostResource.class, newCanonicalHosts, now));
     checkArgument(missingHosts.isEmpty(), "Hosts do not exist: %s", missingHosts);
     checkArgument(
         locksToPreserve.isEmpty() || undo,

core/src/main/java/google/registry/tools/UnlockDomainCommand.java

@@ -14,6 +14,8 @@
 
 package google.registry.tools;
 
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
+
 import com.beust.jcommander.Parameters;
 import java.util.Optional;
 
@@ -27,6 +29,7 @@ public class UnlockDomainCommand extends LockOrUnlockDomainCommand {
 
   @Override
   protected void createAndApplyRequest(String domain) {
-    domainLockUtils.administrativelyApplyUnlock(domain, clientId, true, Optional.empty());
+    domainLockUtils.administrativelyApplyUnlock(
+        canonicalizeHostname(domain), clientId, true, Optional.empty());
   }
 }

core/src/main/java/google/registry/tools/params/InternetDomainNameParameter.java

@@ -15,6 +15,7 @@
 package google.registry.tools.params;
 
 import com.google.common.net.InternetDomainName;
+import google.registry.util.DomainNameUtils;
 
 /** InternetDomainName CLI parameter converter/validator. */
 public final class InternetDomainNameParameter
@@ -26,6 +27,6 @@ public final class InternetDomainNameParameter
 
   @Override
   public InternetDomainName convert(String value) {
-    return InternetDomainName.from(value);
+    return InternetDomainName.from(DomainNameUtils.canonicalizeHostname(value));
   }
 }

core/src/main/java/google/registry/ui/server/RegistrarFormFields.java

@@ -17,7 +17,7 @@ package google.registry.ui.server;
 import static com.google.common.collect.Range.atLeast;
 import static com.google.common.collect.Range.atMost;
 import static com.google.common.collect.Range.closed;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.google.common.base.Ascii;
 import com.google.common.base.Splitter;
@@ -334,7 +334,7 @@ public final class RegistrarFormFields {
     if (!InternetDomainName.isValid(input)) {
       throw new FormFieldException("Not a valid hostname.");
     }
-    return canonicalizeDomainName(input);
+    return canonicalizeHostname(input);
   }
 
   public static @Nullable DateTime parseDateTime(@Nullable String input) {

core/src/main/java/google/registry/whois/WhoisReader.java

@@ -17,7 +17,7 @@ package google.registry.whois;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static google.registry.model.tld.Registries.findTldForName;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 
 import com.google.common.base.Joiner;
@@ -122,7 +122,7 @@ class WhoisReader {
         logger.atInfo().log(
             "Attempting domain lookup command using domain name '%s'.", tokens.get(1));
         return commandFactory.domainLookup(
-            InternetDomainName.from(canonicalizeDomainName(tokens.get(1))),
+            InternetDomainName.from(canonicalizeHostname(tokens.get(1))),
             fullOutput,
             whoisRedactedEmailText);
       } catch (IllegalArgumentException iae) {
@@ -152,8 +152,8 @@ class WhoisReader {
       try {
         logger.atInfo().log(
             "Attempting nameserver lookup command using %s as a hostname.", tokens.get(1));
-        return commandFactory.nameserverLookupByHost(InternetDomainName.from(
-            canonicalizeDomainName(tokens.get(1))));
+        return commandFactory.nameserverLookupByHost(
+            InternetDomainName.from(canonicalizeHostname(tokens.get(1))));
       } catch (IllegalArgumentException iae) {
         // Silently ignore this exception.
       }
@@ -187,7 +187,7 @@ class WhoisReader {
 
       // Try to parse it as a domain name or host name.
       try {
-        final InternetDomainName targetName = InternetDomainName.from(canonicalizeDomainName(arg1));
+        final InternetDomainName targetName = InternetDomainName.from(canonicalizeHostname(arg1));
 
         // We don't know at this point whether we have a domain name or a host name. We have to
         // search through our configured TLDs to see if there's one that prefixes the name.

core/src/main/resources/google/registry/beam/invoicing/sql/cloud_sql_billing_events.sql

@@ -31,7 +31,7 @@ JOIN Domain d ON b.domainRepoId = d.repoId
 JOIN Tld t ON t.tldStrId = d.tld
 LEFT JOIN BillingCancellation c ON b.id = c.refOneTime.billingId
 LEFT JOIN BillingCancellation cr ON b.cancellationMatchingBillingEvent = cr.refRecurring.billingId
-WHERE r.billingIdentifier IS NOT NULL
+WHERE r.billingAccountMap IS NOT NULL
 AND r.type = 'REAL'
 AND t.invoicingEnabled IS TRUE
 AND b.billingTime BETWEEN CAST('%FIRST_TIMESTAMP_OF_MONTH%' AS timestamp) AND CAST('%LAST_TIMESTAMP_OF_MONTH%' AS timestamp)

core/src/main/resources/google/registry/beam/rde_pipeline_metadata.json

@@ -49,22 +49,6 @@
       "regexes": [
         "[A-Za-z0-9\\-_]+"
       ]
-    },
-    {
-      "name": "workerMachineType",
-      "label": "The GCE machine type for the dataflow job workers.",
-      "helpText": "See https://cloud.google.com/dataflow/quotas#compute-engine-quotas for available machine types.",
-      "regexes": [
-        "[a-z0-9\\-]+"
-      ]
-    },
-    {
-      "name": "usePublicIps",
-      "label": "Whether the GCE workers are assigned public IPs",
-      "helpText": "Public IPs have an associated cost and there's a quota per region on the total number of public IPs assigned at a given time. If the service only needs to access GCP APIs, it's better to not use public IP, but one needs to configure the network accordingly. See https://cloud.google.com/dataflow/docs/guides/routes-firewall.",
-      "regexes": [
-        "true|false"
-      ]
     }
   ]
 }

core/src/test/java/google/registry/beam/common/DatabaseSnapshotTest.java

@@ -37,6 +37,7 @@ import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junitpioneer.jupiter.RetryingTest;
 import org.testcontainers.containers.PostgreSQLContainer;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
@@ -106,7 +107,7 @@ public class DatabaseSnapshotTest {
     try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {}
   }
 
-  @Test
+  @RetryingTest(2) // TODO(b/226945065) revert to @Test when DatbaseMigrationSchedule is removed.
   void readSnapshot() {
     try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
       Registry snapshotRegistry =
@@ -120,7 +121,7 @@ public class DatabaseSnapshotTest {
     }
   }
 
-  @Test
+  @RetryingTest(2) // TODO(b/226945065) revert to @Test when DatbaseMigrationSchedule is removed.
   void readSnapshot_withSubsequentChange() {
     try (DatabaseSnapshot databaseSnapshot = DatabaseSnapshot.createSnapshot()) {
       Registry updated =

core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java

@@ -24,6 +24,10 @@ import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static org.joda.money.CurrencyUnit.CAD;
+import static org.joda.money.CurrencyUnit.JPY;
+import static org.joda.money.CurrencyUnit.USD;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -55,6 +59,7 @@ import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.Map.Entry;
 import java.util.Optional;
+import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.PAssert;
@@ -294,6 +299,37 @@ class InvoicingPipelineTest {
     pipeline.run().waitUntilFinish();
   }
 
+  @Test
+  void testFailure_readFromCloudSqlMissingPAK() throws Exception {
+    Registrar registrar = persistNewRegistrar("TheRegistrar");
+    registrar =
+        registrar
+            .asBuilder()
+            .setBillingAccountMap(ImmutableMap.of(USD, "789"))
+            .setPoNumber(Optional.of("22446688"))
+            .build();
+    persistResource(registrar);
+    Registry test =
+        newRegistry("test", "_TEST", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
+            .asBuilder()
+            .setInvoicingEnabled(true)
+            .build();
+    persistResource(test);
+    DomainBase domain = persistActiveDomain("mycanadiandomain.test");
+
+    persistOneTimeBillingEvent(1, domain, registrar, Reason.RENEW, 3, Money.of(CAD, 20.5));
+    PCollection<BillingEvent> billingEvents = InvoicingPipeline.readFromCloudSql(options, pipeline);
+    billingEvents = billingEvents.apply(new ChangeDomainRepo());
+    PAssert.that(billingEvents).empty();
+    PipelineExecutionException thrown =
+        assertThrows(PipelineExecutionException.class, () -> pipeline.run().waitUntilFinish());
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains(
+            "Registrar TheRegistrar does not have a product account key for the currency unit:"
+                + " CAD");
+  }
+
   @Test
   void testSuccess_saveInvoiceCsv() throws Exception {
     InvoicingPipeline.saveInvoiceCsv(billingEvents, options);
@@ -338,7 +374,7 @@ class InvoicingPipelineTest {
                 + "LEFT JOIN BillingCancellation c ON b.id = c.refOneTime.billingId\n"
                 + "LEFT JOIN BillingCancellation cr ON b.cancellationMatchingBillingEvent ="
                 + " cr.refRecurring.billingId\n"
-                + "WHERE r.billingIdentifier IS NOT NULL\n"
+                + "WHERE r.billingAccountMap IS NOT NULL\n"
                 + "AND r.type = 'REAL'\n"
                 + "AND t.invoicingEnabled IS TRUE\n"
                 + "AND b.billingTime BETWEEN CAST('2017-10-01' AS timestamp) AND CAST('2017-11-01'"
@@ -362,18 +398,22 @@ class InvoicingPipelineTest {
     persistNewRegistrar("NewRegistrar");
     persistNewRegistrar("TheRegistrar");
     Registrar registrar1 = persistNewRegistrar("theRegistrar");
-    registrar1 = registrar1.asBuilder().setBillingIdentifier(234L).build();
+    registrar1 =
+        registrar1
+            .asBuilder()
+            .setBillingAccountMap(ImmutableMap.of(JPY, "234", USD, "234"))
+            .build();
     persistResource(registrar1);
     Registrar registrar2 = persistNewRegistrar("bestdomains");
     registrar2 =
         registrar2
             .asBuilder()
-            .setBillingIdentifier(456L)
+            .setBillingAccountMap(ImmutableMap.of(USD, "456"))
             .setPoNumber(Optional.of("116688"))
             .build();
     persistResource(registrar2);
     Registrar registrar3 = persistNewRegistrar("anotherRegistrar");
-    registrar3 = registrar3.asBuilder().setBillingIdentifier(789L).build();
+    registrar3 = registrar3.asBuilder().setBillingAccountMap(ImmutableMap.of(USD, "789")).build();
     persistResource(registrar3);
 
     Registry test =
@@ -397,10 +437,8 @@ class InvoicingPipelineTest {
     DomainBase domain6 = persistActiveDomain("locked.test");
     DomainBase domain7 = persistActiveDomain("update-prohibited.test");
 
-    persistOneTimeBillingEvent(
-        1, domain1, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
-    persistOneTimeBillingEvent(
-        2, domain2, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(1, domain1, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
+    persistOneTimeBillingEvent(2, domain2, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
     persistOneTimeBillingEvent(
         3,
         domain3,
@@ -410,31 +448,27 @@ class InvoicingPipelineTest {
         Money.ofMajor(CurrencyUnit.JPY, 70),
         DateTime.parse("2017-09-29T00:00:00.0Z"),
         DateTime.parse("2017-10-02T00:00:00.0Z"));
-    persistOneTimeBillingEvent(
-        4, domain4, registrar2, Reason.RENEW, 1, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(4, domain4, registrar2, Reason.RENEW, 1, Money.of(USD, 20.5));
     persistOneTimeBillingEvent(
         5,
         domain5,
         registrar3,
         Reason.CREATE,
         1,
-        Money.of(CurrencyUnit.USD, 0),
+        Money.of(USD, 0),
         DateTime.parse("2017-10-04T00:00:00.0Z"),
         DateTime.parse("2017-10-04T00:00:00.0Z"),
         Flag.SUNRISE,
         Flag.ANCHOR_TENANT);
-    persistOneTimeBillingEvent(
-        6, domain6, registrar1, Reason.SERVER_STATUS, 0, Money.of(CurrencyUnit.USD, 0));
-    persistOneTimeBillingEvent(
-        7, domain7, registrar1, Reason.SERVER_STATUS, 0, Money.of(CurrencyUnit.USD, 20));
+    persistOneTimeBillingEvent(6, domain6, registrar1, Reason.SERVER_STATUS, 0, Money.of(USD, 0));
+    persistOneTimeBillingEvent(7, domain7, registrar1, Reason.SERVER_STATUS, 0, Money.of(USD, 20));
 
     // Add billing event for a non-billable registrar
     Registrar registrar4 = persistNewRegistrar("noBillRegistrar");
-    registrar4 = registrar4.asBuilder().setBillingIdentifier(null).build();
+    registrar4 = registrar4.asBuilder().setBillingAccountMap(null).build();
     persistResource(registrar4);
     DomainBase domain8 = persistActiveDomain("non-billable.test");
-    persistOneTimeBillingEvent(
-        8, domain8, registrar4, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(8, domain8, registrar4, Reason.RENEW, 3, Money.of(USD, 20.5));
 
     // Add billing event for a non-real registrar
     Registrar registrar5 = persistNewRegistrar("notRealRegistrar");
@@ -442,19 +476,17 @@ class InvoicingPipelineTest {
         registrar5
             .asBuilder()
             .setIanaIdentifier(null)
-            .setBillingIdentifier(456L)
+            .setBillingAccountMap(ImmutableMap.of(USD, "456"))
             .setType(Registrar.Type.OTE)
             .build();
     persistResource(registrar5);
     DomainBase domain9 = persistActiveDomain("not-real.test");
-    persistOneTimeBillingEvent(
-        9, domain9, registrar5, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(9, domain9, registrar5, Reason.RENEW, 3, Money.of(USD, 20.5));
 
     // Add billing event for a non-invoicing TLD
     createTld("nobill");
     DomainBase domain10 = persistActiveDomain("test.nobill");
-    persistOneTimeBillingEvent(
-        10, domain10, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(10, domain10, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
 
     // Add billing event before October 2017
     DomainBase domain11 = persistActiveDomain("july.test");
@@ -471,8 +503,7 @@ class InvoicingPipelineTest {
     // Add a billing event with a corresponding cancellation
     DomainBase domain12 = persistActiveDomain("cancel.test");
     OneTime oneTime =
-        persistOneTimeBillingEvent(
-            12, domain12, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+        persistOneTimeBillingEvent(12, domain12, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
     DomainHistory domainHistory = persistDomainHistory(domain12, registrar1);
 
     Cancellation cancellation =
@@ -507,8 +538,7 @@ class InvoicingPipelineTest {
             .build();
     persistResource(recurring);
     OneTime oneTimeRecurring =
-        persistOneTimeBillingEvent(
-            13, domain13, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+        persistOneTimeBillingEvent(13, domain13, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
     oneTimeRecurring =
         oneTimeRecurring
             .asBuilder()

core/src/test/java/google/registry/flows/contact/ContactDeleteFlowTest.java

@@ -254,6 +254,15 @@ class ContactDeleteFlowTest extends ResourceFlowTestCase<ContactDeleteFlow, Cont
     assertIcannReportingActivityFieldLogged("srs-cont-delete");
   }
 
+  @TestOfyOnly
+  void testModification_duringNoAsyncPhase() throws Exception {
+    persistActiveContact(getUniqueIdFromCommand());
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   @TestOfyOnly
   void testModification_duringReadOnlyPhase() throws Exception {
     persistActiveContact(getUniqueIdFromCommand());

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -840,6 +840,15 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     doSuccessfulTest();
   }
 
+  @TestOfyOnly
+  void testSuccess_inNoAsyncPhase() throws Exception {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    persistContactsAndHosts();
+    runFlowAssertResponse(
+        loadFile("domain_create_response_noasync.xml", ImmutableMap.of("DOMAIN", "example.tld")));
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   @TestOfyAndSql
   void testSuccess_maxNumberOfNameservers() throws Exception {
     setEppInput("domain_create_13_nameservers.xml");

core/src/test/java/google/registry/flows/host/HostDeleteFlowTest.java

@@ -358,6 +358,15 @@ class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, HostResour
     DatabaseHelper.removeDatabaseMigrationSchedule();
   }
 
+  @TestOfyOnly
+  void testModification_duringNoAsyncPhase() {
+    persistActiveHost("ns1.example.tld");
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   private void assertOfyDeleteSuccess(String registrarId, String clientTrid, boolean isSuperuser)
       throws Exception {
     HostResource deletedHost = reloadResourceByForeignKey();

core/src/test/java/google/registry/flows/host/HostUpdateFlowTest.java

@@ -1355,7 +1355,43 @@ class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, HostResour
   }
 
   @TestOfyOnly
-  void testModification_duringReadOnlyPhase() throws Exception {
+  void testSuccess_nonHostRename_inNoAsyncPhase_succeeds() throws Exception {
+    setEppInput("host_update_name_unchanged.xml");
+    createTld("tld");
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    DomainBase domain = persistActiveDomain("example.tld");
+    HostResource oldHost = persistActiveSubordinateHost(oldHostName(), domain);
+    clock.advanceOneMilli();
+    runFlowAssertResponse(loadFile("generic_success_response.xml"));
+    // The example xml doesn't do a host rename, so reloading the host should work.
+    assertAboutHosts()
+        .that(reloadResourceByForeignKey())
+        .hasLastSuperordinateChange(oldHost.getLastSuperordinateChange())
+        .and()
+        .hasSuperordinateDomain(domain.createVKey())
+        .and()
+        .hasPersistedCurrentSponsorRegistrarId("TheRegistrar")
+        .and()
+        .hasLastTransferTime(null)
+        .and()
+        .hasOnlyOneHistoryEntryWhich()
+        .hasType(HistoryEntry.Type.HOST_UPDATE);
+    assertDnsTasksEnqueued("ns1.example.tld");
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
+  @TestOfyOnly
+  void testRename_duringNoAsyncPhase_fails() throws Exception {
+    createTld("tld");
+    persistActiveSubordinateHost(oldHostName(), persistActiveDomain("example.tld"));
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase_fails() throws Exception {
     createTld("tld");
     persistActiveSubordinateHost(oldHostName(), persistActiveDomain("example.tld"));
     DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);

core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java

@@ -17,6 +17,7 @@ package google.registry.model.common;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_ONLY;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY;
+import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_READ_ONLY;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_ONLY;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY;
@@ -71,10 +72,12 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
     runValidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY);
 
     runValidTransition(DATASTORE_PRIMARY, DATASTORE_ONLY);
-    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
+    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC);
+    runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY);
 
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_ONLY);
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY);
+    runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_NO_ASYNC);
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY);
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY);
 
@@ -94,6 +97,7 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
     runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY);
     runInvalidTransition(DATASTORE_ONLY, SQL_ONLY);
 
+    runInvalidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
     runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY_READ_ONLY);
     runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY);
     runInvalidTransition(DATASTORE_PRIMARY, SQL_ONLY);
@@ -124,7 +128,8 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
         ImmutableSortedMap.<DateTime, MigrationState>naturalOrder()
             .put(START_OF_TIME, DATASTORE_ONLY)
             .put(startTime.plusHours(1), DATASTORE_PRIMARY)
-            .put(startTime.plusHours(2), DATASTORE_PRIMARY_READ_ONLY)
+            .put(startTime.plusHours(2), DATASTORE_PRIMARY_NO_ASYNC)
+            .put(startTime.plusHours(3), DATASTORE_PRIMARY_READ_ONLY)
             .build();
     assertThat(
             assertThrows(
@@ -163,7 +168,8 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
     fakeClock.setTo(START_OF_TIME.plusDays(1));
     AllocationToken token =
         new AllocationToken.Builder().setToken("token").setTokenType(TokenType.SINGLE_USE).build();
-    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
+    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC);
+    runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY);
     assertThrows(ReadOnlyModeException.class, () -> persistResource(token));
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY);
     assertThrows(ReadOnlyModeException.class, () -> persistResource(token));

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -44,6 +44,7 @@ import google.registry.model.server.Lock;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
 import google.registry.persistence.transaction.TransactionEntity;
+import google.registry.persistence.transaction.TransactionManagerFactory;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.FakeClock;
@@ -52,6 +53,7 @@ import google.registry.testing.InjectExtension;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestObject;
 import google.registry.util.RequestStatusChecker;
+import google.registry.util.SystemClock;
 import java.lang.reflect.Proxy;
 import java.util.List;
 import java.util.logging.Level;
@@ -105,6 +107,7 @@ public class ReplicateToDatastoreActionTest {
   void tearDown() {
     DatabaseHelper.removeDatabaseMigrationSchedule();
     fakeClock.disableAutoIncrement();
+    TransactionManagerFactory.setClockForTesting(new SystemClock());
   }
 
   @RetryingTest(4)
@@ -366,8 +369,9 @@ public class ReplicateToDatastoreActionTest {
                     ImmutableSortedMap.<DateTime, MigrationState>naturalOrder()
                         .put(START_OF_TIME, MigrationState.DATASTORE_ONLY)
                         .put(START_OF_TIME.plusHours(1), MigrationState.DATASTORE_PRIMARY)
-                        .put(START_OF_TIME.plusHours(2), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
-                        .put(START_OF_TIME.plusHours(3), MigrationState.SQL_PRIMARY)
+                        .put(START_OF_TIME.plusHours(2), MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
+                        .put(START_OF_TIME.plusHours(3), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusHours(4), MigrationState.SQL_PRIMARY)
                         .put(now.plusHours(1), MigrationState.SQL_PRIMARY_READ_ONLY)
                         .put(now.plusHours(2), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
                         .put(now.plusHours(3), MigrationState.DATASTORE_PRIMARY)
@@ -391,6 +395,50 @@ public class ReplicateToDatastoreActionTest {
                 + " DATASTORE_PRIMARY.");
   }
 
+  @Test
+  void replicationWorksInReadOnly() {
+
+    // Put us in SQL primary now, readonly in an hour, then in datastore primary after 25 hours.
+    // And we'll need the TransactionManagerFactory to use the fake clock.
+    DateTime now = fakeClock.nowUtc();
+    TransactionManagerFactory.setClockForTesting(fakeClock);
+    jpaTm()
+        .transact(
+            () ->
+                DatabaseMigrationStateSchedule.set(
+                    ImmutableSortedMap.<DateTime, MigrationState>naturalOrder()
+                        .put(START_OF_TIME, MigrationState.DATASTORE_ONLY)
+                        .put(START_OF_TIME.plusHours(1), MigrationState.DATASTORE_PRIMARY)
+                        .put(START_OF_TIME.plusHours(2), MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
+                        .put(START_OF_TIME.plusHours(3), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusHours(4), MigrationState.SQL_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusHours(5), MigrationState.SQL_PRIMARY)
+                        .put(now.plusHours(1), MigrationState.SQL_PRIMARY_READ_ONLY)
+                        .put(now.plusHours(25), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .put(now.plusHours(26), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .build()));
+
+    TestObject foo = TestObject.create("foo");
+    insertInDb(foo);
+    TestObject bar = TestObject.create("bar");
+    insertInDb(bar);
+    TestObject baz = TestObject.create("baz");
+    insertInDb(baz);
+    jpaTm().transact(() -> jpaTm().delete(baz.key()));
+
+    // get to read-only
+    fakeClock.advanceBy(Duration.standardDays(1));
+
+    // process the transaction in readonly.
+    action.run();
+
+    // Forward the next day (datastore primary).  Verify that datastore has all of the changes.
+    fakeClock.advanceBy(Duration.standardDays(1));
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(foo.key()))).isEqualTo(foo);
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(bar.key()))).isEqualTo(bar);
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(baz.key()).isPresent())).isFalse();
+  }
+
   @Test
   void testFailure_cannotAcquireLock() {
     assumeTrue(ReplayExtension.replayTestsEnabled());

core/src/test/java/google/registry/persistence/converter/DatabaseMigrationScheduleTransitionConverterTest.java

@@ -47,7 +47,9 @@ public class DatabaseMigrationScheduleTransitionConverterTest {
           MigrationState.DATASTORE_ONLY,
           DateTime.parse("2001-01-01T00:00:00.0Z"),
           MigrationState.DATASTORE_PRIMARY,
-          DateTime.parse("2002-01-01T00:00:00.0Z"),
+          DateTime.parse("2002-01-01T01:00:00.0Z"),
+          MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
+          DateTime.parse("2002-01-01T02:00:00.0Z"),
           MigrationState.DATASTORE_PRIMARY_READ_ONLY,
           DateTime.parse("2002-01-02T00:00:00.0Z"),
           MigrationState.SQL_PRIMARY,

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.rde;
 
+import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
 import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
@@ -28,13 +29,20 @@ import static javax.servlet.http.HttpServletResponse.SC_OK;
 import static org.joda.time.Duration.standardDays;
 import static org.joda.time.Duration.standardSeconds;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
+import com.google.appengine.api.urlfetch.HTTPHeader;
+import com.google.appengine.api.urlfetch.HTTPRequest;
+import com.google.appengine.api.urlfetch.HTTPResponse;
+import com.google.appengine.api.urlfetch.URLFetchService;
 import com.google.cloud.storage.BlobId;
 import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
+import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteSource;
 import google.registry.gcs.GcsUtils;
 import google.registry.model.common.Cursor;
@@ -49,21 +57,20 @@ import google.registry.testing.FakeClock;
 import google.registry.testing.FakeKeyringModule;
 import google.registry.testing.FakeResponse;
 import google.registry.testing.FakeSleeper;
-import google.registry.testing.FakeUrlConnectionService;
 import google.registry.testing.TestOfyAndSql;
 import google.registry.util.Retrier;
 import google.registry.xjc.XjcXmlTransformer;
 import google.registry.xjc.rdereport.XjcRdeReportReport;
 import google.registry.xml.XmlException;
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
+import java.util.Map;
 import java.util.Optional;
 import org.bouncycastle.openpgp.PGPPublicKey;
 import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.ArgumentCaptor;
 
 /** Unit tests for {@link RdeReportAction}. */
 @DualDatabaseTest
@@ -82,21 +89,20 @@ public class RdeReportActionTest {
 
   private final FakeResponse response = new FakeResponse();
   private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
+  private final URLFetchService urlFetchService = mock(URLFetchService.class);
+  private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
+  private final HTTPResponse httpResponse = mock(HTTPResponse.class);
   private final PGPPublicKey encryptKey =
       new FakeKeyringModule().get().getRdeStagingEncryptionKey();
   private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
   private final BlobId reportFile =
       BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde");
-  private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
-  private final FakeUrlConnectionService urlConnectionService =
-      new FakeUrlConnectionService(httpUrlConnection);
-  private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
 
   private RdeReportAction createAction() {
     RdeReporter reporter = new RdeReporter();
     reporter.reportUrlPrefix = "https://rde-report.example";
+    reporter.urlFetchService = urlFetchService;
     reporter.password = "foo";
-    reporter.urlConnectionService = urlConnectionService;
     reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
     RdeReportAction action = new RdeReportAction();
     action.gcsUtils = gcsUtils;
@@ -121,7 +127,6 @@ public class RdeReportActionTest {
         Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test")));
     gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
-    when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
   }
 
   @TestOfyAndSql
@@ -137,22 +142,24 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock() throws Exception {
-    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
-    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
+    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    verify(httpUrlConnection).setRequestMethod("PUT");
-    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
-    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
-    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
-    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
+    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
+    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
+    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
+    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
+    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
+    assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
+    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -160,8 +167,9 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_withPrefix() throws Exception {
-    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
-    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
+    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     RdeReportAction action = createAction();
     action.prefix = Optional.of("job-name/");
     gcsUtils.delete(reportFile);
@@ -174,14 +182,15 @@ public class RdeReportActionTest {
     assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
 
     // Verify the HTTP request was correct.
-    verify(httpUrlConnection).setRequestMethod("PUT");
-    assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
-    assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
-    verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
-    verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
+    assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
+    assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
+    assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
+    Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
+    assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
+    assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
 
     // Verify the payload XML was the same as what's in testdata/report.xml.
-    XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
+    XjcRdeReportReport report = parseReport(request.getValue().getPayload());
     assertThat(report.getId()).isEqualTo("20101017001");
     assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
     assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
@@ -194,8 +203,9 @@ public class RdeReportActionTest {
     PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
     gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
-    when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK);
-    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
+    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
   }
@@ -228,8 +238,9 @@ public class RdeReportActionTest {
 
   @TestOfyAndSql
   void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
-    when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST);
-    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream());
+    when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
+    when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
+    when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
     InternalServerErrorException thrown =
         assertThrows(
             InternalServerErrorException.class,
@@ -240,17 +251,18 @@ public class RdeReportActionTest {
   @TestOfyAndSql
   void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
     class ExpectedThrownException extends RuntimeException {}
-    when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException());
+    when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
     assertThrows(
         ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
   }
 
   @TestOfyAndSql
   void testRunWithLock_socketTimeout_doesRetry() throws Exception {
-    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream());
-    when(httpUrlConnection.getResponseCode())
+    when(httpResponse.getResponseCode()).thenReturn(SC_OK);
+    when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
+    when(urlFetchService.fetch(request.capture()))
         .thenThrow(new SocketTimeoutException())
-        .thenReturn(SC_OK);
+        .thenReturn(httpResponse);
     createAction().runWithLock(loadRdeReportCursor());
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@@ -261,6 +273,14 @@ public class RdeReportActionTest {
     return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime();
   }
 
+  private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
+    ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
+    for (HTTPHeader header : headers) {
+      builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
+    }
+    return builder.build();
+  }
+
   private static XjcRdeReportReport parseReport(byte[] data) {
     try {
       return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));

core/src/test/java/google/registry/rde/RdeUploadActionTest.java

@@ -69,6 +69,8 @@ import google.registry.testing.FakeSleeper;
 import google.registry.testing.GpgSystemCommandExtension;
 import google.registry.testing.Lazies;
 import google.registry.testing.TestOfyAndSql;
+import google.registry.testing.TestOfyOnly;
+import google.registry.testing.TestSqlOnly;
 import google.registry.testing.sftp.SftpServerExtension;
 import google.registry.util.Retrier;
 import java.io.File;
@@ -91,6 +93,7 @@ public class RdeUploadActionTest {
 
   private static final ByteSource REPORT_XML = RdeTestData.loadBytes("report.xml");
   private static final ByteSource DEPOSIT_XML = RdeTestData.loadBytes("deposit_full.xml");
+  private static final String JOB_PREFIX = "rde-2010-10-17t00-00-00z";
 
   private static final BlobId GHOSTRYDE_FILE =
       BlobId.of("bucket", "tld_2010-10-17_full_S1_R0.xml.ghostryde");
@@ -99,11 +102,11 @@ public class RdeUploadActionTest {
   private static final BlobId REPORT_FILE =
       BlobId.of("bucket", "tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
   private static final BlobId GHOSTRYDE_FILE_WITH_PREFIX =
-      BlobId.of("bucket", "job-name/tld_2010-10-17_full_S1_R0.xml.ghostryde");
+      BlobId.of("bucket", JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R0.xml.ghostryde");
   private static final BlobId LENGTH_FILE_WITH_PREFIX =
-      BlobId.of("bucket", "job-name/tld_2010-10-17_full_S1_R0.xml.length");
+      BlobId.of("bucket", JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R0.xml.length");
   private static final BlobId REPORT_FILE_WITH_PREFIX =
-      BlobId.of("bucket", "job-name/tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
+      BlobId.of("bucket", JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
 
   private static final BlobId GHOSTRYDE_R1_FILE =
       BlobId.of("bucket", "tld_2010-10-17_full_S1_R1.xml.ghostryde");
@@ -181,7 +184,6 @@ public class RdeUploadActionTest {
     return jschSpy;
   }
 
-
   @BeforeEach
   void beforeEach() throws Exception {
     // Force "development" mode so we don't try to really connect to GCS.
@@ -194,6 +196,13 @@ public class RdeUploadActionTest {
     gcsUtils.createFromBytes(LENGTH_R1_FILE, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
     gcsUtils.createFromBytes(REPORT_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     gcsUtils.createFromBytes(REPORT_R1_FILE, Ghostryde.encode(REPORT_XML.read(), encryptKey));
+    gcsUtils.createFromBytes(
+        GHOSTRYDE_FILE_WITH_PREFIX, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
+    gcsUtils.createFromBytes(
+        LENGTH_FILE_WITH_PREFIX, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
+    gcsUtils.createFromBytes(
+        REPORT_FILE_WITH_PREFIX, Ghostryde.encode(REPORT_XML.read(), encryptKey));
+
     tm().transact(
             () -> {
               RdeRevision.saveRevision("lol", DateTime.parse("2010-10-17TZ"), FULL, 0);
@@ -284,13 +293,36 @@ public class RdeUploadActionTest {
     assertThat(thrown).hasMessageThat().contains("The crow flies in square circles.");
   }
 
-  @TestOfyAndSql
+  @TestSqlOnly
+  void testRunWithLock_cannotGuessPrefix() throws Exception {
+    int port = sftpd.serve("user", "password", folder);
+    URI uploadUrl = URI.create(String.format("sftp://user:password@localhost:%d/", port));
+    DateTime stagingCursor = DateTime.parse("2010-10-18TZ");
+    DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
+    persistResource(Cursor.create(RDE_STAGING, stagingCursor, Registry.get("tld")));
+    gcsUtils.delete(GHOSTRYDE_FILE_WITH_PREFIX);
+    gcsUtils.delete(LENGTH_FILE_WITH_PREFIX);
+    gcsUtils.delete(REPORT_FILE_WITH_PREFIX);
+    RdeUploadAction action = createAction(uploadUrl);
+    NoContentException thrown =
+        assertThrows(NoContentException.class, () -> action.runWithLock(uploadCursor));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo("RDE deposit for TLD tld on 2010-10-17T00:00:00.000Z does not exist");
+    cloudTasksHelper.assertNoTasksEnqueued("rde-upload");
+    assertThat(folder.list()).isEmpty();
+  }
+
+  @TestOfyOnly
   void testRunWithLock_copiesOnGcs() throws Exception {
     int port = sftpd.serve("user", "password", folder);
     URI uploadUrl = URI.create(String.format("sftp://user:password@localhost:%d/", port));
     DateTime stagingCursor = DateTime.parse("2010-10-18TZ");
     DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
     persistResource(Cursor.create(RDE_STAGING, stagingCursor, Registry.get("tld")));
+    gcsUtils.delete(GHOSTRYDE_FILE_WITH_PREFIX);
+    gcsUtils.delete(LENGTH_FILE_WITH_PREFIX);
+    gcsUtils.delete(REPORT_FILE_WITH_PREFIX);
     createAction(uploadUrl).runWithLock(uploadCursor);
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@@ -306,7 +338,7 @@ public class RdeUploadActionTest {
         .isEqualTo(Files.toByteArray(new File(folder, sigFilename)));
   }
 
-  @TestOfyAndSql
+  @TestSqlOnly
   void testRunWithLock_copiesOnGcs_withPrefix() throws Exception {
     int port = sftpd.serve("user", "password", folder);
     URI uploadUrl = URI.create(String.format("sftp://user:password@localhost:%d/", port));
@@ -314,16 +346,10 @@ public class RdeUploadActionTest {
     DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
     persistResource(Cursor.create(RDE_STAGING, stagingCursor, Registry.get("tld")));
     RdeUploadAction action = createAction(uploadUrl);
-    action.prefix = Optional.of("job-name/");
+    action.prefix = Optional.of(JOB_PREFIX + "-job-name/");
     gcsUtils.delete(GHOSTRYDE_FILE);
-    gcsUtils.createFromBytes(
-        GHOSTRYDE_FILE_WITH_PREFIX, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
     gcsUtils.delete(LENGTH_FILE);
-    gcsUtils.createFromBytes(
-        LENGTH_FILE_WITH_PREFIX, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
     gcsUtils.delete(REPORT_FILE);
-    gcsUtils.createFromBytes(
-        REPORT_FILE_WITH_PREFIX, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     action.runWithLock(uploadCursor);
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
@@ -331,9 +357,49 @@ public class RdeUploadActionTest {
     cloudTasksHelper.assertNoTasksEnqueued("rde-upload");
     // Assert that both files are written to SFTP and GCS, and that the contents are identical.
     String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde";
-    String rydeGcsFilename = "job-name/tld_2010-10-17_full_S1_R0.ryde";
+    String rydeGcsFilename = JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R0.ryde";
     String sigFilename = "tld_2010-10-17_full_S1_R0.sig";
-    String sigGcsFilename = "job-name/tld_2010-10-17_full_S1_R0.sig";
+    String sigGcsFilename = JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R0.sig";
+    assertThat(folder.list()).asList().containsExactly(rydeFilename, sigFilename);
+    assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", rydeGcsFilename)))
+        .isEqualTo(Files.toByteArray(new File(folder, rydeFilename)));
+    assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", sigGcsFilename)))
+        .isEqualTo(Files.toByteArray(new File(folder, sigFilename)));
+  }
+
+  @TestSqlOnly
+  void testRunWithLock_copiesOnGcs_withoutPrefix() throws Exception {
+    int port = sftpd.serve("user", "password", folder);
+    URI uploadUrl = URI.create(String.format("sftp://user:password@localhost:%d/", port));
+    DateTime stagingCursor = DateTime.parse("2010-10-18TZ");
+    DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
+    persistResource(Cursor.create(RDE_STAGING, stagingCursor, Registry.get("tld")));
+    RdeUploadAction action = createAction(uploadUrl);
+    gcsUtils.delete(GHOSTRYDE_FILE);
+    gcsUtils.delete(LENGTH_FILE);
+    gcsUtils.delete(REPORT_FILE);
+    // Add a folder that is alphabetically before the desired folder and fill it will nonsense data.
+    // It should NOT be picked up.
+    BlobId ghostrydeFileWithPrefixBefore =
+        BlobId.of("bucket", JOB_PREFIX + "-job-nama/tld_2010-10-17_full_S1_R0.xml.ghostryde");
+    BlobId lengthFileWithPrefixBefore =
+        BlobId.of("bucket", JOB_PREFIX + "-job-nama/tld_2010-10-17_full_S1_R0.xml.length");
+    BlobId reportFileWithPrefixBefore =
+        BlobId.of(
+            "bucket", JOB_PREFIX + "-job-nama/tld_2010-10-17_full_S1_R0-report.xml.ghostryde");
+    gcsUtils.createFromBytes(ghostrydeFileWithPrefixBefore, "foo".getBytes(UTF_8));
+    gcsUtils.createFromBytes(lengthFileWithPrefixBefore, "bar".getBytes(UTF_8));
+    gcsUtils.createFromBytes(reportFileWithPrefixBefore, "baz".getBytes(UTF_8));
+    action.runWithLock(uploadCursor);
+    assertThat(response.getStatus()).isEqualTo(200);
+    assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
+    assertThat(response.getPayload()).isEqualTo("OK tld 2010-10-17T00:00:00.000Z\n");
+    cloudTasksHelper.assertNoTasksEnqueued("rde-upload");
+    // Assert that both files are written to SFTP and GCS, and that the contents are identical.
+    String rydeFilename = "tld_2010-10-17_full_S1_R0.ryde";
+    String rydeGcsFilename = JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R0.ryde";
+    String sigFilename = "tld_2010-10-17_full_S1_R0.sig";
+    String sigGcsFilename = JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R0.sig";
     assertThat(folder.list()).asList().containsExactly(rydeFilename, sigFilename);
     assertThat(gcsUtils.readBytesFrom(BlobId.of("bucket", rydeGcsFilename)))
         .isEqualTo(Files.toByteArray(new File(folder, rydeFilename)));
@@ -349,6 +415,19 @@ public class RdeUploadActionTest {
     DateTime stagingCursor = DateTime.parse("2010-10-18TZ");
     DateTime uploadCursor = DateTime.parse("2010-10-17TZ");
     persistSimpleResource(Cursor.create(RDE_STAGING, stagingCursor, Registry.get("tld")));
+    BlobId ghostrydeR1FileWithPrefix =
+        BlobId.of("bucket", JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R1.xml.ghostryde");
+    BlobId lengthR1FileWithPrefix =
+        BlobId.of("bucket", JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R1.xml.length");
+    BlobId reportR1FileWithPrefix =
+        BlobId.of(
+            "bucket", JOB_PREFIX + "-job-name/tld_2010-10-17_full_S1_R1-report.xml.ghostryde");
+    gcsUtils.createFromBytes(
+        ghostrydeR1FileWithPrefix, Ghostryde.encode(DEPOSIT_XML.read(), encryptKey));
+    gcsUtils.createFromBytes(
+        lengthR1FileWithPrefix, Long.toString(DEPOSIT_XML.size()).getBytes(UTF_8));
+    gcsUtils.createFromBytes(
+        reportR1FileWithPrefix, Ghostryde.encode(REPORT_XML.read(), encryptKey));
     createAction(uploadUrl).runWithLock(uploadCursor);
     assertThat(response.getStatus()).isEqualTo(200);
     assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -1429,6 +1429,33 @@ public class DatabaseHelper {
     return entity;
   }
 
+  /**
+   * Sets a DATASTORE_PRIMARY_NO_ASYNC state on the {@link DatabaseMigrationStateSchedule}.
+   *
+   * <p>In order to allow for tests to manipulate the clock how they need, we start the transitions
+   * one millisecond after the clock's current time (in case the clock's current value is
+   * START_OF_TIME). We then advance the clock one second so that we're in the
+   * DATASTORE_PRIMARY_READ_ONLY phase.
+   *
+   * <p>We must use the current time, otherwise the setting of the migration state will fail due to
+   * an invalid transition.
+   */
+  public static void setMigrationScheduleToDatastorePrimaryNoAsync(FakeClock fakeClock) {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm()
+        .transact(
+            () ->
+                DatabaseMigrationStateSchedule.set(
+                    ImmutableSortedMap.of(
+                        START_OF_TIME,
+                        MigrationState.DATASTORE_ONLY,
+                        now.plusMillis(1),
+                        MigrationState.DATASTORE_PRIMARY,
+                        now.plusMillis(2),
+                        MigrationState.DATASTORE_PRIMARY_NO_ASYNC)));
+    fakeClock.advanceBy(Duration.standardSeconds(1));
+  }
+
   /**
    * Sets a DATASTORE_PRIMARY_READ_ONLY state on the {@link DatabaseMigrationStateSchedule}.
    *
@@ -1452,6 +1479,8 @@ public class DatabaseHelper {
                         now.plusMillis(1),
                         MigrationState.DATASTORE_PRIMARY,
                         now.plusMillis(2),
+                        MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
+                        now.plusMillis(3),
                         MigrationState.DATASTORE_PRIMARY_READ_ONLY)));
     fakeClock.advanceBy(Duration.standardSeconds(1));
   }
@@ -1478,8 +1507,10 @@ public class DatabaseHelper {
                         now.plusMillis(1),
                         MigrationState.DATASTORE_PRIMARY,
                         now.plusMillis(2),
-                        MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                        MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                         now.plusMillis(3),
+                        MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                        now.plusMillis(4),
                         MigrationState.SQL_PRIMARY)));
     fakeClock.advanceBy(Duration.standardSeconds(1));
   }

core/src/test/java/google/registry/tools/CheckDomainClaimsCommandTest.java

@@ -37,6 +37,12 @@ class CheckDomainClaimsCommandTest extends EppToolCommandTestCase<CheckDomainCla
     eppVerifier.expectDryRun().verifySent("domain_check_claims.xml");
   }
 
+  @Test
+  void testSuccess_canonicalizesInput() throws Exception {
+    runCommand("--client=NewRegistrar", "exaMPle.TLD");
+    eppVerifier.expectDryRun().verifySent("domain_check_claims.xml");
+  }
+
   @Test
   void testSuccess_multipleTlds() throws Exception {
     runCommand("--client=NewRegistrar", "example.tld", "example.tld2");

core/src/test/java/google/registry/tools/CreateHostCommandTest.java

@@ -29,7 +29,7 @@ class CreateHostCommandTest extends EppToolCommandTestCase<CreateHostCommand> {
     createTld("tld");
     runCommandForced(
         "--client=NewRegistrar",
-        "--host=example.tld",
+        "--host=example.TLD",
         "--addresses=162.100.102.99,2001:0db8:85a3:0000:0000:8a2e:0370:7334,4.5.6.7");
     eppVerifier.verifySent("host_create_complete.xml");
   }

core/src/test/java/google/registry/tools/EnqueuePollMessageCommandTest.java

@@ -55,7 +55,8 @@ class EnqueuePollMessageCommandTest extends CommandTestCase<EnqueuePollMessageCo
 
   @TestOfyAndSql
   void testSuccess_domainAndMessage() throws Exception {
-    runCommandForced("--domain=example.tld", "--message=This domain is bad");
+    // Test canonicalization to lowercase example.tld as well.
+    runCommandForced("--domain=EXAMPLE.TLD", "--message=This domain is bad");
 
     HistoryEntry synthetic = getOnlyHistoryEntryOfType(domain, SYNTHETIC);
     assertAboutHistoryEntries()

core/src/test/java/google/registry/tools/GenerateAllocationTokensCommandTest.java

@@ -18,7 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
 import static google.registry.testing.DatabaseHelper.assertAllocationTokens;
-import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.createTlds;
 import static google.registry.testing.DatabaseHelper.loadAllOf;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -138,16 +138,18 @@ class GenerateAllocationTokensCommandTest extends CommandTestCase<GenerateAlloca
 
   @TestOfyAndSql
   void testSuccess_domainNames() throws Exception {
-    createTld("tld");
+    createTlds("tld", "xn--q9jyb4c");
     File domainNamesFile = tmpDir.resolve("domain_names.txt").toFile();
-    Files.asCharSink(domainNamesFile, UTF_8).write("foo1.tld\nboo2.tld\nbaz9.tld\n");
+    Files.asCharSink(domainNamesFile, UTF_8).write("foo1.tld\nboo2.tld\nçauçalito.みんな\n");
     runCommand("--domain_names_file", domainNamesFile.getPath());
     assertAllocationTokens(
         createToken("123456789ABCDEFG", null, "foo1.tld"),
         createToken("HJKLMNPQRSTUVWXY", null, "boo2.tld"),
-        createToken("Zabcdefghijkmnop", null, "baz9.tld"));
+        createToken("Zabcdefghijkmnop", null, "xn--aualito-txac.xn--q9jyb4c"));
     assertInStdout(
-        "foo1.tld,123456789ABCDEFG\nboo2.tld,HJKLMNPQRSTUVWXY\nbaz9.tld,Zabcdefghijkmnop");
+        "foo1.tld,123456789ABCDEFG",
+        "boo2.tld,HJKLMNPQRSTUVWXY",
+        "xn--aualito-txac.xn--q9jyb4c,Zabcdefghijkmnop");
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java

@@ -54,10 +54,12 @@ public class GetDatabaseMigrationStateCommandTest
             now.plusHours(1),
             MigrationState.DATASTORE_PRIMARY,
             now.plusHours(2),
-            MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+            MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
             now.plusHours(3),
-            MigrationState.SQL_PRIMARY,
+            MigrationState.DATASTORE_PRIMARY_READ_ONLY,
             now.plusHours(4),
+            MigrationState.SQL_PRIMARY,
+            now.plusHours(5),
             MigrationState.SQL_ONLY);
     jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions));
     runCommand();

core/src/test/java/google/registry/tools/GetDomainCommandTest.java

@@ -64,6 +64,15 @@ class GetDomainCommandTest extends CommandTestCase<GetDomainCommand> {
     assertNotInStdout("LiveRef");
   }
 
+  @Test
+  void testSuccess_canonicalizeDomainName() throws Exception {
+    createTld("xn--q9jyb4c");
+    persistActiveDomain("xn--aualito-txac.xn--q9jyb4c");
+    runCommand("çauçalito.みんな", "--expand");
+    assertInStdout("fullyQualifiedDomainName=xn--aualito-txac.xn--q9jyb4c");
+    assertInStdout("contactId=contact1234");
+  }
+
   @Test
   void testSuccess_multipleArguments() throws Exception {
     persistActiveDomain("example.tld");

core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java

@@ -64,14 +64,21 @@ public class SetDatabaseMigrationStateCommandTest
   void testSuccess_fullSchedule() throws Exception {
     DateTime now = fakeClock.nowUtc();
     DateTime datastorePrimary = now.plusHours(1);
-    DateTime datastorePrimaryReadOnly = now.plusHours(2);
-    DateTime sqlPrimary = now.plusHours(3);
-    DateTime sqlOnly = now.plusHours(4);
+    DateTime datastorePrimaryNoAsync = now.plusHours(2);
+    DateTime datastorePrimaryReadOnly = now.plusHours(3);
+    DateTime sqlPrimary = now.plusHours(4);
+    DateTime sqlOnly = now.plusHours(5);
     runCommandForced(
         String.format(
             "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
-                + "%s=DATASTORE_PRIMARY_READ_ONLY,%s=SQL_PRIMARY,%s=SQL_ONLY",
-            START_OF_TIME, datastorePrimary, datastorePrimaryReadOnly, sqlPrimary, sqlOnly));
+                + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY,"
+                + "%s=SQL_PRIMARY,%s=SQL_ONLY",
+            START_OF_TIME,
+            datastorePrimary,
+            datastorePrimaryNoAsync,
+            datastorePrimaryReadOnly,
+            sqlPrimary,
+            sqlOnly));
     assertThat(DatabaseMigrationStateSchedule.get().toValueMap())
         .containsExactlyEntriesIn(
             ImmutableSortedMap.of(
@@ -79,6 +86,8 @@ public class SetDatabaseMigrationStateCommandTest
                 MigrationState.DATASTORE_ONLY,
                 datastorePrimary,
                 MigrationState.DATASTORE_PRIMARY,
+                datastorePrimaryNoAsync,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                 datastorePrimaryReadOnly,
                 MigrationState.DATASTORE_PRIMARY_READ_ONLY,
                 sqlPrimary,
@@ -110,8 +119,9 @@ public class SetDatabaseMigrationStateCommandTest
     runCommandForced(
         String.format(
             "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
-                + "%s=DATASTORE_PRIMARY_READ_ONLY,%s=DATASTORE_PRIMARY",
-            START_OF_TIME, now.plusHours(1), now.plusHours(2), now.plusHours(3)));
+                + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY,"
+                + "%s=DATASTORE_PRIMARY",
+            START_OF_TIME, now.plusHours(1), now.plusHours(2), now.plusHours(3), now.plusHours(4)));
     assertThat(DatabaseMigrationStateSchedule.get().toValueMap())
         .containsExactlyEntriesIn(
             ImmutableSortedMap.of(
@@ -120,8 +130,10 @@ public class SetDatabaseMigrationStateCommandTest
                 now.plusHours(1),
                 MigrationState.DATASTORE_PRIMARY,
                 now.plusHours(2),
-                MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                 now.plusHours(3),
+                MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                now.plusHours(4),
                 MigrationState.DATASTORE_PRIMARY));
   }
 
@@ -152,9 +164,12 @@ public class SetDatabaseMigrationStateCommandTest
                 () ->
                     runCommandForced(
                         String.format(
-                            "--migration_schedule=%s=DATASTORE_ONLY,"
-                                + "%s=DATASTORE_PRIMARY,%s=DATASTORE_PRIMARY_READ_ONLY",
-                            START_OF_TIME, now.minusHours(2), now.minusHours(1)))))
+                            "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
+                                + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY",
+                            START_OF_TIME,
+                            now.minusHours(3),
+                            now.minusHours(2),
+                            now.minusHours(1)))))
         .hasMessageThat()
         .isEqualTo(
             "Cannot transition from current state-as-of-now DATASTORE_ONLY "

core/src/test/resources/google/registry/flows/domain/domain_create_response_noasync.xml

@@ -0,0 +1,19 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <domain:creData
+       xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>%DOMAIN%</domain:name>
+        <domain:crDate>1999-04-03T22:00:01.0Z</domain:crDate>
+        <domain:exDate>2001-04-03T22:00:01.0Z</domain:exDate>
+      </domain:creData>
+    </resData>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</epp>

db/src/main/resources/sql/er_diagram/brief_er_diagram.html

@@ -261,11 +261,11 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-     <td class="property_value">2022-03-16 18:17:28.401054</td> 
+     <td class="property_value">2022-04-01 16:53:26.766163</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V114__add_allocation_token_indexes.sql</td>
+     <td id="lastFlywayFile" class="property_value">V116__add_renewal_column_to_allocation_token.sql</td>
     </tr>
    </tbody>
   </table> 
@@ -284,7 +284,7 @@ td.section {
      generated on
     </text> 
     <text text-anchor="start" x="4055.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2022-03-16 18:17:28.401054
+     2022-04-01 16:53:26.766163
     </text> 
     <polygon fill="none" stroke="#888888" points="3968,-4 3968,-44 4233,-44 4233,-4 3968,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 

db/src/main/resources/sql/flyway.txt

@@ -112,3 +112,5 @@ V111__add_billingcancellation_missing_indexes.sql
 V112__add_billingrecurrence_missing_indexes.sql
 V113__add_host_missing_indexes.sql
 V114__add_allocation_token_indexes.sql
+V115__add_renewal_columns_to_billing_recurrence.sql
+V116__add_renewal_column_to_allocation_token.sql

db/src/main/resources/sql/flyway/V115__add_renewal_columns_to_billing_recurrence.sql

@@ -0,0 +1,17 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "BillingRecurrence" add column renewal_price_behavior text default 'DEFAULT' not null;
+alter table "BillingRecurrence" add column renewal_price_currency text;
+alter table "BillingRecurrence" add column renewal_price_amount numeric(19, 2);

db/src/main/resources/sql/flyway/V116__add_renewal_column_to_allocation_token.sql

@@ -0,0 +1,15 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "AllocationToken" add column renewal_price_behavior text default 'DEFAULT' not null;

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -51,7 +51,8 @@ CREATE TABLE public."AllocationToken" (
     redemption_domain_repo_id text,
     token_status_transitions public.hstore,
     token_type text,
-    redemption_domain_history_id bigint
+    redemption_domain_history_id bigint,
+    renewal_price_behavior text DEFAULT 'DEFAULT'::text NOT NULL
 );
 
 
@@ -116,7 +117,10 @@ CREATE TABLE public."BillingRecurrence" (
     reason text NOT NULL,
     domain_name text NOT NULL,
     recurrence_end_time timestamp with time zone,
-    recurrence_time_of_year text
+    recurrence_time_of_year text,
+    renewal_price_behavior text DEFAULT 'DEFAULT'::text NOT NULL,
+    renewal_price_currency text,
+    renewal_price_amount numeric(19,2)
 );
 
 

java-format/google-java-format-git-diff.sh

@@ -139,10 +139,11 @@ function isJavaFormatNeededOnDiffs() {
 
   if [[ ${modifiedLineCount} -ne 0 ]]; then
     echo "true"
+    exit 1
   else
     echo "false"
+    exit 0
   fi
-  exit 0
 }
 
 # The main function of this script:

util/src/main/java/google/registry/util/DomainNameUtils.java

@@ -39,8 +39,8 @@ public final class DomainNameUtils {
             .equals(potentialParent.parts());
   }
 
-  /** Canonicalizes a domain name by lowercasing and converting unicode to punycode. */
-  public static String canonicalizeDomainName(String label) {
+  /** Canonicalizes a hostname/domain name by lowercasing and converting unicode to punycode. */
+  public static String canonicalizeHostname(String label) {
     String labelLowercased = Ascii.toLowerCase(label);
     try {
       return Idn.toASCII(labelLowercased);

util/src/test/java/google/registry/util/DomainNameUtilsTest.java

@@ -15,7 +15,7 @@
 package google.registry.util;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.DomainNameUtils.getSecondLevelDomain;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -25,37 +25,38 @@ import org.junit.jupiter.api.Test;
 class DomainNameUtilsTest {
 
   @Test
-  void testCanonicalizeDomainName_succeeds() {
-    assertThat(canonicalizeDomainName("foo")).isEqualTo("foo");
-    assertThat(canonicalizeDomainName("FOO")).isEqualTo("foo");
-    assertThat(canonicalizeDomainName("foo.tld")).isEqualTo("foo.tld");
-    assertThat(canonicalizeDomainName("xn--q9jyb4c")).isEqualTo("xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("XN--Q9JYB4C")).isEqualTo("xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("みんな")).isEqualTo("xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("みんな.みんな")).isEqualTo("xn--q9jyb4c.xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("みんな.foo")).isEqualTo("xn--q9jyb4c.foo");
-    assertThat(canonicalizeDomainName("foo.みんな")).isEqualTo("foo.xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("ħ")).isEqualTo("xn--1ea");
+  void testCanonicalizeHostname_succeeds() {
+    assertThat(canonicalizeHostname("foo")).isEqualTo("foo");
+    assertThat(canonicalizeHostname("FOO")).isEqualTo("foo");
+    assertThat(canonicalizeHostname("foo.tld")).isEqualTo("foo.tld");
+    assertThat(canonicalizeHostname("xn--q9jyb4c")).isEqualTo("xn--q9jyb4c");
+    assertThat(canonicalizeHostname("XN--Q9JYB4C")).isEqualTo("xn--q9jyb4c");
+    assertThat(canonicalizeHostname("みんな")).isEqualTo("xn--q9jyb4c");
+    assertThat(canonicalizeHostname("みんな.みんな")).isEqualTo("xn--q9jyb4c.xn--q9jyb4c");
+    assertThat(canonicalizeHostname("みんな.foo")).isEqualTo("xn--q9jyb4c.foo");
+    assertThat(canonicalizeHostname("foo.みんな")).isEqualTo("foo.xn--q9jyb4c");
+    assertThat(canonicalizeHostname("BAR.foo.みんな")).isEqualTo("bar.foo.xn--q9jyb4c");
+    assertThat(canonicalizeHostname("ħ")).isEqualTo("xn--1ea");
   }
 
   @Test
-  void testCanonicalizeDomainName_allowsRdnsNames() {
-    assertThat(canonicalizeDomainName("119.63.227.45-ns1.jhz-tt.uk"))
+  void testCanonicalizeHostname_allowsRdnsNames() {
+    assertThat(canonicalizeHostname("119.63.227.45-ns1.jhz-tt.uk"))
         .isEqualTo("119.63.227.45-ns1.jhz-tt.uk");
   }
 
   @Test
-  void testCanonicalizeDomainName_throwsOn34HyphenRule() {
+  void testCanonicalizeHostname_throwsOn34HyphenRule() {
     IllegalArgumentException thrown =
         assertThrows(
             IllegalArgumentException.class,
-            () -> canonicalizeDomainName("119.63.227.45--ns1.jhz-tt.uk"));
+            () -> canonicalizeHostname("119.63.227.45--ns1.jhz-tt.uk"));
     assertThat(thrown).hasCauseThat().hasMessageThat().contains("HYPHEN_3_4");
   }
 
   @Test
-  void testCanonicalizeDomainName_acePrefixUnicodeChars() {
-    assertThrows(IllegalArgumentException.class, () -> canonicalizeDomainName("xn--みんな"));
+  void testCanonicalizeHostname_acePrefixUnicodeChars() {
+    assertThrows(IllegalArgumentException.class, () -> canonicalizeHostname("xn--みんな"));
   }
 
   @Test