Remove Optional.isEmpty() in code (#1585)

* Remove Optional.isEmpty() in code

core/src/main/java/google/registry/beam/initsql/Transforms.java

@@ -22,7 +22,7 @@ import static google.registry.beam.initsql.BackupPaths.getExportFilePatterns;
 import static google.registry.model.ofy.ObjectifyService.auditedOfy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static java.util.Comparator.comparing;
 import static org.apache.beam.sdk.values.TypeDescriptors.kvs;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -343,11 +343,11 @@ public final class Transforms {
       // Canonicalize old domain/host names from 2016 and earlier before we were enforcing this.
       entity.setIndexedProperty(
           "fullyQualifiedDomainName",
-          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedDomainName")));
+          canonicalizeHostname((String) entity.getProperty("fullyQualifiedDomainName")));
     } else if (entity.getKind().equals("HostResource")) {
       entity.setIndexedProperty(
           "fullyQualifiedHostName",
-          canonicalizeDomainName((String) entity.getProperty("fullyQualifiedHostName")));
+          canonicalizeHostname((String) entity.getProperty("fullyQualifiedHostName")));
     }
     return entity;
   }

core/src/main/java/google/registry/beam/invoicing/InvoicingPipeline.java

@@ -14,6 +14,7 @@
 
 package google.registry.beam.invoicing;
 
+import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.beam.BeamUtils.getQueryFromFile;
 import static org.apache.beam.sdk.values.TypeDescriptors.strings;
@@ -53,6 +54,7 @@ import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.KV;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.TypeDescriptor;
+import org.joda.money.CurrencyUnit;
 
 /**
  * Definition of a Dataflow Flex pipeline template, which generates a given month's invoices.
@@ -122,12 +124,19 @@ public class InvoicingPipeline implements Serializable {
     google.registry.model.billing.BillingEvent.OneTime oneTime =
         (google.registry.model.billing.BillingEvent.OneTime) row[0];
     Registrar registrar = (Registrar) row[1];
+    CurrencyUnit currency = oneTime.getCost().getCurrencyUnit();
+    checkState(
+        registrar.getBillingAccountMap().containsKey(currency),
+        "Registrar %s does not have a product account key for the currency unit: %s",
+        registrar.getRegistrarId(),
+        currency);
+
     return BillingEvent.create(
         oneTime.getId(),
         DateTimeUtils.toZonedDateTime(oneTime.getBillingTime(), ZoneId.of("UTC")),
         DateTimeUtils.toZonedDateTime(oneTime.getEventTime(), ZoneId.of("UTC")),
         registrar.getRegistrarId(),
-        registrar.getBillingIdentifier().toString(),
+        registrar.getBillingAccountMap().get(currency),
         registrar.getPoNumber().orElse(""),
         DomainNameUtils.getTldFromDomainName(oneTime.getTargetId()),
         oneTime.getReason().toString(),

core/src/main/java/google/registry/flows/CheckApiAction.java

@@ -31,7 +31,7 @@ import static google.registry.monitoring.whitebox.CheckApiMetric.Status.UNKNOWN_
 import static google.registry.monitoring.whitebox.CheckApiMetric.Tier.PREMIUM;
 import static google.registry.monitoring.whitebox.CheckApiMetric.Tier.STANDARD;
 import static google.registry.pricing.PricingEngineProxy.isDomainPremium;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static org.json.simple.JSONValue.toJSONString;
 
 import com.google.common.collect.ImmutableList;
@@ -105,7 +105,7 @@ public class CheckApiAction implements Runnable {
     String domainString;
     InternetDomainName domainName;
     try {
-      domainString = canonicalizeDomainName(nullToEmpty(domain));
+      domainString = canonicalizeHostname(nullToEmpty(domain));
       domainName = validateDomainName(domainString);
     } catch (IllegalArgumentException | EppException e) {
       metricBuilder.status(INVALID_NAME);

core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java

@@ -25,6 +25,7 @@ import static google.registry.model.ResourceTransferUtils.handlePendingTransferO
 import static google.registry.model.eppoutput.Result.Code.SUCCESS;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
 import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED;
+import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
@@ -94,6 +95,7 @@ public final class ContactDeleteFlow implements TransactionalFlow {
     extensionManager.register(MetadataExtension.class);
     validateRegistrarIsLoggedIn(registrarId);
     extensionManager.validate();
+    assertAsyncActionsAreAllowed();
     DateTime now = tm().getTransactionTime();
     checkLinkedDomains(targetId, now, ContactResource.class, DomainBase::getReferencedContacts);
     ContactResource existingContact = loadAndVerifyExistence(ContactResource.class, targetId, now);

core/src/main/java/google/registry/flows/host/HostDeleteFlow.java

@@ -22,6 +22,7 @@ import static google.registry.flows.ResourceFlowUtils.verifyResourceOwnership;
 import static google.registry.flows.host.HostFlowUtils.validateHostName;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS;
 import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
+import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
 import com.google.common.collect.ImmutableSet;
@@ -96,6 +97,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
     extensionManager.register(MetadataExtension.class);
     validateRegistrarIsLoggedIn(registrarId);
     extensionManager.validate();
+    assertAsyncActionsAreAllowed();
     DateTime now = tm().getTransactionTime();
     validateHostName(targetId);
     checkLinkedDomains(targetId, now, HostResource.class, DomainBase::getNameservers);

core/src/main/java/google/registry/flows/host/HostUpdateFlow.java

@@ -28,6 +28,7 @@ import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomain
 import static google.registry.flows.host.HostFlowUtils.verifySuperordinateDomainOwnership;
 import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
 import static google.registry.model.reporting.HistoryEntry.Type.HOST_UPDATE;
+import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
 
@@ -136,6 +137,9 @@ public final class HostUpdateFlow implements TransactionalFlow {
     validateHostName(targetId);
     HostResource existingHost = loadAndVerifyExistence(HostResource.class, targetId, now);
     boolean isHostRename = suppliedNewHostName != null;
+    if (isHostRename) {
+      assertAsyncActionsAreAllowed();
+    }
     String oldHostName = targetId;
     String newHostName = firstNonNull(suppliedNewHostName, oldHostName);
     DomainBase oldSuperordinateDomain =

core/src/main/java/google/registry/model/common/DatabaseMigrationStateSchedule.java

@@ -30,6 +30,7 @@ import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.common.TimedTransitionProperty.TimedTransition;
 import google.registry.model.replay.SqlOnlyEntity;
 import java.time.Duration;
+import java.util.Arrays;
 import javax.persistence.Entity;
 import javax.persistence.PersistenceException;
 import org.joda.time.DateTime;
@@ -62,11 +63,28 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
    * not the phase is read-only.
    */
   public enum MigrationState {
+    /** Datastore is the only DB being used. */
     DATASTORE_ONLY(PrimaryDatabase.DATASTORE, false, ReplayDirection.NO_REPLAY),
+
+    /** Datastore is the primary DB, with changes replicated to Cloud SQL. */
     DATASTORE_PRIMARY(PrimaryDatabase.DATASTORE, false, ReplayDirection.DATASTORE_TO_SQL),
+
+    /** Datastore is the primary DB, with replication, and async actions are disallowed. */
+    DATASTORE_PRIMARY_NO_ASYNC(PrimaryDatabase.DATASTORE, false, ReplayDirection.DATASTORE_TO_SQL),
+
+    /** Datastore is the primary DB, with replication, and all mutating actions are disallowed. */
     DATASTORE_PRIMARY_READ_ONLY(PrimaryDatabase.DATASTORE, true, ReplayDirection.DATASTORE_TO_SQL),
+
+    /**
+     * Cloud SQL is the primary DB, with replication back to Datastore, and all mutating actions are
+     * disallowed.
+     */
     SQL_PRIMARY_READ_ONLY(PrimaryDatabase.CLOUD_SQL, true, ReplayDirection.SQL_TO_DATASTORE),
+
+    /** Cloud SQL is the primary DB, with changes replicated to Datastore. */
     SQL_PRIMARY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.SQL_TO_DATASTORE),
+
+    /** Cloud SQL is the only DB being used. */
     SQL_ONLY(PrimaryDatabase.CLOUD_SQL, false, ReplayDirection.NO_REPLAY);
 
     private final PrimaryDatabase primaryDatabase;
@@ -146,11 +164,17 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
             .putAll(
                 MigrationState.DATASTORE_PRIMARY,
                 MigrationState.DATASTORE_ONLY,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
+            .putAll(
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
+                MigrationState.DATASTORE_ONLY,
+                MigrationState.DATASTORE_PRIMARY,
                 MigrationState.DATASTORE_PRIMARY_READ_ONLY)
             .putAll(
                 MigrationState.DATASTORE_PRIMARY_READ_ONLY,
                 MigrationState.DATASTORE_ONLY,
                 MigrationState.DATASTORE_PRIMARY,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                 MigrationState.SQL_PRIMARY_READ_ONLY,
                 MigrationState.SQL_PRIMARY)
             .putAll(
@@ -165,10 +189,9 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
                 MigrationState.SQL_ONLY,
                 MigrationState.SQL_PRIMARY_READ_ONLY,
                 MigrationState.SQL_PRIMARY);
+
     // In addition, we can always transition from a state to itself (useful when updating the map).
-    for (MigrationState migrationState : MigrationState.values()) {
-      builder.put(migrationState, migrationState);
-    }
+    Arrays.stream(MigrationState.values()).forEach(state -> builder.put(state, state));
     return builder.build();
   }
 
@@ -246,7 +269,7 @@ public class DatabaseMigrationStateSchedule extends CrossTldSingleton implements
    * A provided map of transitions may be valid by itself (i.e. it shifts states properly, doesn't
    * skip states, and doesn't backtrack incorrectly) while still being invalid. In addition to the
    * transitions in the map being valid, the single transition from the current map at the current
-   * time to the new map at the current time time must also be valid.
+   * time to the new map at the current time must also be valid.
    */
   private static void validateTransitionAtCurrentTime(
       TimedTransitionProperty<MigrationState, MigrationStateTransition> newTransitions) {

core/src/main/java/google/registry/model/domain/DomainBase.java

@@ -175,7 +175,7 @@ public class DomainBase extends DomainContent
 
   @Override
   public void beforeSqlSaveOnReplay() {
-    fullyQualifiedDomainName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedDomainName);
+    fullyQualifiedDomainName = DomainNameUtils.canonicalizeHostname(fullyQualifiedDomainName);
     dsData =
         dsData.stream()
             .filter(datum -> datum.getDigest() != null && datum.getDigest().length > 0)

core/src/main/java/google/registry/model/domain/DomainContent.java

@@ -34,7 +34,7 @@ import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.earliestOf;
 import static google.registry.util.DateTimeUtils.isBeforeOrAt;
 import static google.registry.util.DateTimeUtils.leapSafeAddYears;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.DomainNameUtils.getTldFromDomainName;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
@@ -890,7 +890,7 @@ public class DomainContent extends EppResource
 
     public B setDomainName(String domainName) {
       checkArgument(
-          domainName.equals(canonicalizeDomainName(domainName)),
+          domainName.equals(canonicalizeHostname(domainName)),
           "Domain name %s not in puny-coded, lower-case form",
           domainName);
       getInstance().fullyQualifiedDomainName = domainName;

core/src/main/java/google/registry/model/domain/DomainHistory.java

@@ -309,7 +309,7 @@ public class DomainHistory extends HistoryEntry implements SqlEntity {
     if (domainContent == null) {
       domainContent = jpaTm().getEntityManager().find(DomainBase.class, getDomainRepoId());
       domainContent.fullyQualifiedDomainName =
-          DomainNameUtils.canonicalizeDomainName(domainContent.fullyQualifiedDomainName);
+          DomainNameUtils.canonicalizeHostname(domainContent.fullyQualifiedDomainName);
       fillAuxiliaryFieldsFromDomain(this);
     }
   }

core/src/main/java/google/registry/model/host/HostBase.java

@@ -19,7 +19,7 @@ import static com.google.common.collect.Sets.difference;
 import static com.google.common.collect.Sets.union;
 import static google.registry.util.CollectionUtils.nullToEmptyImmutableCopy;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.google.common.collect.ImmutableSet;
 import com.googlecode.objectify.annotation.IgnoreSave;
@@ -195,7 +195,7 @@ public class HostBase extends EppResource {
 
     public B setHostName(String hostName) {
       checkArgument(
-          hostName.equals(canonicalizeDomainName(hostName)),
+          hostName.equals(canonicalizeHostname(hostName)),
           "Host name %s not in puny-coded, lower-case form",
           hostName);
       getInstance().fullyQualifiedHostName = hostName;

core/src/main/java/google/registry/model/host/HostHistory.java

@@ -143,7 +143,7 @@ public class HostHistory extends HistoryEntry implements SqlEntity, UnsafeSerial
     if (hostBase == null) {
       hostBase = jpaTm().getEntityManager().find(HostResource.class, getHostRepoId());
       hostBase.fullyQualifiedHostName =
-          DomainNameUtils.canonicalizeDomainName(hostBase.fullyQualifiedHostName);
+          DomainNameUtils.canonicalizeHostname(hostBase.fullyQualifiedHostName);
     }
   }
 

core/src/main/java/google/registry/model/host/HostResource.java

@@ -73,7 +73,7 @@ public class HostResource extends HostBase
 
   @Override
   public void beforeSqlSaveOnReplay() {
-    fullyQualifiedHostName = DomainNameUtils.canonicalizeDomainName(fullyQualifiedHostName);
+    fullyQualifiedHostName = DomainNameUtils.canonicalizeHostname(fullyQualifiedHostName);
   }
 
   @Override

core/src/main/java/google/registry/model/ofy/CommitLogMutation.java

@@ -69,7 +69,7 @@ public class CommitLogMutation extends ImmutableObject implements DatastoreOnlyE
    * converted to a raw Datastore Entity, serialized to bytes, and stored within the mutation.
    */
   public static CommitLogMutation create(Key<CommitLogManifest> parent, Object entity) {
-    return createFromRaw(parent, auditedOfy().save().toEntity(entity));
+    return createFromRaw(parent, auditedOfy().saveIgnoringReadOnlyWithBackup().toEntity(entity));
   }
 
   /**

core/src/main/java/google/registry/model/ofy/CommitLoggedWork.java

@@ -156,7 +156,7 @@ public class CommitLoggedWork<R> implements Runnable {
             .map(entity -> (ImmutableObject) CommitLogMutation.create(manifestKey, entity))
             .collect(toImmutableSet());
     auditedOfy()
-        .saveWithoutBackup()
+        .saveIgnoringReadOnlyWithoutBackup()
         .entities(
             new ImmutableSet.Builder<>()
                 .add(manifest)

core/src/main/java/google/registry/model/tld/label/DomainLabelEntry.java

@@ -16,7 +16,7 @@ package google.registry.model.tld.label;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Strings.emptyToNull;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
 
 import com.google.common.net.InternetDomainName;
@@ -77,7 +77,7 @@ public abstract class DomainLabelEntry<T extends Comparable<?>, D extends Domain
     public T build() {
       checkArgumentNotNull(emptyToNull(getInstance().domainLabel), "Label must be specified");
       checkArgument(
-          getInstance().domainLabel.equals(canonicalizeDomainName(getInstance().domainLabel)),
+          getInstance().domainLabel.equals(canonicalizeHostname(getInstance().domainLabel)),
           "Label '%s' must be in puny-coded, lower-case form",
           getInstance().domainLabel);
       checkArgumentNotNull(getInstance().getValue(), "Value must be specified");

core/src/main/java/google/registry/persistence/transaction/TransactionManagerFactory.java

@@ -15,23 +15,25 @@
 package google.registry.persistence.transaction;
 
 import static com.google.common.base.Preconditions.checkState;
+import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC;
 import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
-import static org.joda.time.DateTimeZone.UTC;
 
 import com.google.appengine.api.utils.SystemProperty;
 import com.google.appengine.api.utils.SystemProperty.Environment.Value;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Suppliers;
 import google.registry.config.RegistryEnvironment;
+import google.registry.model.annotations.DeleteAfterMigration;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.PrimaryDatabase;
 import google.registry.model.ofy.DatastoreTransactionManager;
 import google.registry.persistence.DaggerPersistenceComponent;
 import google.registry.tools.RegistryToolEnvironment;
+import google.registry.util.Clock;
 import google.registry.util.NonFinalForTesting;
+import google.registry.util.SystemClock;
 import java.util.Optional;
 import java.util.function.Supplier;
-import org.joda.time.DateTime;
 
 /** Factory class to create {@link TransactionManager} instance. */
 // TODO: Rename this to PersistenceFactory and move to persistence package.
@@ -42,6 +44,9 @@ public final class TransactionManagerFactory {
   /** Optional override to manually set the transaction manager per-test. */
   private static Optional<TransactionManager> tmForTest = Optional.empty();
 
+  /** The current clock (defined as a variable so we can override it in tests) */
+  private static Clock clock = new SystemClock();
+
   /** Supplier for jpaTm so that it is initialized only once, upon first usage. */
   @NonFinalForTesting
   private static Supplier<JpaTransactionManager> jpaTm =
@@ -103,7 +108,7 @@ public final class TransactionManagerFactory {
     if (onBeam) {
       return jpaTm();
     }
-    return DatabaseMigrationStateSchedule.getValueAtTime(DateTime.now(UTC))
+    return DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc())
             .getPrimaryDatabase()
             .equals(PrimaryDatabase.DATASTORE)
         ? ofyTm()
@@ -193,11 +198,33 @@ public final class TransactionManagerFactory {
   }
 
   public static void assertNotReadOnlyMode() {
-    if (DatabaseMigrationStateSchedule.getValueAtTime(DateTime.now(UTC)).isReadOnly()) {
+    if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc()).isReadOnly()) {
       throw new ReadOnlyModeException();
     }
   }
 
+  /**
+   * Asserts that async actions (contact/host deletes and host renames) are allowed.
+   *
+   * <p>These are allowed at all times except during the {@link
+   * DatabaseMigrationStateSchedule.MigrationState#DATASTORE_PRIMARY_NO_ASYNC} stage. Note that
+   * {@link ReadOnlyModeException} may well be thrown during other read-only stages inside the
+   * transaction manager; this method specifically checks only async actions.
+   */
+  @DeleteAfterMigration
+  public static void assertAsyncActionsAreAllowed() {
+    if (DatabaseMigrationStateSchedule.getValueAtTime(clock.nowUtc())
+        .equals(DATASTORE_PRIMARY_NO_ASYNC)) {
+      throw new ReadOnlyModeException();
+    }
+  }
+
+  /** Allows us to set the clock used by the factory in unit tests. */
+  @VisibleForTesting
+  public static void setClockForTesting(Clock clock) {
+    TransactionManagerFactory.clock = clock;
+  }
+
   /** Registry is currently undergoing maintenance and is in read-only mode. */
   public static class ReadOnlyModeException extends IllegalStateException {
     public ReadOnlyModeException() {

core/src/main/java/google/registry/rdap/RdapActionBase.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Charsets.UTF_8;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.net.HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN;
 import static google.registry.request.Actions.getPathForAction;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -245,7 +245,7 @@ public abstract class RdapActionBase implements Runnable {
   }
 
   String canonicalizeName(String name) {
-    name = canonicalizeDomainName(name);
+    name = canonicalizeHostname(name);
     if (name.endsWith(".")) {
       name = name.substring(0, name.length() - 1);
     }

core/src/main/java/google/registry/rde/RdeReporter.java

@@ -19,7 +19,7 @@ import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
 import static com.google.common.io.BaseEncoding.base64;
 import static com.google.common.net.HttpHeaders.AUTHORIZATION;
 import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 import static javax.servlet.http.HttpServletResponse.SC_OK;
@@ -123,7 +123,7 @@ public class RdeReporter {
 
   private URL makeReportUrl(String tld, String id) {
     try {
-      return new URL(String.format("%s/%s/%s", reportUrlPrefix, canonicalizeDomainName(tld), id));
+      return new URL(String.format("%s/%s/%s", reportUrlPrefix, canonicalizeHostname(tld), id));
     } catch (MalformedURLException e) {
       throw new RuntimeException(e);
     }

core/src/main/java/google/registry/rde/RdeUploadAction.java

@@ -143,7 +143,7 @@ public final class RdeUploadAction implements Runnable, EscrowTask {
     // If a prefix is not provided, but we are in SQL mode, try to determine the prefix. This should
     // only happen when the RDE upload cron job runs to catch up any un-retried (i. e. expected)
     // RDE failures.
-    if (prefix.isEmpty() && !tm().isOfy()) {
+    if (!prefix.isPresent() && !tm().isOfy()) {
       // The prefix is always in the format of: rde-2022-02-21t00-00-00z-2022-02-21t00-07-33z, where
       // the first datetime is the watermark and the second one is the time when the RDE beam job
       // launched. We search for the latest folder that starts with "rde-[watermark]".

core/src/main/java/google/registry/tools/CanonicalizeLabelsCommand.java

@@ -14,7 +14,7 @@
 
 package google.registry.tools;
 
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.beust.jcommander.Parameter;
@@ -80,7 +80,7 @@ final class CanonicalizeLabelsCommand implements Command {
 
   private String canonicalize(String rawLabel) {
     try {
-      return canonicalizeDomainName(rawLabel.replaceAll(" ", ""));
+      return canonicalizeHostname(rawLabel.replaceAll(" ", ""));
     } catch (Exception e) {
       System.err.printf("Error canonicalizing %s: %s\n", rawLabel, e.getMessage());
       return "";

core/src/main/java/google/registry/tools/ConvertIdnCommand.java

@@ -15,7 +15,7 @@
 package google.registry.tools;
 
 import static google.registry.util.DomainNameUtils.ACE_PREFIX;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -38,7 +38,7 @@ final class ConvertIdnCommand implements Command {
       if (label.startsWith(ACE_PREFIX)) {
         System.out.println(Idn.toUnicode(Ascii.toLowerCase(label)));
       } else {
-        System.out.println(canonicalizeDomainName(label));
+        System.out.println(canonicalizeHostname(label));
       }
     }
   }

core/src/main/java/google/registry/tools/CreateHostCommand.java

@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.net.InetAddresses;
 import com.google.template.soy.data.SoyMapData;
 import google.registry.tools.soy.HostCreateSoyInfo;
+import google.registry.util.DomainNameUtils;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
@@ -68,7 +69,7 @@ final class CreateHostCommand extends MutatingEppToolCommand {
     addSoyRecord(
         clientId,
         new SoyMapData(
-            "hostname", hostName,
+            "hostname", DomainNameUtils.canonicalizeHostname(hostName),
             "ipv4addresses", ipv4Addresses.build(),
             "ipv6addresses", ipv6Addresses.build()));
   }

core/src/main/java/google/registry/tools/CreateOrUpdateRegistrarCommand.java

@@ -18,7 +18,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Predicates.isNull;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.RegistrarUtils.normalizeRegistrarName;
 import static java.nio.charset.StandardCharsets.US_ASCII;
 import static org.joda.time.DateTimeZone.UTC;
@@ -319,7 +319,7 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
             addAllowedTlds.isEmpty(), "Can't specify both --allowedTlds and --addAllowedTlds");
         ImmutableSet.Builder<String> allowedTldsBuilder = new ImmutableSet.Builder<>();
         for (String allowedTld : allowedTlds) {
-          allowedTldsBuilder.add(canonicalizeDomainName(allowedTld));
+          allowedTldsBuilder.add(canonicalizeHostname(allowedTld));
         }
         builder.setAllowedTlds(allowedTldsBuilder.build());
       }
@@ -329,7 +329,7 @@ abstract class CreateOrUpdateRegistrarCommand extends MutatingCommand {
           allowedTldsBuilder.addAll(oldRegistrar.getAllowedTlds());
         }
         for (String allowedTld : addAllowedTlds) {
-          allowedTldsBuilder.add(canonicalizeDomainName(allowedTld));
+          allowedTldsBuilder.add(canonicalizeHostname(allowedTld));
         }
         builder.setAllowedTlds(allowedTldsBuilder.build());
       }

core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java

@@ -16,7 +16,7 @@ package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static google.registry.util.CollectionUtils.findDuplicates;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.beust.jcommander.Parameter;
 import com.google.common.base.Joiner;
@@ -263,10 +263,10 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
         "Can't update roid suffixes on multiple TLDs simultaneously");
     for (String tld : tlds) {
       checkArgument(
-          tld.equals(canonicalizeDomainName(tld)),
+          tld.equals(canonicalizeHostname(tld)),
           "TLD '%s' should be given in the canonical form '%s'",
           tld,
-          canonicalizeDomainName(tld));
+          canonicalizeHostname(tld));
       checkArgument(
           !Character.isDigit(tld.charAt(0)),
           "TLDs cannot begin with a number");

core/src/main/java/google/registry/tools/DeleteHostCommand.java

@@ -18,6 +18,7 @@ import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import com.google.template.soy.data.SoyMapData;
 import google.registry.tools.soy.HostDeleteSoyInfo;
+import google.registry.util.DomainNameUtils;
 
 /** A command to delete a host via EPP. */
 @Parameters(separators = " =", commandDescription = "Delete host")
@@ -50,9 +51,11 @@ final class DeleteHostCommand extends MutatingEppToolCommand {
   @Override
   protected void initMutatingEppToolCommand() {
     setSoyTemplate(HostDeleteSoyInfo.getInstance(), HostDeleteSoyInfo.DELETEHOST);
-    addSoyRecord(clientId, new SoyMapData(
-        "hostName", hostName,
-        "reason", reason,
-        "requestedByRegistrar", requestedByRegistrar));
+    addSoyRecord(
+        clientId,
+        new SoyMapData(
+            "hostName", DomainNameUtils.canonicalizeHostname(hostName),
+            "reason", reason,
+            "requestedByRegistrar", requestedByRegistrar));
   }
 }

core/src/main/java/google/registry/tools/EncryptEscrowDepositCommand.java

@@ -14,7 +14,7 @@
 
 package google.registry.tools;
 
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
@@ -61,6 +61,6 @@ class EncryptEscrowDepositCommand implements CommandWithRemoteApi {
 
   @Override
   public final void run() throws Exception {
-    encryptor.encrypt(mode, canonicalizeDomainName(tld), revision, input, outdir);
+    encryptor.encrypt(mode, canonicalizeHostname(tld), revision, input, outdir);
   }
 }

core/src/main/java/google/registry/tools/EnqueuePollMessageCommand.java

@@ -30,6 +30,7 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.poll.PollMessage;
 import google.registry.model.registrar.Registrar;
 import google.registry.model.reporting.HistoryEntry;
+import google.registry.util.DomainNameUtils;
 import java.util.List;
 import java.util.Optional;
 import javax.inject.Inject;
@@ -79,6 +80,7 @@ class EnqueuePollMessageCommand extends MutatingCommand {
 
   @Override
   protected final void init() {
+    domainName = DomainNameUtils.canonicalizeHostname(domainName);
     checkArgument(
         !sendToAll || isNullOrEmpty(clientIds), "Cannot specify both --all and --clients");
     tm().transact(

core/src/main/java/google/registry/tools/EppToolCommand.java

@@ -20,6 +20,7 @@ import static com.google.common.collect.Maps.filterValues;
 import static com.google.common.io.Resources.getResource;
 import static google.registry.model.tld.Registries.findTldForNameOrThrow;
 import static google.registry.tools.CommandUtilities.addHeader;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.PreconditionsUtils.checkArgumentPresent;
 import static google.registry.xml.XmlTransformer.prettyPrint;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -83,8 +84,9 @@ abstract class EppToolCommand extends ConfirmingCommand
   protected static Multimap<String, String> validateAndGroupDomainNamesByTld(List<String> names) {
     ImmutableMultimap.Builder<String, String> builder = new ImmutableMultimap.Builder<>();
     for (String name : names) {
-      InternetDomainName tld = findTldForNameOrThrow(InternetDomainName.from(name));
-      builder.put(tld.toString(), name);
+      String canonicalDomain = canonicalizeHostname(name);
+      InternetDomainName tld = findTldForNameOrThrow(InternetDomainName.from(canonicalDomain));
+      builder.put(tld.toString(), canonicalDomain);
     }
     return builder.build();
   }

core/src/main/java/google/registry/tools/GenerateAllocationTokensCommand.java

@@ -16,7 +16,6 @@ package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static com.google.common.collect.Queues.newArrayDeque;
 import static com.google.common.collect.Sets.difference;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
@@ -44,15 +43,18 @@ import google.registry.model.domain.token.AllocationToken.TokenType;
 import google.registry.persistence.VKey;
 import google.registry.tools.params.TransitionListParameter.TokenStatusTransitions;
 import google.registry.util.CollectionUtils;
+import google.registry.util.DomainNameUtils;
 import google.registry.util.NonFinalForTesting;
 import google.registry.util.Retrier;
 import google.registry.util.StringGenerator;
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayDeque;
 import java.util.Collection;
 import java.util.Deque;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -164,11 +166,12 @@ class GenerateAllocationTokensCommand implements CommandWithRemoteApi {
       domainNames = null;
     } else {
       domainNames =
-          newArrayDeque(
-              Splitter.on('\n')
-                  .omitEmptyStrings()
-                  .trimResults()
-                  .split(Files.asCharSource(new File(domainNamesFile), UTF_8).read()));
+          Splitter.on('\n')
+              .omitEmptyStrings()
+              .trimResults()
+              .splitToStream(Files.asCharSource(new File(domainNamesFile), UTF_8).read())
+              .map(DomainNameUtils::canonicalizeHostname)
+              .collect(Collectors.toCollection(ArrayDeque::new));
       numTokens = domainNames.size();
     }
 

core/src/main/java/google/registry/tools/GetDomainCommand.java

@@ -19,6 +19,7 @@ import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import google.registry.model.domain.DomainBase;
+import google.registry.util.DomainNameUtils;
 import java.util.List;
 
 /** Command to show a domain resource. */
@@ -33,8 +34,11 @@ final class GetDomainCommand extends GetEppResourceCommand {
   @Override
   public void runAndPrint() {
     for (String domainName : mainParameters) {
+      String canonicalDomain = DomainNameUtils.canonicalizeHostname(domainName);
       printResource(
-          "Domain", domainName, loadByForeignKey(DomainBase.class, domainName, readTimestamp));
+          "Domain",
+          canonicalDomain,
+          loadByForeignKey(DomainBase.class, canonicalDomain, readTimestamp));
     }
   }
 }

core/src/main/java/google/registry/tools/GetHostCommand.java

@@ -19,6 +19,7 @@ import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 import google.registry.model.host.HostResource;
+import google.registry.util.DomainNameUtils;
 import java.util.List;
 
 /** Command to show one or more host resources. */
@@ -32,7 +33,9 @@ final class GetHostCommand extends GetEppResourceCommand {
 
   @Override
   public void runAndPrint() {
-    mainParameters.forEach(
-        h -> printResource("Host", h, loadByForeignKey(HostResource.class, h, readTimestamp)));
+    mainParameters.stream()
+        .map(DomainNameUtils::canonicalizeHostname)
+        .forEach(
+            h -> printResource("Host", h, loadByForeignKey(HostResource.class, h, readTimestamp)));
   }
 }

core/src/main/java/google/registry/tools/LockDomainCommand.java

@@ -14,6 +14,8 @@
 
 package google.registry.tools;
 
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
+
 import com.beust.jcommander.Parameters;
 
 /**
@@ -26,6 +28,6 @@ public class LockDomainCommand extends LockOrUnlockDomainCommand {
 
   @Override
   protected void createAndApplyRequest(String domain) {
-    domainLockUtils.administrativelyApplyLock(domain, clientId, null, true);
+    domainLockUtils.administrativelyApplyLock(canonicalizeHostname(domain), clientId, null, true);
   }
 }

core/src/main/java/google/registry/tools/RegistryCli.java

@@ -38,7 +38,9 @@ import java.io.ByteArrayInputStream;
 import java.net.URL;
 import java.security.Security;
 import java.util.Map;
+import java.util.Optional;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.postgresql.util.PSQLException;
 
 /** Container class to create and run remote commands against a Datastore instance. */
 @Parameters(separators = " =", commandDescription = "Command-line interface to the registry")
@@ -178,14 +180,30 @@ final class RegistryCli implements AutoCloseable, CommandRunner {
 
     try {
       runCommand(command);
-    } catch (RuntimeException ex) {
-      if (Throwables.getRootCause(ex) instanceof LoginRequiredException) {
+    } catch (RuntimeException e) {
+      if (Throwables.getRootCause(e) instanceof LoginRequiredException) {
         System.err.println("===================================================================");
         System.err.println("You must login using 'nomulus login' prior to running this command.");
         System.err.println("===================================================================");
         System.exit(1);
       } else {
-        throw ex;
+        // See if this looks like the error we get when there's another instance of nomulus tool
+        // running against SQL and give the user some additional guidance if so.
+        Optional<Throwable> psqlException =
+            Throwables.getCausalChain(e).stream()
+                .filter(x -> x instanceof PSQLException)
+                .findFirst();
+        if (psqlException.isPresent() && psqlException.get().getMessage().contains("google:5432")) {
+          e.printStackTrace();
+          System.err.println("===================================================================");
+          System.err.println(
+              "This error is likely the result of having another instance of\n"
+                  + "nomulus running at the same time.  Check your system, shut down\n"
+                  + "the other instance, and try again.");
+          System.err.println("===================================================================");
+        } else {
+          throw e;
+        }
       }
     }
   }

core/src/main/java/google/registry/tools/UniformRapidSuspensionCommand.java

@@ -38,6 +38,7 @@ import google.registry.model.eppcommon.StatusValue;
 import google.registry.model.host.HostResource;
 import google.registry.tools.soy.DomainRenewSoyInfo;
 import google.registry.tools.soy.UniformRapidSuspensionSoyInfo;
+import google.registry.util.DomainNameUtils;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -122,12 +123,14 @@ final class UniformRapidSuspensionCommand extends MutatingEppToolCommand {
   protected void initMutatingEppToolCommand() {
     superuser = true;
     DateTime now = DateTime.now(UTC);
-    ImmutableSet<String> newHostsSet = ImmutableSet.copyOf(newHosts);
+    ImmutableList<String> newCanonicalHosts =
+        newHosts.stream().map(DomainNameUtils::canonicalizeHostname).collect(toImmutableList());
+    ImmutableSet<String> newHostsSet = ImmutableSet.copyOf(newCanonicalHosts);
     Optional<DomainBase> domainOpt = loadByForeignKey(DomainBase.class, domainName, now);
     checkArgumentPresent(domainOpt, "Domain '%s' does not exist or is deleted", domainName);
     DomainBase domain = domainOpt.get();
     Set<String> missingHosts =
-        difference(newHostsSet, checkResourcesExist(HostResource.class, newHosts, now));
+        difference(newHostsSet, checkResourcesExist(HostResource.class, newCanonicalHosts, now));
     checkArgument(missingHosts.isEmpty(), "Hosts do not exist: %s", missingHosts);
     checkArgument(
         locksToPreserve.isEmpty() || undo,

core/src/main/java/google/registry/tools/UnlockDomainCommand.java

@@ -14,6 +14,8 @@
 
 package google.registry.tools;
 
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
+
 import com.beust.jcommander.Parameters;
 import java.util.Optional;
 
@@ -27,6 +29,7 @@ public class UnlockDomainCommand extends LockOrUnlockDomainCommand {
 
   @Override
   protected void createAndApplyRequest(String domain) {
-    domainLockUtils.administrativelyApplyUnlock(domain, clientId, true, Optional.empty());
+    domainLockUtils.administrativelyApplyUnlock(
+        canonicalizeHostname(domain), clientId, true, Optional.empty());
   }
 }

core/src/main/java/google/registry/tools/params/InternetDomainNameParameter.java

@@ -15,6 +15,7 @@
 package google.registry.tools.params;
 
 import com.google.common.net.InternetDomainName;
+import google.registry.util.DomainNameUtils;
 
 /** InternetDomainName CLI parameter converter/validator. */
 public final class InternetDomainNameParameter
@@ -26,6 +27,6 @@ public final class InternetDomainNameParameter
 
   @Override
   public InternetDomainName convert(String value) {
-    return InternetDomainName.from(value);
+    return InternetDomainName.from(DomainNameUtils.canonicalizeHostname(value));
   }
 }

core/src/main/java/google/registry/ui/server/RegistrarFormFields.java

@@ -17,7 +17,7 @@ package google.registry.ui.server;
 import static com.google.common.collect.Range.atLeast;
 import static com.google.common.collect.Range.atMost;
 import static com.google.common.collect.Range.closed;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 
 import com.google.common.base.Ascii;
 import com.google.common.base.Splitter;
@@ -334,7 +334,7 @@ public final class RegistrarFormFields {
     if (!InternetDomainName.isValid(input)) {
       throw new FormFieldException("Not a valid hostname.");
     }
-    return canonicalizeDomainName(input);
+    return canonicalizeHostname(input);
   }
 
   public static @Nullable DateTime parseDateTime(@Nullable String input) {

core/src/main/java/google/registry/whois/WhoisReader.java

@@ -17,7 +17,7 @@ package google.registry.whois;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static google.registry.model.tld.Registries.findTldForName;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
 
 import com.google.common.base.Joiner;
@@ -122,7 +122,7 @@ class WhoisReader {
         logger.atInfo().log(
             "Attempting domain lookup command using domain name '%s'.", tokens.get(1));
         return commandFactory.domainLookup(
-            InternetDomainName.from(canonicalizeDomainName(tokens.get(1))),
+            InternetDomainName.from(canonicalizeHostname(tokens.get(1))),
             fullOutput,
             whoisRedactedEmailText);
       } catch (IllegalArgumentException iae) {
@@ -152,8 +152,8 @@ class WhoisReader {
       try {
         logger.atInfo().log(
             "Attempting nameserver lookup command using %s as a hostname.", tokens.get(1));
-        return commandFactory.nameserverLookupByHost(InternetDomainName.from(
-            canonicalizeDomainName(tokens.get(1))));
+        return commandFactory.nameserverLookupByHost(
+            InternetDomainName.from(canonicalizeHostname(tokens.get(1))));
       } catch (IllegalArgumentException iae) {
         // Silently ignore this exception.
       }
@@ -187,7 +187,7 @@ class WhoisReader {
 
       // Try to parse it as a domain name or host name.
       try {
-        final InternetDomainName targetName = InternetDomainName.from(canonicalizeDomainName(arg1));
+        final InternetDomainName targetName = InternetDomainName.from(canonicalizeHostname(arg1));
 
         // We don't know at this point whether we have a domain name or a host name. We have to
         // search through our configured TLDs to see if there's one that prefixes the name.

core/src/main/resources/google/registry/beam/invoicing/sql/cloud_sql_billing_events.sql

@@ -31,7 +31,7 @@ JOIN Domain d ON b.domainRepoId = d.repoId
 JOIN Tld t ON t.tldStrId = d.tld
 LEFT JOIN BillingCancellation c ON b.id = c.refOneTime.billingId
 LEFT JOIN BillingCancellation cr ON b.cancellationMatchingBillingEvent = cr.refRecurring.billingId
-WHERE r.billingIdentifier IS NOT NULL
+WHERE r.billingAccountMap IS NOT NULL
 AND r.type = 'REAL'
 AND t.invoicingEnabled IS TRUE
 AND b.billingTime BETWEEN CAST('%FIRST_TIMESTAMP_OF_MONTH%' AS timestamp) AND CAST('%LAST_TIMESTAMP_OF_MONTH%' AS timestamp)

core/src/test/java/google/registry/beam/invoicing/InvoicingPipelineTest.java

@@ -24,6 +24,10 @@ import static google.registry.testing.DatabaseHelper.persistNewRegistrar;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.util.DateTimeUtils.END_OF_TIME;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static org.joda.money.CurrencyUnit.CAD;
+import static org.joda.money.CurrencyUnit.JPY;
+import static org.joda.money.CurrencyUnit.USD;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -55,6 +59,7 @@ import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.Map.Entry;
 import java.util.Optional;
+import org.apache.beam.sdk.Pipeline.PipelineExecutionException;
 import org.apache.beam.sdk.coders.SerializableCoder;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.testing.PAssert;
@@ -294,6 +299,37 @@ class InvoicingPipelineTest {
     pipeline.run().waitUntilFinish();
   }
 
+  @Test
+  void testFailure_readFromCloudSqlMissingPAK() throws Exception {
+    Registrar registrar = persistNewRegistrar("TheRegistrar");
+    registrar =
+        registrar
+            .asBuilder()
+            .setBillingAccountMap(ImmutableMap.of(USD, "789"))
+            .setPoNumber(Optional.of("22446688"))
+            .build();
+    persistResource(registrar);
+    Registry test =
+        newRegistry("test", "_TEST", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
+            .asBuilder()
+            .setInvoicingEnabled(true)
+            .build();
+    persistResource(test);
+    DomainBase domain = persistActiveDomain("mycanadiandomain.test");
+
+    persistOneTimeBillingEvent(1, domain, registrar, Reason.RENEW, 3, Money.of(CAD, 20.5));
+    PCollection<BillingEvent> billingEvents = InvoicingPipeline.readFromCloudSql(options, pipeline);
+    billingEvents = billingEvents.apply(new ChangeDomainRepo());
+    PAssert.that(billingEvents).empty();
+    PipelineExecutionException thrown =
+        assertThrows(PipelineExecutionException.class, () -> pipeline.run().waitUntilFinish());
+    assertThat(thrown)
+        .hasMessageThat()
+        .contains(
+            "Registrar TheRegistrar does not have a product account key for the currency unit:"
+                + " CAD");
+  }
+
   @Test
   void testSuccess_saveInvoiceCsv() throws Exception {
     InvoicingPipeline.saveInvoiceCsv(billingEvents, options);
@@ -338,7 +374,7 @@ class InvoicingPipelineTest {
                 + "LEFT JOIN BillingCancellation c ON b.id = c.refOneTime.billingId\n"
                 + "LEFT JOIN BillingCancellation cr ON b.cancellationMatchingBillingEvent ="
                 + " cr.refRecurring.billingId\n"
-                + "WHERE r.billingIdentifier IS NOT NULL\n"
+                + "WHERE r.billingAccountMap IS NOT NULL\n"
                 + "AND r.type = 'REAL'\n"
                 + "AND t.invoicingEnabled IS TRUE\n"
                 + "AND b.billingTime BETWEEN CAST('2017-10-01' AS timestamp) AND CAST('2017-11-01'"
@@ -362,18 +398,22 @@ class InvoicingPipelineTest {
     persistNewRegistrar("NewRegistrar");
     persistNewRegistrar("TheRegistrar");
     Registrar registrar1 = persistNewRegistrar("theRegistrar");
-    registrar1 = registrar1.asBuilder().setBillingIdentifier(234L).build();
+    registrar1 =
+        registrar1
+            .asBuilder()
+            .setBillingAccountMap(ImmutableMap.of(JPY, "234", USD, "234"))
+            .build();
     persistResource(registrar1);
     Registrar registrar2 = persistNewRegistrar("bestdomains");
     registrar2 =
         registrar2
             .asBuilder()
-            .setBillingIdentifier(456L)
+            .setBillingAccountMap(ImmutableMap.of(USD, "456"))
             .setPoNumber(Optional.of("116688"))
             .build();
     persistResource(registrar2);
     Registrar registrar3 = persistNewRegistrar("anotherRegistrar");
-    registrar3 = registrar3.asBuilder().setBillingIdentifier(789L).build();
+    registrar3 = registrar3.asBuilder().setBillingAccountMap(ImmutableMap.of(USD, "789")).build();
     persistResource(registrar3);
 
     Registry test =
@@ -397,10 +437,8 @@ class InvoicingPipelineTest {
     DomainBase domain6 = persistActiveDomain("locked.test");
     DomainBase domain7 = persistActiveDomain("update-prohibited.test");
 
-    persistOneTimeBillingEvent(
-        1, domain1, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
-    persistOneTimeBillingEvent(
-        2, domain2, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(1, domain1, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
+    persistOneTimeBillingEvent(2, domain2, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
     persistOneTimeBillingEvent(
         3,
         domain3,
@@ -410,31 +448,27 @@ class InvoicingPipelineTest {
         Money.ofMajor(CurrencyUnit.JPY, 70),
         DateTime.parse("2017-09-29T00:00:00.0Z"),
         DateTime.parse("2017-10-02T00:00:00.0Z"));
-    persistOneTimeBillingEvent(
-        4, domain4, registrar2, Reason.RENEW, 1, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(4, domain4, registrar2, Reason.RENEW, 1, Money.of(USD, 20.5));
     persistOneTimeBillingEvent(
         5,
         domain5,
         registrar3,
         Reason.CREATE,
         1,
-        Money.of(CurrencyUnit.USD, 0),
+        Money.of(USD, 0),
         DateTime.parse("2017-10-04T00:00:00.0Z"),
         DateTime.parse("2017-10-04T00:00:00.0Z"),
         Flag.SUNRISE,
         Flag.ANCHOR_TENANT);
-    persistOneTimeBillingEvent(
-        6, domain6, registrar1, Reason.SERVER_STATUS, 0, Money.of(CurrencyUnit.USD, 0));
-    persistOneTimeBillingEvent(
-        7, domain7, registrar1, Reason.SERVER_STATUS, 0, Money.of(CurrencyUnit.USD, 20));
+    persistOneTimeBillingEvent(6, domain6, registrar1, Reason.SERVER_STATUS, 0, Money.of(USD, 0));
+    persistOneTimeBillingEvent(7, domain7, registrar1, Reason.SERVER_STATUS, 0, Money.of(USD, 20));
 
     // Add billing event for a non-billable registrar
     Registrar registrar4 = persistNewRegistrar("noBillRegistrar");
-    registrar4 = registrar4.asBuilder().setBillingIdentifier(null).build();
+    registrar4 = registrar4.asBuilder().setBillingAccountMap(null).build();
     persistResource(registrar4);
     DomainBase domain8 = persistActiveDomain("non-billable.test");
-    persistOneTimeBillingEvent(
-        8, domain8, registrar4, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(8, domain8, registrar4, Reason.RENEW, 3, Money.of(USD, 20.5));
 
     // Add billing event for a non-real registrar
     Registrar registrar5 = persistNewRegistrar("notRealRegistrar");
@@ -442,19 +476,17 @@ class InvoicingPipelineTest {
         registrar5
             .asBuilder()
             .setIanaIdentifier(null)
-            .setBillingIdentifier(456L)
+            .setBillingAccountMap(ImmutableMap.of(USD, "456"))
             .setType(Registrar.Type.OTE)
             .build();
     persistResource(registrar5);
     DomainBase domain9 = persistActiveDomain("not-real.test");
-    persistOneTimeBillingEvent(
-        9, domain9, registrar5, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(9, domain9, registrar5, Reason.RENEW, 3, Money.of(USD, 20.5));
 
     // Add billing event for a non-invoicing TLD
     createTld("nobill");
     DomainBase domain10 = persistActiveDomain("test.nobill");
-    persistOneTimeBillingEvent(
-        10, domain10, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+    persistOneTimeBillingEvent(10, domain10, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
 
     // Add billing event before October 2017
     DomainBase domain11 = persistActiveDomain("july.test");
@@ -471,8 +503,7 @@ class InvoicingPipelineTest {
     // Add a billing event with a corresponding cancellation
     DomainBase domain12 = persistActiveDomain("cancel.test");
     OneTime oneTime =
-        persistOneTimeBillingEvent(
-            12, domain12, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+        persistOneTimeBillingEvent(12, domain12, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
     DomainHistory domainHistory = persistDomainHistory(domain12, registrar1);
 
     Cancellation cancellation =
@@ -507,8 +538,7 @@ class InvoicingPipelineTest {
             .build();
     persistResource(recurring);
     OneTime oneTimeRecurring =
-        persistOneTimeBillingEvent(
-            13, domain13, registrar1, Reason.RENEW, 3, Money.of(CurrencyUnit.USD, 20.5));
+        persistOneTimeBillingEvent(13, domain13, registrar1, Reason.RENEW, 3, Money.of(USD, 20.5));
     oneTimeRecurring =
         oneTimeRecurring
             .asBuilder()

core/src/test/java/google/registry/flows/contact/ContactDeleteFlowTest.java

@@ -254,6 +254,15 @@ class ContactDeleteFlowTest extends ResourceFlowTestCase<ContactDeleteFlow, Cont
     assertIcannReportingActivityFieldLogged("srs-cont-delete");
   }
 
+  @TestOfyOnly
+  void testModification_duringNoAsyncPhase() throws Exception {
+    persistActiveContact(getUniqueIdFromCommand());
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   @TestOfyOnly
   void testModification_duringReadOnlyPhase() throws Exception {
     persistActiveContact(getUniqueIdFromCommand());

core/src/test/java/google/registry/flows/domain/DomainCreateFlowTest.java

@@ -840,6 +840,15 @@ class DomainCreateFlowTest extends ResourceFlowTestCase<DomainCreateFlow, Domain
     doSuccessfulTest();
   }
 
+  @TestOfyOnly
+  void testSuccess_inNoAsyncPhase() throws Exception {
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    persistContactsAndHosts();
+    runFlowAssertResponse(
+        loadFile("domain_create_response_noasync.xml", ImmutableMap.of("DOMAIN", "example.tld")));
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   @TestOfyAndSql
   void testSuccess_maxNumberOfNameservers() throws Exception {
     setEppInput("domain_create_13_nameservers.xml");

core/src/test/java/google/registry/flows/host/HostDeleteFlowTest.java

@@ -358,6 +358,15 @@ class HostDeleteFlowTest extends ResourceFlowTestCase<HostDeleteFlow, HostResour
     DatabaseHelper.removeDatabaseMigrationSchedule();
   }
 
+  @TestOfyOnly
+  void testModification_duringNoAsyncPhase() {
+    persistActiveHost("ns1.example.tld");
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
   private void assertOfyDeleteSuccess(String registrarId, String clientTrid, boolean isSuperuser)
       throws Exception {
     HostResource deletedHost = reloadResourceByForeignKey();

core/src/test/java/google/registry/flows/host/HostUpdateFlowTest.java

@@ -1355,7 +1355,43 @@ class HostUpdateFlowTest extends ResourceFlowTestCase<HostUpdateFlow, HostResour
   }
 
   @TestOfyOnly
-  void testModification_duringReadOnlyPhase() throws Exception {
+  void testSuccess_nonHostRename_inNoAsyncPhase_succeeds() throws Exception {
+    setEppInput("host_update_name_unchanged.xml");
+    createTld("tld");
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    DomainBase domain = persistActiveDomain("example.tld");
+    HostResource oldHost = persistActiveSubordinateHost(oldHostName(), domain);
+    clock.advanceOneMilli();
+    runFlowAssertResponse(loadFile("generic_success_response.xml"));
+    // The example xml doesn't do a host rename, so reloading the host should work.
+    assertAboutHosts()
+        .that(reloadResourceByForeignKey())
+        .hasLastSuperordinateChange(oldHost.getLastSuperordinateChange())
+        .and()
+        .hasSuperordinateDomain(domain.createVKey())
+        .and()
+        .hasPersistedCurrentSponsorRegistrarId("TheRegistrar")
+        .and()
+        .hasLastTransferTime(null)
+        .and()
+        .hasOnlyOneHistoryEntryWhich()
+        .hasType(HistoryEntry.Type.HOST_UPDATE);
+    assertDnsTasksEnqueued("ns1.example.tld");
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
+  @TestOfyOnly
+  void testRename_duringNoAsyncPhase_fails() throws Exception {
+    createTld("tld");
+    persistActiveSubordinateHost(oldHostName(), persistActiveDomain("example.tld"));
+    DatabaseHelper.setMigrationScheduleToDatastorePrimaryNoAsync(clock);
+    EppException thrown = assertThrows(ReadOnlyModeEppException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+    DatabaseHelper.removeDatabaseMigrationSchedule();
+  }
+
+  @TestOfyOnly
+  void testModification_duringReadOnlyPhase_fails() throws Exception {
     createTld("tld");
     persistActiveSubordinateHost(oldHostName(), persistActiveDomain("example.tld"));
     DatabaseHelper.setMigrationScheduleToDatastorePrimaryReadOnly(clock);

core/src/test/java/google/registry/model/common/DatabaseMigrationStateScheduleTest.java

@@ -17,6 +17,7 @@ package google.registry.model.common;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_ONLY;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY;
+import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_NO_ASYNC;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.DATASTORE_PRIMARY_READ_ONLY;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_ONLY;
 import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState.SQL_PRIMARY;
@@ -71,10 +72,12 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
     runValidTransition(DATASTORE_ONLY, DATASTORE_PRIMARY);
 
     runValidTransition(DATASTORE_PRIMARY, DATASTORE_ONLY);
-    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
+    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC);
+    runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY);
 
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_ONLY);
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY);
+    runValidTransition(DATASTORE_PRIMARY_READ_ONLY, DATASTORE_PRIMARY_NO_ASYNC);
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY);
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY);
 
@@ -94,6 +97,7 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
     runInvalidTransition(DATASTORE_ONLY, SQL_PRIMARY);
     runInvalidTransition(DATASTORE_ONLY, SQL_ONLY);
 
+    runInvalidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
     runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY_READ_ONLY);
     runInvalidTransition(DATASTORE_PRIMARY, SQL_PRIMARY);
     runInvalidTransition(DATASTORE_PRIMARY, SQL_ONLY);
@@ -124,7 +128,8 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
         ImmutableSortedMap.<DateTime, MigrationState>naturalOrder()
             .put(START_OF_TIME, DATASTORE_ONLY)
             .put(startTime.plusHours(1), DATASTORE_PRIMARY)
-            .put(startTime.plusHours(2), DATASTORE_PRIMARY_READ_ONLY)
+            .put(startTime.plusHours(2), DATASTORE_PRIMARY_NO_ASYNC)
+            .put(startTime.plusHours(3), DATASTORE_PRIMARY_READ_ONLY)
             .build();
     assertThat(
             assertThrows(
@@ -163,7 +168,8 @@ public class DatabaseMigrationStateScheduleTest extends EntityTestCase {
     fakeClock.setTo(START_OF_TIME.plusDays(1));
     AllocationToken token =
         new AllocationToken.Builder().setToken("token").setTokenType(TokenType.SINGLE_USE).build();
-    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_READ_ONLY);
+    runValidTransition(DATASTORE_PRIMARY, DATASTORE_PRIMARY_NO_ASYNC);
+    runValidTransition(DATASTORE_PRIMARY_NO_ASYNC, DATASTORE_PRIMARY_READ_ONLY);
     assertThrows(ReadOnlyModeException.class, () -> persistResource(token));
     runValidTransition(DATASTORE_PRIMARY_READ_ONLY, SQL_PRIMARY_READ_ONLY);
     assertThrows(ReadOnlyModeException.class, () -> persistResource(token));

core/src/test/java/google/registry/model/replay/ReplicateToDatastoreActionTest.java

@@ -44,6 +44,7 @@ import google.registry.model.server.Lock;
 import google.registry.persistence.VKey;
 import google.registry.persistence.transaction.JpaTransactionManagerImpl;
 import google.registry.persistence.transaction.TransactionEntity;
+import google.registry.persistence.transaction.TransactionManagerFactory;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.DatabaseHelper;
 import google.registry.testing.FakeClock;
@@ -52,6 +53,7 @@ import google.registry.testing.InjectExtension;
 import google.registry.testing.ReplayExtension;
 import google.registry.testing.TestObject;
 import google.registry.util.RequestStatusChecker;
+import google.registry.util.SystemClock;
 import java.lang.reflect.Proxy;
 import java.util.List;
 import java.util.logging.Level;
@@ -105,6 +107,7 @@ public class ReplicateToDatastoreActionTest {
   void tearDown() {
     DatabaseHelper.removeDatabaseMigrationSchedule();
     fakeClock.disableAutoIncrement();
+    TransactionManagerFactory.setClockForTesting(new SystemClock());
   }
 
   @RetryingTest(4)
@@ -366,8 +369,9 @@ public class ReplicateToDatastoreActionTest {
                     ImmutableSortedMap.<DateTime, MigrationState>naturalOrder()
                         .put(START_OF_TIME, MigrationState.DATASTORE_ONLY)
                         .put(START_OF_TIME.plusHours(1), MigrationState.DATASTORE_PRIMARY)
-                        .put(START_OF_TIME.plusHours(2), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
-                        .put(START_OF_TIME.plusHours(3), MigrationState.SQL_PRIMARY)
+                        .put(START_OF_TIME.plusHours(2), MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
+                        .put(START_OF_TIME.plusHours(3), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusHours(4), MigrationState.SQL_PRIMARY)
                         .put(now.plusHours(1), MigrationState.SQL_PRIMARY_READ_ONLY)
                         .put(now.plusHours(2), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
                         .put(now.plusHours(3), MigrationState.DATASTORE_PRIMARY)
@@ -391,6 +395,50 @@ public class ReplicateToDatastoreActionTest {
                 + " DATASTORE_PRIMARY.");
   }
 
+  @Test
+  void replicationWorksInReadOnly() {
+
+    // Put us in SQL primary now, readonly in an hour, then in datastore primary after 25 hours.
+    // And we'll need the TransactionManagerFactory to use the fake clock.
+    DateTime now = fakeClock.nowUtc();
+    TransactionManagerFactory.setClockForTesting(fakeClock);
+    jpaTm()
+        .transact(
+            () ->
+                DatabaseMigrationStateSchedule.set(
+                    ImmutableSortedMap.<DateTime, MigrationState>naturalOrder()
+                        .put(START_OF_TIME, MigrationState.DATASTORE_ONLY)
+                        .put(START_OF_TIME.plusHours(1), MigrationState.DATASTORE_PRIMARY)
+                        .put(START_OF_TIME.plusHours(2), MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
+                        .put(START_OF_TIME.plusHours(3), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusHours(4), MigrationState.SQL_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusHours(5), MigrationState.SQL_PRIMARY)
+                        .put(now.plusHours(1), MigrationState.SQL_PRIMARY_READ_ONLY)
+                        .put(now.plusHours(25), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .put(now.plusHours(26), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .build()));
+
+    TestObject foo = TestObject.create("foo");
+    insertInDb(foo);
+    TestObject bar = TestObject.create("bar");
+    insertInDb(bar);
+    TestObject baz = TestObject.create("baz");
+    insertInDb(baz);
+    jpaTm().transact(() -> jpaTm().delete(baz.key()));
+
+    // get to read-only
+    fakeClock.advanceBy(Duration.standardDays(1));
+
+    // process the transaction in readonly.
+    action.run();
+
+    // Forward the next day (datastore primary).  Verify that datastore has all of the changes.
+    fakeClock.advanceBy(Duration.standardDays(1));
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(foo.key()))).isEqualTo(foo);
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKey(bar.key()))).isEqualTo(bar);
+    assertThat(ofyTm().transact(() -> ofyTm().loadByKeyIfPresent(baz.key()).isPresent())).isFalse();
+  }
+
   @Test
   void testFailure_cannotAcquireLock() {
     assumeTrue(ReplayExtension.replayTestsEnabled());

core/src/test/java/google/registry/persistence/converter/DatabaseMigrationScheduleTransitionConverterTest.java

@@ -47,7 +47,9 @@ public class DatabaseMigrationScheduleTransitionConverterTest {
           MigrationState.DATASTORE_ONLY,
           DateTime.parse("2001-01-01T00:00:00.0Z"),
           MigrationState.DATASTORE_PRIMARY,
-          DateTime.parse("2002-01-01T00:00:00.0Z"),
+          DateTime.parse("2002-01-01T01:00:00.0Z"),
+          MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
+          DateTime.parse("2002-01-01T02:00:00.0Z"),
           MigrationState.DATASTORE_PRIMARY_READ_ONLY,
           DateTime.parse("2002-01-02T00:00:00.0Z"),
           MigrationState.SQL_PRIMARY,

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -1429,6 +1429,33 @@ public class DatabaseHelper {
     return entity;
   }
 
+  /**
+   * Sets a DATASTORE_PRIMARY_NO_ASYNC state on the {@link DatabaseMigrationStateSchedule}.
+   *
+   * <p>In order to allow for tests to manipulate the clock how they need, we start the transitions
+   * one millisecond after the clock's current time (in case the clock's current value is
+   * START_OF_TIME). We then advance the clock one second so that we're in the
+   * DATASTORE_PRIMARY_READ_ONLY phase.
+   *
+   * <p>We must use the current time, otherwise the setting of the migration state will fail due to
+   * an invalid transition.
+   */
+  public static void setMigrationScheduleToDatastorePrimaryNoAsync(FakeClock fakeClock) {
+    DateTime now = fakeClock.nowUtc();
+    jpaTm()
+        .transact(
+            () ->
+                DatabaseMigrationStateSchedule.set(
+                    ImmutableSortedMap.of(
+                        START_OF_TIME,
+                        MigrationState.DATASTORE_ONLY,
+                        now.plusMillis(1),
+                        MigrationState.DATASTORE_PRIMARY,
+                        now.plusMillis(2),
+                        MigrationState.DATASTORE_PRIMARY_NO_ASYNC)));
+    fakeClock.advanceBy(Duration.standardSeconds(1));
+  }
+
   /**
    * Sets a DATASTORE_PRIMARY_READ_ONLY state on the {@link DatabaseMigrationStateSchedule}.
    *
@@ -1452,6 +1479,8 @@ public class DatabaseHelper {
                         now.plusMillis(1),
                         MigrationState.DATASTORE_PRIMARY,
                         now.plusMillis(2),
+                        MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
+                        now.plusMillis(3),
                         MigrationState.DATASTORE_PRIMARY_READ_ONLY)));
     fakeClock.advanceBy(Duration.standardSeconds(1));
   }
@@ -1478,8 +1507,10 @@ public class DatabaseHelper {
                         now.plusMillis(1),
                         MigrationState.DATASTORE_PRIMARY,
                         now.plusMillis(2),
-                        MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                        MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                         now.plusMillis(3),
+                        MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                        now.plusMillis(4),
                         MigrationState.SQL_PRIMARY)));
     fakeClock.advanceBy(Duration.standardSeconds(1));
   }

core/src/test/java/google/registry/tools/CheckDomainClaimsCommandTest.java

@@ -37,6 +37,12 @@ class CheckDomainClaimsCommandTest extends EppToolCommandTestCase<CheckDomainCla
     eppVerifier.expectDryRun().verifySent("domain_check_claims.xml");
   }
 
+  @Test
+  void testSuccess_canonicalizesInput() throws Exception {
+    runCommand("--client=NewRegistrar", "exaMPle.TLD");
+    eppVerifier.expectDryRun().verifySent("domain_check_claims.xml");
+  }
+
   @Test
   void testSuccess_multipleTlds() throws Exception {
     runCommand("--client=NewRegistrar", "example.tld", "example.tld2");

core/src/test/java/google/registry/tools/CreateHostCommandTest.java

@@ -29,7 +29,7 @@ class CreateHostCommandTest extends EppToolCommandTestCase<CreateHostCommand> {
     createTld("tld");
     runCommandForced(
         "--client=NewRegistrar",
-        "--host=example.tld",
+        "--host=example.TLD",
         "--addresses=162.100.102.99,2001:0db8:85a3:0000:0000:8a2e:0370:7334,4.5.6.7");
     eppVerifier.verifySent("host_create_complete.xml");
   }

core/src/test/java/google/registry/tools/EnqueuePollMessageCommandTest.java

@@ -55,7 +55,8 @@ class EnqueuePollMessageCommandTest extends CommandTestCase<EnqueuePollMessageCo
 
   @TestOfyAndSql
   void testSuccess_domainAndMessage() throws Exception {
-    runCommandForced("--domain=example.tld", "--message=This domain is bad");
+    // Test canonicalization to lowercase example.tld as well.
+    runCommandForced("--domain=EXAMPLE.TLD", "--message=This domain is bad");
 
     HistoryEntry synthetic = getOnlyHistoryEntryOfType(domain, SYNTHETIC);
     assertAboutHistoryEntries()

core/src/test/java/google/registry/tools/GenerateAllocationTokensCommandTest.java

@@ -18,7 +18,7 @@ import static com.google.common.truth.Truth.assertThat;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
 import static google.registry.testing.DatabaseHelper.assertAllocationTokens;
-import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.createTlds;
 import static google.registry.testing.DatabaseHelper.loadAllOf;
 import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -138,16 +138,18 @@ class GenerateAllocationTokensCommandTest extends CommandTestCase<GenerateAlloca
 
   @TestOfyAndSql
   void testSuccess_domainNames() throws Exception {
-    createTld("tld");
+    createTlds("tld", "xn--q9jyb4c");
     File domainNamesFile = tmpDir.resolve("domain_names.txt").toFile();
-    Files.asCharSink(domainNamesFile, UTF_8).write("foo1.tld\nboo2.tld\nbaz9.tld\n");
+    Files.asCharSink(domainNamesFile, UTF_8).write("foo1.tld\nboo2.tld\nçauçalito.みんな\n");
     runCommand("--domain_names_file", domainNamesFile.getPath());
     assertAllocationTokens(
         createToken("123456789ABCDEFG", null, "foo1.tld"),
         createToken("HJKLMNPQRSTUVWXY", null, "boo2.tld"),
-        createToken("Zabcdefghijkmnop", null, "baz9.tld"));
+        createToken("Zabcdefghijkmnop", null, "xn--aualito-txac.xn--q9jyb4c"));
     assertInStdout(
-        "foo1.tld,123456789ABCDEFG\nboo2.tld,HJKLMNPQRSTUVWXY\nbaz9.tld,Zabcdefghijkmnop");
+        "foo1.tld,123456789ABCDEFG",
+        "boo2.tld,HJKLMNPQRSTUVWXY",
+        "xn--aualito-txac.xn--q9jyb4c,Zabcdefghijkmnop");
   }
 
   @TestOfyAndSql

core/src/test/java/google/registry/tools/GetDatabaseMigrationStateCommandTest.java

@@ -54,10 +54,12 @@ public class GetDatabaseMigrationStateCommandTest
             now.plusHours(1),
             MigrationState.DATASTORE_PRIMARY,
             now.plusHours(2),
-            MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+            MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
             now.plusHours(3),
-            MigrationState.SQL_PRIMARY,
+            MigrationState.DATASTORE_PRIMARY_READ_ONLY,
             now.plusHours(4),
+            MigrationState.SQL_PRIMARY,
+            now.plusHours(5),
             MigrationState.SQL_ONLY);
     jpaTm().transact(() -> DatabaseMigrationStateSchedule.set(transitions));
     runCommand();

core/src/test/java/google/registry/tools/GetDomainCommandTest.java

@@ -64,6 +64,15 @@ class GetDomainCommandTest extends CommandTestCase<GetDomainCommand> {
     assertNotInStdout("LiveRef");
   }
 
+  @Test
+  void testSuccess_canonicalizeDomainName() throws Exception {
+    createTld("xn--q9jyb4c");
+    persistActiveDomain("xn--aualito-txac.xn--q9jyb4c");
+    runCommand("çauçalito.みんな", "--expand");
+    assertInStdout("fullyQualifiedDomainName=xn--aualito-txac.xn--q9jyb4c");
+    assertInStdout("contactId=contact1234");
+  }
+
   @Test
   void testSuccess_multipleArguments() throws Exception {
     persistActiveDomain("example.tld");

core/src/test/java/google/registry/tools/SetDatabaseMigrationStateCommandTest.java

@@ -64,14 +64,21 @@ public class SetDatabaseMigrationStateCommandTest
   void testSuccess_fullSchedule() throws Exception {
     DateTime now = fakeClock.nowUtc();
     DateTime datastorePrimary = now.plusHours(1);
-    DateTime datastorePrimaryReadOnly = now.plusHours(2);
-    DateTime sqlPrimary = now.plusHours(3);
-    DateTime sqlOnly = now.plusHours(4);
+    DateTime datastorePrimaryNoAsync = now.plusHours(2);
+    DateTime datastorePrimaryReadOnly = now.plusHours(3);
+    DateTime sqlPrimary = now.plusHours(4);
+    DateTime sqlOnly = now.plusHours(5);
     runCommandForced(
         String.format(
             "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
-                + "%s=DATASTORE_PRIMARY_READ_ONLY,%s=SQL_PRIMARY,%s=SQL_ONLY",
-            START_OF_TIME, datastorePrimary, datastorePrimaryReadOnly, sqlPrimary, sqlOnly));
+                + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY,"
+                + "%s=SQL_PRIMARY,%s=SQL_ONLY",
+            START_OF_TIME,
+            datastorePrimary,
+            datastorePrimaryNoAsync,
+            datastorePrimaryReadOnly,
+            sqlPrimary,
+            sqlOnly));
     assertThat(DatabaseMigrationStateSchedule.get().toValueMap())
         .containsExactlyEntriesIn(
             ImmutableSortedMap.of(
@@ -79,6 +86,8 @@ public class SetDatabaseMigrationStateCommandTest
                 MigrationState.DATASTORE_ONLY,
                 datastorePrimary,
                 MigrationState.DATASTORE_PRIMARY,
+                datastorePrimaryNoAsync,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                 datastorePrimaryReadOnly,
                 MigrationState.DATASTORE_PRIMARY_READ_ONLY,
                 sqlPrimary,
@@ -110,8 +119,9 @@ public class SetDatabaseMigrationStateCommandTest
     runCommandForced(
         String.format(
             "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
-                + "%s=DATASTORE_PRIMARY_READ_ONLY,%s=DATASTORE_PRIMARY",
-            START_OF_TIME, now.plusHours(1), now.plusHours(2), now.plusHours(3)));
+                + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY,"
+                + "%s=DATASTORE_PRIMARY",
+            START_OF_TIME, now.plusHours(1), now.plusHours(2), now.plusHours(3), now.plusHours(4)));
     assertThat(DatabaseMigrationStateSchedule.get().toValueMap())
         .containsExactlyEntriesIn(
             ImmutableSortedMap.of(
@@ -120,8 +130,10 @@ public class SetDatabaseMigrationStateCommandTest
                 now.plusHours(1),
                 MigrationState.DATASTORE_PRIMARY,
                 now.plusHours(2),
-                MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                MigrationState.DATASTORE_PRIMARY_NO_ASYNC,
                 now.plusHours(3),
+                MigrationState.DATASTORE_PRIMARY_READ_ONLY,
+                now.plusHours(4),
                 MigrationState.DATASTORE_PRIMARY));
   }
 
@@ -152,9 +164,12 @@ public class SetDatabaseMigrationStateCommandTest
                 () ->
                     runCommandForced(
                         String.format(
-                            "--migration_schedule=%s=DATASTORE_ONLY,"
-                                + "%s=DATASTORE_PRIMARY,%s=DATASTORE_PRIMARY_READ_ONLY",
-                            START_OF_TIME, now.minusHours(2), now.minusHours(1)))))
+                            "--migration_schedule=%s=DATASTORE_ONLY,%s=DATASTORE_PRIMARY,"
+                                + "%s=DATASTORE_PRIMARY_NO_ASYNC,%s=DATASTORE_PRIMARY_READ_ONLY",
+                            START_OF_TIME,
+                            now.minusHours(3),
+                            now.minusHours(2),
+                            now.minusHours(1)))))
         .hasMessageThat()
         .isEqualTo(
             "Cannot transition from current state-as-of-now DATASTORE_ONLY "

core/src/test/resources/google/registry/flows/domain/domain_create_response_noasync.xml

@@ -0,0 +1,19 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <domain:creData
+       xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>%DOMAIN%</domain:name>
+        <domain:crDate>1999-04-03T22:00:01.0Z</domain:crDate>
+        <domain:exDate>2001-04-03T22:00:01.0Z</domain:exDate>
+      </domain:creData>
+    </resData>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</epp>

db/src/main/resources/sql/er_diagram/brief_er_diagram.html

@@ -261,11 +261,11 @@ td.section {
     </tr> 
     <tr> 
      <td class="property_name">generated on</td> 
-     <td class="property_value">2022-03-30 20:53:20.574091</td> 
+     <td class="property_value">2022-04-01 16:53:26.766163</td> 
     </tr> 
     <tr>
      <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V115__add_renewal_columns_to_billing_recurrence.sql</td>
+     <td id="lastFlywayFile" class="property_value">V116__add_renewal_column_to_allocation_token.sql</td>
     </tr>
    </tbody>
   </table> 
@@ -284,7 +284,7 @@ td.section {
      generated on
     </text> 
     <text text-anchor="start" x="4055.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-     2022-03-30 20:53:20.574091
+     2022-04-01 16:53:26.766163
     </text> 
     <polygon fill="none" stroke="#888888" points="3968,-4 3968,-44 4233,-44 4233,-4 3968,-4" /> <!-- allocationtoken_a08ccbef --> 
     <g id="node1" class="node"> 

db/src/main/resources/sql/flyway.txt

@@ -113,3 +113,4 @@ V112__add_billingrecurrence_missing_indexes.sql
 V113__add_host_missing_indexes.sql
 V114__add_allocation_token_indexes.sql
 V115__add_renewal_columns_to_billing_recurrence.sql
+V116__add_renewal_column_to_allocation_token.sql

db/src/main/resources/sql/flyway/V116__add_renewal_column_to_allocation_token.sql

@@ -0,0 +1,15 @@
+-- Copyright 2022 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+alter table "AllocationToken" add column renewal_price_behavior text default 'DEFAULT' not null;

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -51,7 +51,8 @@ CREATE TABLE public."AllocationToken" (
     redemption_domain_repo_id text,
     token_status_transitions public.hstore,
     token_type text,
-    redemption_domain_history_id bigint
+    redemption_domain_history_id bigint,
+    renewal_price_behavior text DEFAULT 'DEFAULT'::text NOT NULL
 );
 
 

util/src/main/java/google/registry/util/DomainNameUtils.java

@@ -39,8 +39,8 @@ public final class DomainNameUtils {
             .equals(potentialParent.parts());
   }
 
-  /** Canonicalizes a domain name by lowercasing and converting unicode to punycode. */
-  public static String canonicalizeDomainName(String label) {
+  /** Canonicalizes a hostname/domain name by lowercasing and converting unicode to punycode. */
+  public static String canonicalizeHostname(String label) {
     String labelLowercased = Ascii.toLowerCase(label);
     try {
       return Idn.toASCII(labelLowercased);

util/src/test/java/google/registry/util/DomainNameUtilsTest.java

@@ -15,7 +15,7 @@
 package google.registry.util;
 
 import static com.google.common.truth.Truth.assertThat;
-import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
+import static google.registry.util.DomainNameUtils.canonicalizeHostname;
 import static google.registry.util.DomainNameUtils.getSecondLevelDomain;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -25,37 +25,38 @@ import org.junit.jupiter.api.Test;
 class DomainNameUtilsTest {
 
   @Test
-  void testCanonicalizeDomainName_succeeds() {
-    assertThat(canonicalizeDomainName("foo")).isEqualTo("foo");
-    assertThat(canonicalizeDomainName("FOO")).isEqualTo("foo");
-    assertThat(canonicalizeDomainName("foo.tld")).isEqualTo("foo.tld");
-    assertThat(canonicalizeDomainName("xn--q9jyb4c")).isEqualTo("xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("XN--Q9JYB4C")).isEqualTo("xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("みんな")).isEqualTo("xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("みんな.みんな")).isEqualTo("xn--q9jyb4c.xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("みんな.foo")).isEqualTo("xn--q9jyb4c.foo");
-    assertThat(canonicalizeDomainName("foo.みんな")).isEqualTo("foo.xn--q9jyb4c");
-    assertThat(canonicalizeDomainName("ħ")).isEqualTo("xn--1ea");
+  void testCanonicalizeHostname_succeeds() {
+    assertThat(canonicalizeHostname("foo")).isEqualTo("foo");
+    assertThat(canonicalizeHostname("FOO")).isEqualTo("foo");
+    assertThat(canonicalizeHostname("foo.tld")).isEqualTo("foo.tld");
+    assertThat(canonicalizeHostname("xn--q9jyb4c")).isEqualTo("xn--q9jyb4c");
+    assertThat(canonicalizeHostname("XN--Q9JYB4C")).isEqualTo("xn--q9jyb4c");
+    assertThat(canonicalizeHostname("みんな")).isEqualTo("xn--q9jyb4c");
+    assertThat(canonicalizeHostname("みんな.みんな")).isEqualTo("xn--q9jyb4c.xn--q9jyb4c");
+    assertThat(canonicalizeHostname("みんな.foo")).isEqualTo("xn--q9jyb4c.foo");
+    assertThat(canonicalizeHostname("foo.みんな")).isEqualTo("foo.xn--q9jyb4c");
+    assertThat(canonicalizeHostname("BAR.foo.みんな")).isEqualTo("bar.foo.xn--q9jyb4c");
+    assertThat(canonicalizeHostname("ħ")).isEqualTo("xn--1ea");
   }
 
   @Test
-  void testCanonicalizeDomainName_allowsRdnsNames() {
-    assertThat(canonicalizeDomainName("119.63.227.45-ns1.jhz-tt.uk"))
+  void testCanonicalizeHostname_allowsRdnsNames() {
+    assertThat(canonicalizeHostname("119.63.227.45-ns1.jhz-tt.uk"))
         .isEqualTo("119.63.227.45-ns1.jhz-tt.uk");
   }
 
   @Test
-  void testCanonicalizeDomainName_throwsOn34HyphenRule() {
+  void testCanonicalizeHostname_throwsOn34HyphenRule() {
     IllegalArgumentException thrown =
         assertThrows(
             IllegalArgumentException.class,
-            () -> canonicalizeDomainName("119.63.227.45--ns1.jhz-tt.uk"));
+            () -> canonicalizeHostname("119.63.227.45--ns1.jhz-tt.uk"));
     assertThat(thrown).hasCauseThat().hasMessageThat().contains("HYPHEN_3_4");
   }
 
   @Test
-  void testCanonicalizeDomainName_acePrefixUnicodeChars() {
-    assertThrows(IllegalArgumentException.class, () -> canonicalizeDomainName("xn--みんな"));
+  void testCanonicalizeHostname_acePrefixUnicodeChars() {
+    assertThrows(IllegalArgumentException.class, () -> canonicalizeHostname("xn--みんな"));
   }
 
   @Test