Add SQL-based DNS refresh processing mechanism (#1971)

This includes changes to make sure that we use the proper per-TLD IDN
tables as well as setting/updating/removing them via the Create/Update
TLD commands.

core/src/main/java/google/registry/config/RegistryConfig.java

@@ -32,6 +32,8 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import dagger.Module;
 import dagger.Provides;
+import google.registry.dns.ReadDnsRefreshRequestsAction;
+import google.registry.model.common.DnsRefreshRequest;
 import google.registry.persistence.transaction.JpaTransactionManager;
 import google.registry.util.YamlUtils;
 import java.lang.annotation.Documented;
@@ -294,9 +296,9 @@ public final class RegistryConfig {
 
     /**
      * The maximum number of domain and host updates to batch together to send to
-     * PublishDnsUpdatesAction, to avoid exceeding AppEngine's limits.
+     * PublishDnsUpdatesAction, to avoid exceeding HTTP request timeout limits.
      *
-     * @see google.registry.dns.ReadDnsQueueAction
+     * @see google.registry.dns.ReadDnsRefreshRequestsAction
      */
     @Provides
     @Config("dnsTldUpdateBatchSize")
@@ -327,23 +329,30 @@ public final class RegistryConfig {
     }
 
     /**
-     * The requested maximum duration for ReadDnsQueueAction.
+     * The requested maximum duration for {@link ReadDnsRefreshRequestsAction}.
      *
-     * <p>ReadDnsQueueAction reads update tasks from the dns-pull queue. It will continue reading
-     * tasks until either the queue is empty, or this duration has passed.
+     * <p>{@link ReadDnsRefreshRequestsAction} reads refresh requests from {@link DnsRefreshRequest}
+     * It will continue reading requests until either no requests exist that matche the condition,
+     * or this duration has passed.
      *
-     * <p>This time is the maximum duration between the first and last attempt to lease tasks from
-     * the dns-pull queue. The actual running time might be slightly longer, as we process the
-     * tasks.
+     * <p>This time is the maximum duration between the first and last attempt to read requests from
+     * {@link DnsRefreshRequest}. The actual running time might be slightly longer, as we process
+     * the requests.
      *
-     * <p>This value should be less than the cron-job repeat rate for ReadDnsQueueAction, to make
-     * sure we don't have multiple ReadDnsActions leasing tasks simultaneously.
+     * <p>The requests that are read will not be read again by any action until after this period
+     * has passed, so concurrent runs (or runs that are very close to each other) of {@link
+     * ReadDnsRefreshRequestsAction} will not keep reading the same requests with the earliest
+     * request time.
      *
-     * @see google.registry.dns.ReadDnsQueueAction
+     * <p>Still, this value should ideally be less than the cloud scheduler job repeat rate for
+     * {@link ReadDnsRefreshRequestsAction}, to not waste resources on multiple actions running at
+     * the same time.
+     *
+     * <p>see google.registry.dns.ReadDnsRefreshRequestsAction
      */
     @Provides
-    @Config("readDnsQueueActionRuntime")
-    public static Duration provideReadDnsQueueRuntime() {
+    @Config("readDnsRefreshRequestsActionRuntime")
+    public static Duration provideReadDnsRefreshRequestsRuntime() {
       return Duration.standardSeconds(45);
     }
 

core/src/main/java/google/registry/dns/DnsModule.java

@@ -19,6 +19,7 @@ import static google.registry.dns.DnsConstants.DNS_PULL_QUEUE_NAME;
 import static google.registry.dns.RefreshDnsOnHostRenameAction.PARAM_HOST_KEY;
 import static google.registry.request.RequestParameters.extractEnumParameter;
 import static google.registry.request.RequestParameters.extractIntParameter;
+import static google.registry.request.RequestParameters.extractOptionalParameter;
 import static google.registry.request.RequestParameters.extractRequiredParameter;
 import static google.registry.request.RequestParameters.extractSetOfParameters;
 
@@ -48,7 +49,7 @@ public abstract class DnsModule {
   public static final String PARAM_DOMAINS = "domains";
   public static final String PARAM_HOSTS = "hosts";
   public static final String PARAM_PUBLISH_TASK_ENQUEUED = "enqueued";
-  public static final String PARAM_REFRESH_REQUEST_CREATED = "itemsCreated";
+  public static final String PARAM_REFRESH_REQUEST_TIME = "requestTime";
 
   @Binds
   @DnsWriterZone
@@ -83,10 +84,13 @@ public abstract class DnsModule {
     return DateTime.parse(extractRequiredParameter(req, PARAM_PUBLISH_TASK_ENQUEUED));
   }
 
+  // TODO: Retire the old header after DNS pull queue migration.
   @Provides
-  @Parameter(PARAM_REFRESH_REQUEST_CREATED)
+  @Parameter(PARAM_REFRESH_REQUEST_TIME)
   static DateTime provideItemsCreateTime(HttpServletRequest req) {
-    return DateTime.parse(extractRequiredParameter(req, PARAM_REFRESH_REQUEST_CREATED));
+    return DateTime.parse(
+        extractOptionalParameter(req, "itemsCreated")
+            .orElse(extractRequiredParameter(req, PARAM_REFRESH_REQUEST_TIME)));
   }
 
   @Provides

core/src/main/java/google/registry/dns/DnsQueue.java

@@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import com.google.common.net.InternetDomainName;
 import com.google.common.util.concurrent.RateLimiter;
+import google.registry.config.RegistryConfig;
 import google.registry.dns.DnsConstants.TargetType;
 import google.registry.model.tld.Registries;
 import google.registry.util.Clock;
@@ -54,13 +55,13 @@ import org.joda.time.Duration;
  * <p>This includes a {@link RateLimiter} to limit the {@link Queue#leaseTasks} call rate to 9 QPS,
  * to stay under the 10 QPS limit for this function.
  *
- * <p>Note that overlapping calls to {@link ReadDnsQueueAction} (the only place where
- * {@link DnsQueue#leaseTasks} is used) will have different rate limiters, so they could exceed the
- * allowed rate. This should be rare though - because {@link DnsQueue#leaseTasks} is only used in
- * {@link ReadDnsQueueAction}, which is run as a cron job with running time shorter than the cron
- * repeat time - meaning there should never be two instances running at once.
+ * <p>Note that overlapping calls to {@link ReadDnsQueueAction} (the only place where {@link
+ * DnsQueue#leaseTasks} is used) will have different rate limiters, so they could exceed the allowed
+ * rate. This should be rare though - because {@link DnsQueue#leaseTasks} is only used in {@link
+ * ReadDnsQueueAction}, which is run as a cron job with running time shorter than the cron repeat
+ * time - meaning there should never be two instances running at once.
  *
- * @see google.registry.config.RegistryConfig.ConfigModule#provideReadDnsQueueRuntime
+ * @see RegistryConfig.ConfigModule#provideReadDnsRefreshRequestsRuntime()
  */
 public class DnsQueue {
 

core/src/main/java/google/registry/dns/DnsUtils.java

@@ -14,15 +14,19 @@
 
 package google.registry.dns;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.net.InternetDomainName;
 import google.registry.dns.DnsConstants.TargetType;
 import google.registry.model.common.DatabaseMigrationStateSchedule;
 import google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
 import google.registry.model.common.DnsRefreshRequest;
 import google.registry.model.tld.Registries;
+import java.util.Collection;
 import javax.inject.Inject;
+import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /** Utility class to handle DNS refresh requests. */
@@ -67,6 +71,61 @@ public class DnsUtils {
     requestDnsRefresh(hostName, TargetType.HOST, Duration.ZERO);
   }
 
+  /**
+   * Returns pending DNS update requests that need further processing up to batch size, in ascending
+   * order of their request time, and updates their processing time to now.
+   *
+   * <p>The criteria to pick the requests to include are:
+   *
+   * <ul>
+   *   <li>They are for the given TLD.
+   *   <li>Their request time is not in the future.
+   *   <li>The last time they were processed is before the cooldown period.
+   * </ul>
+   */
+  public ImmutableList<DnsRefreshRequest> readAndUpdateRequestsWithLatestProcessTime(
+      String tld, Duration cooldown, int batchSize) {
+    return tm().transact(
+            () -> {
+              DateTime transactionTime = tm().getTransactionTime();
+              ImmutableList<DnsRefreshRequest> requests =
+                  tm().query(
+                          "FROM DnsRefreshRequest WHERE tld = :tld "
+                              + "AND requestTime <= :now AND lastProcessTime < :cutoffTime "
+                              + "ORDER BY requestTime ASC, id ASC",
+                          DnsRefreshRequest.class)
+                      .setParameter("tld", tld)
+                      .setParameter("now", transactionTime)
+                      .setParameter("cutoffTime", transactionTime.minus(cooldown))
+                      .setMaxResults(batchSize)
+                      .getResultStream()
+                      // Note that the process time is when the request was last read, batched and
+                      // queued up for publishing, not when it is actually published by the DNS
+                      // writer. This timestamp acts as a cooldown so the same request will not be
+                      // retried too frequently. See DnsRefreshRequest.getLastProcessTime for a
+                      // detailed explaination.
+                      .map(e -> e.updateProcessTime(transactionTime))
+                      .collect(toImmutableList());
+              tm().updateAll(requests);
+              return requests;
+            });
+  }
+
+  /**
+   * Removes the requests that have been processed.
+   *
+   * <p>Note that if a request entity has already been deleted, the method still succeeds without
+   * error because all we care about is that it no longer exists after the method runs.
+   */
+  public void deleteRequests(Collection<DnsRefreshRequest> requests) {
+    tm().transact(
+            () ->
+                tm().delete(
+                        requests.stream()
+                            .map(DnsRefreshRequest::createVKey)
+                            .collect(toImmutableList())));
+  }
+
   private boolean usePullQueue() {
     return !DatabaseMigrationStateSchedule.getValueAtTime(dnsQueue.getClock().nowUtc())
         .equals(MigrationState.DNS_SQL);

core/src/main/java/google/registry/dns/PublishDnsUpdatesAction.java

@@ -22,7 +22,7 @@ import static google.registry.dns.DnsModule.PARAM_HOSTS;
 import static google.registry.dns.DnsModule.PARAM_LOCK_INDEX;
 import static google.registry.dns.DnsModule.PARAM_NUM_PUBLISH_LOCKS;
 import static google.registry.dns.DnsModule.PARAM_PUBLISH_TASK_ENQUEUED;
-import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_CREATED;
+import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_TIME;
 import static google.registry.model.EppResourceUtils.loadByForeignKey;
 import static google.registry.request.Action.Method.POST;
 import static google.registry.request.RequestParameters.PARAM_TLD;
@@ -124,7 +124,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
   public PublishDnsUpdatesAction(
       @Parameter(PARAM_DNS_WRITER) String dnsWriter,
       @Parameter(PARAM_PUBLISH_TASK_ENQUEUED) DateTime enqueuedTime,
-      @Parameter(PARAM_REFRESH_REQUEST_CREATED) DateTime itemsCreateTime,
+      @Parameter(PARAM_REFRESH_REQUEST_TIME) DateTime itemsCreateTime,
       @Parameter(PARAM_LOCK_INDEX) int lockIndex,
       @Parameter(PARAM_NUM_PUBLISH_LOCKS) int numPublishLocks,
       @Parameter(PARAM_DOMAINS) Set<String> domains,
@@ -346,7 +346,7 @@ public final class PublishDnsUpdatesAction implements Runnable, Callable<Void> {
                 .put(PARAM_LOCK_INDEX, Integer.toString(lockIndex))
                 .put(PARAM_NUM_PUBLISH_LOCKS, Integer.toString(numPublishLocks))
                 .put(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-                .put(PARAM_REFRESH_REQUEST_CREATED, itemsCreateTime.toString())
+                .put(PARAM_REFRESH_REQUEST_TIME, itemsCreateTime.toString())
                 .put(PARAM_DOMAINS, Joiner.on(",").join(domains))
                 .put(PARAM_HOSTS, Joiner.on(",").join(hosts))
                 .build()));

core/src/main/java/google/registry/dns/ReadDnsQueueAction.java

@@ -26,7 +26,7 @@ import static google.registry.dns.DnsModule.PARAM_HOSTS;
 import static google.registry.dns.DnsModule.PARAM_LOCK_INDEX;
 import static google.registry.dns.DnsModule.PARAM_NUM_PUBLISH_LOCKS;
 import static google.registry.dns.DnsModule.PARAM_PUBLISH_TASK_ENQUEUED;
-import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_CREATED;
+import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_TIME;
 import static google.registry.request.RequestParameters.PARAM_TLD;
 import static google.registry.util.DomainNameUtils.getSecondLevelDomain;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -109,7 +109,7 @@ public final class ReadDnsQueueAction implements Runnable {
   @Inject
   ReadDnsQueueAction(
       @Config("dnsTldUpdateBatchSize") int tldUpdateBatchSize,
-      @Config("readDnsQueueActionRuntime") Duration requestedMaximumDuration,
+      @Config("readDnsRefreshRequestsActionRuntime") Duration requestedMaximumDuration,
       @Parameter(PARAM_JITTER_SECONDS) Optional<Integer> jitterSeconds,
       Clock clock,
       DnsQueue dnsQueue,
@@ -379,7 +379,7 @@ public final class ReadDnsQueueAction implements Runnable {
                     .put(PARAM_LOCK_INDEX, Integer.toString(lockIndex))
                     .put(PARAM_NUM_PUBLISH_LOCKS, Integer.toString(numPublishLocks))
                     .put(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-                    .put(PARAM_REFRESH_REQUEST_CREATED, earliestCreateTime.toString())
+                    .put(PARAM_REFRESH_REQUEST_TIME, earliestCreateTime.toString())
                     .put(
                         PARAM_DOMAINS,
                         chunk.stream()

core/src/main/java/google/registry/dns/ReadDnsRefreshRequestsAction.java

@@ -0,0 +1,206 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.dns;
+
+import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap;
+import static google.registry.dns.DnsConstants.DNS_PUBLISH_PUSH_QUEUE_NAME;
+import static google.registry.dns.DnsModule.PARAM_DNS_WRITER;
+import static google.registry.dns.DnsModule.PARAM_DOMAINS;
+import static google.registry.dns.DnsModule.PARAM_HOSTS;
+import static google.registry.dns.DnsModule.PARAM_LOCK_INDEX;
+import static google.registry.dns.DnsModule.PARAM_NUM_PUBLISH_LOCKS;
+import static google.registry.dns.DnsModule.PARAM_PUBLISH_TASK_ENQUEUED;
+import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_TIME;
+import static google.registry.request.RequestParameters.PARAM_TLD;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
+import static google.registry.util.DomainNameUtils.getSecondLevelDomain;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.cloud.tasks.v2.Task;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.flogger.FluentLogger;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import google.registry.batch.CloudTasksUtils;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.dns.DnsConstants.TargetType;
+import google.registry.model.common.DnsRefreshRequest;
+import google.registry.model.tld.Registry;
+import google.registry.request.Action;
+import google.registry.request.Action.Service;
+import google.registry.request.Parameter;
+import google.registry.request.auth.Auth;
+import google.registry.util.Clock;
+import java.util.Collection;
+import java.util.Optional;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+
+/**
+ * Action for fanning out DNS refresh tasks by TLD, using data taken from {@link DnsRefreshRequest}
+ * table.
+ */
+@Action(
+    service = Service.BACKEND,
+    path = "/_dr/task/readDnsRefreshRequests",
+    automaticallyPrintOk = true,
+    auth = Auth.AUTH_INTERNAL_OR_ADMIN)
+public final class ReadDnsRefreshRequestsAction implements Runnable {
+
+  // This parameter cannot be named "jitterSeconds", which will be read by TldFanoutAction and not
+  // be passed to this action.
+  private static final String PARAM_JITTER_SECONDS = "dnsJitterSeconds";
+  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
+
+  private final int tldUpdateBatchSize;
+  private final Duration requestedMaximumDuration;
+  private final Optional<Integer> jitterSeconds;
+  private final String tld;
+  private final Clock clock;
+  private final DnsUtils dnsUtils;
+  private final HashFunction hashFunction;
+  private final CloudTasksUtils cloudTasksUtils;
+
+  @Inject
+  ReadDnsRefreshRequestsAction(
+      @Config("dnsTldUpdateBatchSize") int tldUpdateBatchSize,
+      @Config("readDnsRefreshRequestsActionRuntime") Duration requestedMaximumDuration,
+      @Parameter(PARAM_JITTER_SECONDS) Optional<Integer> jitterSeconds,
+      @Parameter(PARAM_TLD) String tld,
+      Clock clock,
+      DnsUtils dnsUtils,
+      HashFunction hashFunction,
+      CloudTasksUtils cloudTasksUtils) {
+    this.tldUpdateBatchSize = tldUpdateBatchSize;
+    this.requestedMaximumDuration = requestedMaximumDuration;
+    this.jitterSeconds = jitterSeconds;
+    this.tld = tld;
+    this.clock = clock;
+    this.dnsUtils = dnsUtils;
+    this.hashFunction = hashFunction;
+    this.cloudTasksUtils = cloudTasksUtils;
+  }
+
+  /**
+   * Reads requests up to the maximum requested runtime, and enqueues update batches from the these
+   * requests.
+   */
+  @Override
+  public void run() {
+    if (Registry.get(tld).getDnsPaused()) {
+      logger.atInfo().log("The queue updated is paused for TLD: %s.", tld);
+      return;
+    }
+    DateTime requestedEndTime = clock.nowUtc().plus(requestedMaximumDuration);
+    // See getLockIndex(), requests are evenly distributed to [1, numDnsPublishLocks], so each
+    // bucket would be roughly the size of tldUpdateBatchSize.
+    int processBatchSize = tldUpdateBatchSize * Registry.get(tld).getNumDnsPublishLocks();
+    while (requestedEndTime.isAfter(clock.nowUtc())) {
+      ImmutableList<DnsRefreshRequest> requests =
+          dnsUtils.readAndUpdateRequestsWithLatestProcessTime(
+              tld, requestedMaximumDuration, processBatchSize);
+      logger.atInfo().log("Read %d DNS update requests for TLD %s.", requests.size(), tld);
+      if (!requests.isEmpty()) {
+        processRequests(requests);
+      }
+      if (requests.size() < processBatchSize) {
+        return;
+      }
+    }
+  }
+  /**
+   * Subdivides {@link DnsRefreshRequest} into buckets by lock index, enqueue a Cloud Tasks task per
+   * bucket, and then delete the requests in each bucket.
+   */
+  void processRequests(Collection<DnsRefreshRequest> requests) {
+    int numPublishLocks = Registry.get(tld).getNumDnsPublishLocks();
+    requests.stream()
+        .collect(
+            toImmutableSetMultimap(
+                request -> getLockIndex(numPublishLocks, request), request -> request))
+        .asMap()
+        .forEach(
+            (lockIndex, bucketedRequests) -> {
+              try {
+                enqueueUpdates(lockIndex, numPublishLocks, bucketedRequests);
+                dnsUtils.deleteRequests(bucketedRequests);
+                logger.atInfo().log(
+                    "Processed %d DNS update requests for TLD %s.", bucketedRequests.size(), tld);
+              } catch (Exception e) {
+                // Log but continue to process the next bucket. The failed tasks will NOT be
+                // deleted and will be retried after the cooldown period has passed.
+                logger.atSevere().withCause(e).log(
+                    "Error processing DNS update requests: %s", bucketedRequests);
+              }
+            });
+  }
+
+  /**
+   * Returns the lock index for a given {@link DnsRefreshRequest}.
+   *
+   * <p>We hash the second level domain for all records, to group in-bailiwick hosts (the only ones
+   * we refresh DNS for) with their superordinate domains. We use consistent hashing to determine
+   * the lock index because it gives us [0,N) bucketing properties out of the box, then add 1 to
+   * make indexes within [1,N].
+   */
+  int getLockIndex(int numPublishLocks, DnsRefreshRequest request) {
+    String domain = getSecondLevelDomain(request.getName(), tld);
+    return Hashing.consistentHash(hashFunction.hashString(domain, UTF_8), numPublishLocks) + 1;
+  }
+
+  /** Creates DNS refresh tasks for all writers for the tld within a lock index. */
+  void enqueueUpdates(int lockIndex, int numPublishLocks, Collection<DnsRefreshRequest> requests) {
+    ImmutableList.Builder<String> domainsBuilder = new ImmutableList.Builder<>();
+    ImmutableList.Builder<String> hostsBuilder = new ImmutableList.Builder<>();
+    DateTime earliestRequestTime = END_OF_TIME;
+    for (DnsRefreshRequest request : requests) {
+      if (request.getRequestTime().isBefore(earliestRequestTime)) {
+        earliestRequestTime = request.getRequestTime();
+      }
+      String name = request.getName();
+      if (request.getType().equals(TargetType.DOMAIN)) {
+        domainsBuilder.add(name);
+      } else {
+        hostsBuilder.add(name);
+      }
+    }
+    ImmutableList<String> domains = domainsBuilder.build();
+    ImmutableList<String> hosts = hostsBuilder.build();
+    for (String dnsWriter : Registry.get(tld).getDnsWriters()) {
+      Task task =
+          cloudTasksUtils.createPostTaskWithJitter(
+              PublishDnsUpdatesAction.PATH,
+              Service.BACKEND,
+              ImmutableMultimap.<String, String>builder()
+                  .put(PARAM_TLD, tld)
+                  .put(PARAM_DNS_WRITER, dnsWriter)
+                  .put(PARAM_LOCK_INDEX, Integer.toString(lockIndex))
+                  .put(PARAM_NUM_PUBLISH_LOCKS, Integer.toString(numPublishLocks))
+                  .put(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
+                  .put(PARAM_REFRESH_REQUEST_TIME, earliestRequestTime.toString())
+                  .put(PARAM_DOMAINS, Joiner.on(',').join(domains))
+                  .put(PARAM_HOSTS, Joiner.on(',').join(hosts))
+                  .build(),
+              jitterSeconds);
+      cloudTasksUtils.enqueue(DNS_PUBLISH_PUSH_QUEUE_NAME, task);
+      logger.atInfo().log(
+          "Enqueued DNS update request for (TLD %s, lock %d) with %d domains and %d hosts.",
+          tld, lockIndex, domains.size(), hosts.size());
+    }
+  }
+}

core/src/main/java/google/registry/env/alpha/default/WEB-INF/cloud-scheduler-tasks.xml

@@ -137,4 +137,14 @@
     </description>
     <schedule>*/1 * * * *</schedule>
   </task>
+
+  <task>
+    <url>
+      <![CDATA[/_dr/cron/fanout?queue=dns-refresh&forEachRealTld&ForEachTestTld&endpoint=/_dr/task/readDnsRefreshRequests&dnsJitterSeconds=45]]></url>
+    <name>readDnsRefreshRequests</name>
+    <description>
+      Enqueue a ReadDnsRefreshRequestAction for each TLD.
+    </description>
+    <schedule>*/1 * * * *</schedule>
+  </task>
 </taskentries>

core/src/main/java/google/registry/env/common/default/WEB-INF/queue.xml

@@ -6,6 +6,12 @@
     <mode>pull</mode>
   </queue>
 
+  <!-- Queue for reading DNS update requests and batching them off to the dns-publish queue. -->
+  <queue>
+    <name>dns-refresh</name>
+    <rate>100/s</rate>
+  </queue>
+
   <!-- Queue for publishing DNS updates in batches. -->
   <queue>
     <name>dns-publish</name>

core/src/main/java/google/registry/env/crash/default/WEB-INF/cloud-scheduler-tasks.xml

@@ -139,6 +139,16 @@
     <schedule>*/1 * * * *</schedule>
   </task>
 
+  <task>
+    <url>
+      <![CDATA[/_dr/cron/fanout?queue=dns-refresh&forEachRealTld&ForEachTestTld&endpoint=/_dr/task/readDnsRefreshRequests&dnsJitterSeconds=45]]></url>
+    <name>readDnsRefreshRequests</name>
+    <description>
+      Enqueue a ReadDnsRefreshRequestAction for each TLD.
+    </description>
+    <schedule>*/1 * * * *</schedule>
+  </task>
+
   <task>
     <url><![CDATA[/_dr/task/deleteExpiredDomains]]></url>
     <name>deleteExpiredDomains</name>

core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml

@@ -210,6 +210,16 @@
     <schedule>*/1 * * * *</schedule>
   </task>
 
+  <task>
+    <url>
+      <![CDATA[/_dr/cron/fanout?queue=dns-refresh&forEachRealTld&ForEachTestTld&endpoint=/_dr/task/readDnsRefreshRequests&dnsJitterSeconds=45]]></url>
+    <name>readDnsRefreshRequests</name>
+    <description>
+      Enqueue a ReadDnsRefreshRequestAction for each TLD.
+    </description>
+    <schedule>*/1 * * * *</schedule>
+  </task>
+
   <task>
     <url><![CDATA[/_dr/cron/fanout?queue=retryable-cron-tasks&endpoint=/_dr/task/icannReportingStaging&runInEmpty]]></url>
     <name>icannReportingStaging</name>
@@ -270,6 +280,6 @@
       This job runs weekly to wipe out PII fields of ContactHistory entities
       that have been in the database for a certain period of time.
     </description>
-    <schedule>0 15 * * 1</schedule>
+    <schedule>0 15 * 12 1</schedule>
   </task>
 </taskentries>

core/src/main/java/google/registry/env/qa/default/WEB-INF/cloud-scheduler-tasks.xml

@@ -21,6 +21,16 @@
     <schedule>*/1 * * * *</schedule>
   </task>
 
+  <task>
+    <url>
+      <![CDATA[/_dr/cron/fanout?queue=dns-refresh&forEachRealTld&ForEachTestTld&endpoint=/_dr/task/readDnsRefreshRequests&dnsJitterSeconds=45]]></url>
+    <name>readDnsRefreshRequests</name>
+    <description>
+      Enqueue a ReadDnsRefreshRequestAction for each TLD.
+    </description>
+    <schedule>*/1 * * * *</schedule>
+  </task>
+
   <task>
     <url><![CDATA[/_dr/cron/fanout?queue=sheet&endpoint=/_dr/task/syncRegistrarsSheet&runInEmpty]]></url>
     <name>syncRegistrarsSheet</name>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cloud-scheduler-tasks.xml

@@ -152,6 +152,16 @@
     <schedule>*/1 * * * *</schedule>
   </task>
 
+  <task>
+    <url>
+      <![CDATA[/_dr/cron/fanout?queue=dns-refresh&forEachRealTld&ForEachTestTld&endpoint=/_dr/task/readDnsRefreshRequests&dnsJitterSeconds=45]]></url>
+    <name>readDnsRefreshRequests</name>
+    <description>
+      Enqueue a ReadDnsRefreshRequestAction for each TLD.
+    </description>
+    <schedule>*/1 * * * *</schedule>
+  </task>
+
   <task>
     <url><![CDATA[/_dr/task/wipeOutContactHistoryPii]]></url>
     <name>wipeOutContactHistoryPii</name>

core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml

@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--TODO: ptkach - Remove this file after cloud scheduler is fully up and running -->
-<cronentries>
-
-</cronentries>

core/src/main/java/google/registry/flows/domain/DomainCheckFlow.java

@@ -38,7 +38,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
-import com.google.common.flogger.FluentLogger;
 import com.google.common.net.InternetDomainName;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.flows.EppException;
@@ -53,6 +52,11 @@ import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponsePar
 import google.registry.flows.custom.DomainCheckFlowCustomLogic.BeforeResponseReturnData;
 import google.registry.flows.domain.token.AllocationTokenDomainCheckResults;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils;
+import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotInPromotionException;
+import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForCommandException;
+import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForDomainException;
+import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForRegistrarException;
+import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForTldException;
 import google.registry.model.EppResource;
 import google.registry.model.ForeignKeyUtils;
 import google.registry.model.billing.BillingEvent;
@@ -117,7 +121,6 @@ import org.joda.time.DateTime;
 @ReportingSpec(ActivityReportField.DOMAIN_CHECK)
 public final class DomainCheckFlow implements Flow {
 
-  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
   @Inject ResourceCommand resourceCommand;
   @Inject ExtensionManager extensionManager;
   @Inject EppInput eppInput;
@@ -277,22 +280,52 @@ public final class DomainCheckFlow implements Flow {
             DomainFlowUtils.checkForDefaultToken(
                 Registry.get(InternetDomainName.from(domainName).parent().toString()),
                 domainName,
+                feeCheckItem.getCommandName(),
                 registrarId,
                 now);
         FeeCheckResponseExtensionItem.Builder<?> builder = feeCheckItem.createResponseBuilder();
         Optional<Domain> domain = Optional.ofNullable(domainObjs.get(domainName));
-        handleFeeRequest(
-            feeCheckItem,
-            builder,
-            domainNames.get(domainName),
-            domain,
-            feeCheck.getCurrency(),
-            now,
-            pricingLogic,
-            allocationToken.isPresent() ? allocationToken : defaultToken,
-            availableDomains.contains(domainName),
-            recurrences.getOrDefault(domainName, null));
-        responseItems.add(builder.setDomainNameIfSupported(domainName).build());
+        try {
+          if (allocationToken.isPresent()) {
+            AllocationTokenFlowUtils.validateToken(
+                InternetDomainName.from(domainName),
+                allocationToken.get(),
+                feeCheckItem.getCommandName(),
+                registrarId,
+                now);
+          }
+          handleFeeRequest(
+              feeCheckItem,
+              builder,
+              domainNames.get(domainName),
+              domain,
+              feeCheck.getCurrency(),
+              now,
+              pricingLogic,
+              allocationToken.isPresent() ? allocationToken : defaultToken,
+              availableDomains.contains(domainName),
+              recurrences.getOrDefault(domainName, null));
+          responseItems.add(builder.setDomainNameIfSupported(domainName).build());
+        } catch (AllocationTokenNotValidForCommandException
+            | AllocationTokenNotValidForDomainException
+            | AllocationTokenNotValidForRegistrarException
+            | AllocationTokenNotValidForTldException
+            | AllocationTokenNotInPromotionException e) {
+          // Allocation token is either not an active token or it is not valid for the EPP command,
+          // registrar, domain, or TLD.
+          Registry registry = Registry.get(InternetDomainName.from(domainName).parent().toString());
+          responseItems.add(
+              builder
+                  .setDomainNameIfSupported(domainName)
+                  .setPeriod(feeCheckItem.getPeriod())
+                  .setCommand(
+                      feeCheckItem.getCommandName(),
+                      feeCheckItem.getPhase(),
+                      feeCheckItem.getSubphase())
+                  .setCurrencyIfSupported(registry.getCurrency())
+                  .setClass("token-not-supported")
+                  .build());
+        }
       }
     }
     return ImmutableList.of(feeCheck.createResponse(responseItems.build()));

core/src/main/java/google/registry/flows/domain/DomainCreateFlow.java

@@ -89,6 +89,7 @@ import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.GracePeriod;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.fee.FeeCreateCommandExtension;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.fee.FeeTransformResponseExtension;
 import google.registry.model.domain.launch.LaunchCreateExtension;
 import google.registry.model.domain.metadata.MetadataExtension;
@@ -274,9 +275,10 @@ public final class DomainCreateFlow implements TransactionalFlow {
             now,
             eppInput.getSingleExtension(AllocationTokenExtension.class));
     boolean defaultTokenUsed = false;
-    if (!allocationToken.isPresent() && !registry.getDefaultPromoTokens().isEmpty()) {
+    if (!allocationToken.isPresent()) {
       allocationToken =
-          DomainFlowUtils.checkForDefaultToken(registry, command.getDomainName(), registrarId, now);
+          DomainFlowUtils.checkForDefaultToken(
+              registry, command.getDomainName(), CommandName.CREATE, registrarId, now);
       if (allocationToken.isPresent()) {
         defaultTokenUsed = true;
       }

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -97,6 +97,7 @@ import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Credit;
 import google.registry.model.domain.fee.Fee;
 import google.registry.model.domain.fee.FeeQueryCommandExtensionItem;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.fee.FeeQueryResponseExtensionItem;
 import google.registry.model.domain.fee.FeeTransformCommandExtension;
 import google.registry.model.domain.fee.FeeTransformResponseExtension;
@@ -175,8 +176,7 @@ public class DomainFlowUtils {
       CharMatcher.inRange('a', 'z').or(CharMatcher.inRange('0', '9').or(CharMatcher.anyOf("-.")));
 
   /** Default validator used to determine if an IDN name can be provisioned on a TLD. */
-  private static final IdnLabelValidator IDN_LABEL_VALIDATOR =
-      IdnLabelValidator.createDefaultIdnLabelValidator();
+  private static final IdnLabelValidator IDN_LABEL_VALIDATOR = new IdnLabelValidator();
 
   /** The maximum number of DS records allowed on a domain. */
   private static final int MAX_DS_RECORDS_PER_DOMAIN = 8;
@@ -674,8 +674,6 @@ public class DomainFlowUtils {
     String feeClass = null;
     ImmutableList<Fee> fees = ImmutableList.of();
     switch (feeRequest.getCommandName()) {
-        // TODO(sarahbot@): Add check of valid EPP actions on token before passing the token to the
-        // fee request.
       case CREATE:
         // Don't return a create price for reserved names.
         if (isReserved(domainName, isSunrise) && !isAvailable) {
@@ -1204,7 +1202,12 @@ public class DomainFlowUtils {
    * token found on the TLD's default token list will be returned.
    */
   public static Optional<AllocationToken> checkForDefaultToken(
-      Registry registry, String domainName, String registrarId, DateTime now) throws EppException {
+      Registry registry,
+      String domainName,
+      CommandName commandName,
+      String registrarId,
+      DateTime now)
+      throws EppException {
     if (isNullOrEmpty(registry.getDefaultPromoTokens())) {
       return Optional.empty();
     }
@@ -1222,7 +1225,7 @@ public class DomainFlowUtils {
     for (Optional<AllocationToken> token : tokenList) {
       try {
         AllocationTokenFlowUtils.validateToken(
-            InternetDomainName.from(domainName), token.get(), registrarId, now);
+            InternetDomainName.from(domainName), token.get(), commandName, registrarId, now);
       } catch (AssociationProhibitsOperationException | StatusProhibitsOperationException e) {
         // Allocation token was not valid for this registration, continue to check the next token in
         // the list

core/src/main/java/google/registry/flows/domain/DomainRenewFlow.java

@@ -66,6 +66,7 @@ import google.registry.model.domain.GracePeriod;
 import google.registry.model.domain.Period;
 import google.registry.model.domain.fee.BaseFee.FeeType;
 import google.registry.model.domain.fee.Fee;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.fee.FeeRenewCommandExtension;
 import google.registry.model.domain.fee.FeeTransformResponseExtension;
 import google.registry.model.domain.metadata.MetadataExtension;
@@ -172,14 +173,25 @@ public final class DomainRenewFlow implements TransactionalFlow {
     Renew command = (Renew) resourceCommand;
     // Loads the target resource if it exists
     Domain existingDomain = loadAndVerifyExistence(Domain.class, targetId, now);
-    // TODO(sarahbot@): Add check for valid EPP actions on the token
+    String tld = existingDomain.getTld();
+    Registry registry = Registry.get(tld);
     Optional<AllocationToken> allocationToken =
         allocationTokenFlowUtils.verifyAllocationTokenIfPresent(
             existingDomain,
-            Registry.get(existingDomain.getTld()),
+            registry,
             registrarId,
             now,
+            CommandName.RENEW,
             eppInput.getSingleExtension(AllocationTokenExtension.class));
+    boolean defaultTokenUsed = false;
+    if (!allocationToken.isPresent()) {
+      allocationToken =
+          DomainFlowUtils.checkForDefaultToken(
+              registry, existingDomain.getDomainName(), CommandName.RENEW, registrarId, now);
+      if (allocationToken.isPresent()) {
+        defaultTokenUsed = true;
+      }
+    }
     verifyRenewAllowed(authInfo, existingDomain, command, allocationToken);
 
     // If client passed an applicable static token this updates the domain
@@ -201,7 +213,7 @@ public final class DomainRenewFlow implements TransactionalFlow {
             years,
             existingRecurringBillingEvent,
             allocationToken);
-    validateFeeChallenge(feeRenew, feesAndCredits, false);
+    validateFeeChallenge(feeRenew, feesAndCredits, defaultTokenUsed);
     flowCustomLogic.afterValidation(
         AfterValidationParameters.newBuilder()
             .setExistingDomain(existingDomain)
@@ -210,7 +222,6 @@ public final class DomainRenewFlow implements TransactionalFlow {
             .build());
     HistoryEntryId domainHistoryId = createHistoryEntryId(existingDomain);
     historyBuilder.setRevisionId(domainHistoryId.getRevisionId());
-    String tld = existingDomain.getTld();
     // Bill for this explicit renew itself.
     BillingEvent.OneTime explicitRenewEvent =
         createRenewBillingEvent(
@@ -243,7 +254,6 @@ public final class DomainRenewFlow implements TransactionalFlow {
                 GracePeriod.forBillingEvent(
                     GracePeriodStatus.RENEW, existingDomain.getRepoId(), explicitRenewEvent))
             .build();
-    Registry registry = Registry.get(existingDomain.getTld());
     DomainHistory domainHistory =
         buildDomainHistory(
             newDomain, now, command.getPeriod(), registry.getRenewGracePeriodLength());

core/src/main/java/google/registry/flows/domain/DomainTransferApproveFlow.java

@@ -53,6 +53,7 @@ import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.metadata.MetadataExtension;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationTokenExtension;
@@ -135,6 +136,7 @@ public final class DomainTransferApproveFlow implements TransactionalFlow {
         Registry.get(existingDomain.getTld()),
         registrarId,
         now,
+        CommandName.TRANSFER,
         eppInput.getSingleExtension(AllocationTokenExtension.class));
     verifyOptionalAuthInfo(authInfo, existingDomain);
     verifyHasPendingTransfer(existingDomain);

core/src/main/java/google/registry/flows/domain/DomainTransferRequestFlow.java

@@ -57,6 +57,7 @@ import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainCommand.Transfer;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.Period;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.fee.FeeTransferCommandExtension;
 import google.registry.model.domain.fee.FeeTransformResponseExtension;
 import google.registry.model.domain.metadata.MetadataExtension;
@@ -173,6 +174,7 @@ public final class DomainTransferRequestFlow implements TransactionalFlow {
         Registry.get(existingDomain.getTld()),
         gainingClientId,
         now,
+        CommandName.TRANSFER,
         eppInput.getSingleExtension(AllocationTokenExtension.class));
     Optional<DomainTransferRequestSuperuserExtension> superuserExtension =
         eppInput.getSingleExtension(DomainTransferRequestSuperuserExtension.class);

core/src/main/java/google/registry/flows/domain/token/AllocationTokenFlowUtils.java

@@ -30,6 +30,7 @@ import google.registry.model.billing.BillingEvent.Recurring;
 import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainCommand;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenBehavior;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
@@ -78,7 +79,7 @@ public class AllocationTokenFlowUtils {
     ImmutableMap.Builder<InternetDomainName, String> resultsBuilder = new ImmutableMap.Builder<>();
     for (InternetDomainName domainName : domainNames) {
       try {
-        validateToken(domainName, tokenEntity, registrarId, now);
+        validateToken(domainName, tokenEntity, CommandName.CREATE, registrarId, now);
         validDomainNames.add(domainName);
       } catch (EppException e) {
         resultsBuilder.put(domainName, e.getMessage());
@@ -109,11 +110,19 @@ public class AllocationTokenFlowUtils {
    * @throws EppException if the token is invalid in any way
    */
   public static void validateToken(
-      InternetDomainName domainName, AllocationToken token, String registrarId, DateTime now)
+      InternetDomainName domainName,
+      AllocationToken token,
+      CommandName commandName,
+      String registrarId,
+      DateTime now)
       throws EppException {
 
     // Only tokens with default behavior require validation
     if (TokenBehavior.DEFAULT.equals(token.getTokenBehavior())) {
+      if (!token.getAllowedEppActions().isEmpty()
+          && !token.getAllowedEppActions().contains(commandName)) {
+        throw new AllocationTokenNotValidForCommandException();
+      }
       if (!token.getAllowedRegistrarIds().isEmpty()
           && !token.getAllowedRegistrarIds().contains(registrarId)) {
         throw new AllocationTokenNotValidForRegistrarException();
@@ -173,7 +182,12 @@ public class AllocationTokenFlowUtils {
       return Optional.empty();
     }
     AllocationToken tokenEntity = loadToken(extension.get().getAllocationToken());
-    validateToken(InternetDomainName.from(command.getDomainName()), tokenEntity, registrarId, now);
+    validateToken(
+        InternetDomainName.from(command.getDomainName()),
+        tokenEntity,
+        CommandName.CREATE,
+        registrarId,
+        now);
     return Optional.of(
         tokenCustomLogic.validateToken(command, tokenEntity, registry, registrarId, now));
   }
@@ -184,6 +198,7 @@ public class AllocationTokenFlowUtils {
       Registry registry,
       String registrarId,
       DateTime now,
+      CommandName commandName,
       Optional<AllocationTokenExtension> extension)
       throws EppException {
     if (!extension.isPresent()) {
@@ -191,7 +206,11 @@ public class AllocationTokenFlowUtils {
     }
     AllocationToken tokenEntity = loadToken(extension.get().getAllocationToken());
     validateToken(
-        InternetDomainName.from(existingDomain.getDomainName()), tokenEntity, registrarId, now);
+        InternetDomainName.from(existingDomain.getDomainName()),
+        tokenEntity,
+        commandName,
+        registrarId,
+        now);
     return Optional.of(
         tokenCustomLogic.validateToken(existingDomain, tokenEntity, registry, registrarId, now));
   }
@@ -280,6 +299,14 @@ public class AllocationTokenFlowUtils {
     }
   }
 
+  /** The allocation token is not valid for this EPP command. */
+  public static class AllocationTokenNotValidForCommandException
+      extends AssociationProhibitsOperationException {
+    AllocationTokenNotValidForCommandException() {
+      super("Allocation token not valid for the EPP command");
+    }
+  }
+
   /** The allocation token is invalid. */
   public static class InvalidAllocationTokenException extends AuthorizationErrorException {
     InvalidAllocationTokenException() {

core/src/main/java/google/registry/idn/Latin-IDN.txt

@@ -68,6 +68,7 @@ U+011F                         # LATIN SMALL LETTER G WITH BREVE
 U+01E7                         # LATIN SMALL LETTER G WITH CARON
 U+0121                         # LATIN SMALL LETTER G WITH DOT ABOVE
 U+0123                         # LATIN SMALL LETTER G WITH CEDILLA
+U+01E5                         # LATIN SMALL LETTER G WITH STROKE
 U+0068                         # LATIN SMALL LETTER H
 U+0127                         # LATIN SMALL LETTER H WITH STROKE
 U+0069                         # LATIN SMALL LETTER I

core/src/main/java/google/registry/model/common/DnsRefreshRequest.java

@@ -21,6 +21,7 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
 import google.registry.dns.DnsConstants.TargetType;
 import google.registry.dns.PublishDnsUpdatesAction;
 import google.registry.model.ImmutableObject;
+import google.registry.persistence.VKey;
 import javax.annotation.Nullable;
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -89,6 +90,11 @@ public class DnsRefreshRequest extends ImmutableObject {
     return lastProcessTime;
   }
 
+  @Override
+  public VKey<DnsRefreshRequest> createVKey() {
+    return VKey.create(DnsRefreshRequest.class, id);
+  }
+
   protected DnsRefreshRequest() {}
 
   private DnsRefreshRequest(

core/src/main/java/google/registry/model/domain/fee/Credit.java

@@ -34,7 +34,7 @@ public class Credit extends BaseFee {
       BigDecimal cost, FeeType type, String description) {
     Credit instance = new Credit();
     instance.cost = checkNotNull(cost);
-    checkArgument(instance.cost.signum() < 0);
+    checkArgument(instance.cost.signum() <= 0, "A credit cannot have a positive cost");
     instance.type = checkNotNull(type);
     instance.description = description;
     return instance;

core/src/main/java/google/registry/model/domain/fee/FeeQueryCommandExtensionItem.java

@@ -14,6 +14,8 @@
 
 package google.registry.model.domain.fee;
 
+import static com.google.common.base.Preconditions.checkArgument;
+
 import google.registry.model.ImmutableObject;
 import google.registry.model.domain.Period;
 import java.util.Optional;
@@ -37,7 +39,19 @@ public abstract class FeeQueryCommandExtensionItem extends ImmutableObject {
     RENEW,
     TRANSFER,
     RESTORE,
-    UPDATE
+    UPDATE;
+
+    public static CommandName parseKnownCommand(String string) {
+      try {
+        CommandName command = valueOf(string);
+        checkArgument(!command.equals(UNKNOWN));
+        return command;
+      } catch (IllegalArgumentException e) {
+        throw new IllegalArgumentException(
+            "Invalid EPP action name. Valid actions are CREATE, RENEW, TRANSFER, RESTORE, and"
+                + " UPDATE");
+      }
+    }
   }
 
   /** The default validity period (if not specified) is 1 year for all operations. */

core/src/main/java/google/registry/model/tld/Registry.java

@@ -52,6 +52,7 @@ import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.ReservedList;
 import google.registry.persistence.VKey;
 import google.registry.persistence.converter.JodaMoneyType;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import google.registry.util.Idn;
 import java.util.List;
 import java.util.Map;
@@ -487,6 +488,9 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
    */
   List<VKey<AllocationToken>> defaultPromoTokens;
 
+  /** A set of allowed {@link IdnTableEnum}s for this TLD, or empty if we should use the default. */
+  Set<IdnTableEnum> idnTables;
+
   public String getTldStr() {
     return tldStr;
   }
@@ -694,6 +698,10 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
     return nullToEmptyImmutableCopy(defaultPromoTokens);
   }
 
+  public ImmutableSet<IdnTableEnum> getIdnTables() {
+    return nullToEmptyImmutableCopy(idnTables);
+  }
+
   @Override
   public Builder asBuilder() {
     return new Builder(clone(this));
@@ -992,6 +1000,11 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
       return this;
     }
 
+    public Builder setIdnTables(ImmutableSet<IdnTableEnum> idnTables) {
+      getInstance().idnTables = idnTables;
+      return this;
+    }
+
     @Override
     public Registry build() {
       final Registry instance = getInstance();

core/src/main/java/google/registry/persistence/converter/IdnTableEnumSetConverter.java

@@ -0,0 +1,34 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import google.registry.tldconfig.idn.IdnTableEnum;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA {@link AttributeConverter} for storing/retrieving {@link IdnTableEnum}s. */
+@Converter(autoApply = true)
+public class IdnTableEnumSetConverter extends StringSetConverterBase<IdnTableEnum> {
+
+  @Override
+  String toString(IdnTableEnum element) {
+    return element.name();
+  }
+
+  @Override
+  IdnTableEnum fromString(String value) {
+    return IdnTableEnum.valueOf(value);
+  }
+}

core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java

@@ -154,6 +154,10 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
 
   @Override
   public <T> T transact(Supplier<T> work) {
+    // This prevents inner transaction from retrying, thus avoiding a cascade retry effect.
+    if (inTransaction()) {
+      return transactNoRetry(work);
+    }
     return retrier.callWithRetry(() -> transactNoRetry(work), JpaRetries::isFailedTxnRetriable);
   }
 

core/src/main/java/google/registry/tldconfig/idn/IdnLabelValidator.java

@@ -17,8 +17,8 @@ package google.registry.tldconfig.idn;
 import static google.registry.tldconfig.idn.IdnTableEnum.EXTENDED_LATIN;
 import static google.registry.tldconfig.idn.IdnTableEnum.JA;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.tld.Registry;
 import google.registry.util.Idn;
 import java.util.Optional;
 
@@ -26,23 +26,8 @@ import java.util.Optional;
 public final class IdnLabelValidator {
 
   /** Most TLDs will use this generic list of IDN tables. */
-  private static final ImmutableList<IdnTableEnum> DEFAULT_IDN_TABLES =
-      ImmutableList.of(EXTENDED_LATIN, JA);
-
-  private static final ImmutableMap<String, ImmutableList<IdnTableEnum>>
-      DEFAULT_IDN_TABLE_LISTS_PER_TLD =
-          ImmutableMap.of("xn--q9jyb4c", ImmutableList.of(EXTENDED_LATIN, JA));
-
-  /** Some TLDs have their own IDN tables, configured here. */
-  private ImmutableMap<String, ImmutableList<IdnTableEnum>> idnTableListsPerTld;
-
-  IdnLabelValidator(ImmutableMap<String, ImmutableList<IdnTableEnum>> indTableListsPerTld) {
-    this.idnTableListsPerTld = indTableListsPerTld;
-  }
-
-  public static IdnLabelValidator createDefaultIdnLabelValidator() {
-    return new IdnLabelValidator(DEFAULT_IDN_TABLE_LISTS_PER_TLD);
-  }
+  private static final ImmutableSet<IdnTableEnum> DEFAULT_IDN_TABLES =
+      ImmutableSet.of(EXTENDED_LATIN, JA);
 
   /**
    * Returns name of first matching {@link IdnTable} if domain label is valid for the given TLD.
@@ -50,10 +35,13 @@ public final class IdnLabelValidator {
    * <p>A label is valid if it is considered valid by at least one configured IDN table for that
    * TLD. If no match is found, an absent value is returned.
    */
-  public Optional<String> findValidIdnTableForTld(String label, String tld) {
+  public Optional<String> findValidIdnTableForTld(String label, String tldStr) {
     String unicodeString = Idn.toUnicode(label);
-    for (IdnTableEnum idnTable :
-        Optional.ofNullable(idnTableListsPerTld.get(tld)).orElse(DEFAULT_IDN_TABLES)) {
+    Registry tld = Registry.get(tldStr); // uses the cache
+    ImmutableSet<IdnTableEnum> idnTablesForTld = tld.getIdnTables();
+    ImmutableSet<IdnTableEnum> idnTables =
+        idnTablesForTld.isEmpty() ? DEFAULT_IDN_TABLES : idnTablesForTld;
+    for (IdnTableEnum idnTable : idnTables) {
       if (idnTable.getTable().isValidLabel(unicodeString)) {
         return Optional.of(idnTable.getTable().getName());
       }

core/src/main/java/google/registry/tldconfig/idn/IdnTableEnum.java

@@ -24,23 +24,48 @@ import java.net.URL;
 
 /** Wrapper enum that loads all {@link IdnTable} resources into memory. */
 public enum IdnTableEnum {
-  EXTENDED_LATIN,
-  JA;
+
+  /**
+   * Extended Latin, as used on our existing TLD launches prior to 2023.
+   *
+   * <p>As of 2023 this table is no longer conformant with ICANN's IDN policies for new launches, so
+   * it is retained solely for legacy compatibility with already-launched TLDs.
+   */
+  EXTENDED_LATIN("extended_latin.txt"),
+
+  /**
+   * Extended Latin, but with confusable characters removed.
+   *
+   * <p>This is compatible with ICANN's requirements as of 2023, and is used for the Dads and Grads
+   * TLDs and all subsequent TLD launches. Note that confusable characters consist of various
+   * letters with diacritic marks on them, e.g. U+00EF (LATIN SMALL LETTER I WITH DIAERESIS) is not
+   * allowed because it is confusable with the standard i.
+   */
+  UNCONFUSABLE_LATIN("unconfusable_latin.txt"),
+
+  /**
+   * Japanese, as used on our existing TLD launches prior to 2023.
+   *
+   * <p>As of 2023 this table is no longer conformant with ICANN's IDN policies for new launches, so
+   * it is retained solely for legacy compatibility with already-launched TLDs.
+   */
+  JA("japanese.txt");
 
   private final IdnTable table;
 
-  IdnTableEnum() {
-    this.table = load(Ascii.toLowerCase(name()));
+  IdnTableEnum(String filename) {
+    this.table = load(Ascii.toLowerCase(name()), filename);
   }
 
   public IdnTable getTable() {
     return table;
   }
 
-  private static IdnTable load(String name) {
+  private static IdnTable load(String tableName, String filename) {
     try {
-      URL resource = Resources.getResource(IdnTableEnum.class, name + ".txt");
-      return IdnTable.createFrom(name, readLines(resource, UTF_8), LanguageValidator.get(name));
+      URL resource = Resources.getResource(IdnTableEnum.class, filename);
+      return IdnTable.createFrom(
+          tableName, readLines(resource, UTF_8), LanguageValidator.get(tableName));
     } catch (IOException e) {
       throw new RuntimeException(e);  // should never happen
     }

core/src/main/java/google/registry/tldconfig/idn/extended_latin.txt

@@ -49,6 +49,7 @@ U+011F                         # LATIN SMALL LETTER G WITH BREVE
 U+01E7                         # LATIN SMALL LETTER G WITH CARON
 U+0121                         # LATIN SMALL LETTER G WITH DOT ABOVE
 U+0123                         # LATIN SMALL LETTER G WITH CEDILLA
+U+01E5                         # LATIN SMALL LETTER G WITH STROKE
 U+0068                         # LATIN SMALL LETTER H
 U+0127                         # LATIN SMALL LETTER H WITH STROKE
 U+0069                         # LATIN SMALL LETTER I

core/src/main/java/google/registry/tldconfig/idn/unconfusable_latin.txt

@@ -0,0 +1,124 @@
+# Registry: Charleston Road Registry Inc.
+# Script: Latn
+# Version: 2.0
+# Effective Date: 2023-04-04
+# URL: https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt
+# Policy: https://www.registry.google/about/policies/domainabuse/
+# Contact Name: CRR Tech
+# Email address: crr-tech@google.com
+# Telephone: +1 (650) 253-0000
+#
+# Code points requiring context rules
+#
+# Code point                        Description of rule/Reference
+#
+# U+002D                            Label must neither start nor end with U+002D. Label
+#    HYPHEN-MINUS                   must not have U+002D in both third and fourth
+#                                   position. RFC 5891 (sec 4.2.3.1)
+#
+
+U+002D                         # HYPHEN-MINUS
+U+0030                         # DIGIT ZERO
+U+0031                         # DIGIT ONE
+U+0032                         # DIGIT TWO
+U+0033                         # DIGIT THREE
+U+0034                         # DIGIT FOUR
+U+0035                         # DIGIT FIVE
+U+0036                         # DIGIT SIX
+U+0037                         # DIGIT SEVEN
+U+0038                         # DIGIT EIGHT
+U+0039                         # DIGIT NINE
+U+0061                         # LATIN SMALL LETTER A
+U+00E0                         # LATIN SMALL LETTER A WITH GRAVE
+U+0103                         # LATIN SMALL LETTER A WITH BREVE
+U+00E2                         # LATIN SMALL LETTER A WITH CIRCUMFLEX
+U+00E5                         # LATIN SMALL LETTER A WITH RING ABOVE
+U+00E4                         # LATIN SMALL LETTER A WITH DIAERESIS
+U+00E3                         # LATIN SMALL LETTER A WITH TILDE
+U+0105                         # LATIN SMALL LETTER A WITH OGONEK
+U+00E6                         # LATIN SMALL LETTER AE
+U+0062                         # LATIN SMALL LETTER B
+U+0063                         # LATIN SMALL LETTER C
+U+0107                         # LATIN SMALL LETTER C WITH ACUTE
+U+010D                         # LATIN SMALL LETTER C WITH CARON
+U+00E7                         # LATIN SMALL LETTER C WITH CEDILLA
+U+0064                         # LATIN SMALL LETTER D
+U+010F                         # LATIN SMALL LETTER D WITH CARON
+U+0111                         # LATIN SMALL LETTER D WITH STROKE
+U+00F0                         # LATIN SMALL LETTER ETH
+U+0065                         # LATIN SMALL LETTER E
+U+00E9                         # LATIN SMALL LETTER E WITH ACUTE
+U+00E8                         # LATIN SMALL LETTER E WITH GRAVE
+U+00EA                         # LATIN SMALL LETTER E WITH CIRCUMFLEX
+U+011B                         # LATIN SMALL LETTER E WITH CARON
+U+00EB                         # LATIN SMALL LETTER E WITH DIAERESIS
+U+0119                         # LATIN SMALL LETTER E WITH OGONEK
+U+0113                         # LATIN SMALL LETTER E WITH MACRON
+U+0117                         # LATIN SMALL LETTER E WITH DOT ABOVE
+U+0259                         # LATIN SMALL LETTER SCHWA
+U+0066                         # LATIN SMALL LETTER F
+U+0067                         # LATIN SMALL LETTER G
+U+011F                         # LATIN SMALL LETTER G WITH BREVE
+U+0121                         # LATIN SMALL LETTER G WITH DOT ABOVE
+U+0068                         # LATIN SMALL LETTER H
+U+0127                         # LATIN SMALL LETTER H WITH STROKE
+U+0069                         # LATIN SMALL LETTER I
+U+00EC                         # LATIN SMALL LETTER I WITH GRAVE
+U+00EE                         # LATIN SMALL LETTER I WITH CIRCUMFLEX
+U+012F                         # LATIN SMALL LETTER I WITH OGONEK
+U+012B                         # LATIN SMALL LETTER I WITH MACRON
+U+006A                         # LATIN SMALL LETTER J
+U+006B                         # LATIN SMALL LETTER K
+U+01E9                         # LATIN SMALL LETTER K WITH CARON
+U+0137                         # LATIN SMALL LETTER K WITH CEDILLA
+U+006C                         # LATIN SMALL LETTER L
+U+013A                         # LATIN SMALL LETTER L WITH ACUTE
+U+013E                         # LATIN SMALL LETTER L WITH CARON
+U+013C                         # LATIN SMALL LETTER L WITH CEDILLA
+U+0142                         # LATIN SMALL LETTER L WITH STROKE
+U+006D                         # LATIN SMALL LETTER M
+U+006E                         # LATIN SMALL LETTER N
+U+0148                         # LATIN SMALL LETTER N WITH CARON
+U+00F1                         # LATIN SMALL LETTER N WITH TILDE
+U+0146                         # LATIN SMALL LETTER N WITH CEDILLA
+U+006F                         # LATIN SMALL LETTER O
+U+00F2                         # LATIN SMALL LETTER O WITH GRAVE
+U+00F4                         # LATIN SMALL LETTER O WITH CIRCUMFLEX
+U+00F6                         # LATIN SMALL LETTER O WITH DIAERESIS
+U+0151                         # LATIN SMALL LETTER O WITH DOUBLE ACUTE
+U+00F5                         # LATIN SMALL LETTER O WITH TILDE
+U+00F8                         # LATIN SMALL LETTER O WITH STROKE
+U+0153                         # LATIN SMALL LIGATURE OE
+U+0070                         # LATIN SMALL LETTER P
+U+0071                         # LATIN SMALL LETTER Q
+U+0072                         # LATIN SMALL LETTER R
+U+0155                         # LATIN SMALL LETTER R WITH ACUTE
+U+0159                         # LATIN SMALL LETTER R WITH CARON
+U+0073                         # LATIN SMALL LETTER S
+U+015B                         # LATIN SMALL LETTER S WITH ACUTE
+U+0161                         # LATIN SMALL LETTER S WITH CARON
+U+015F                         # LATIN SMALL LETTER S WITH CEDILLA
+U+0074                         # LATIN SMALL LETTER T
+U+0165                         # LATIN SMALL LETTER T WITH CARON
+U+0167                         # LATIN SMALL LETTER T WITH STROKE
+U+0075                         # LATIN SMALL LETTER U
+U+00F9                         # LATIN SMALL LETTER U WITH GRAVE
+U+00FB                         # LATIN SMALL LETTER U WITH CIRCUMFLEX
+U+016F                         # LATIN SMALL LETTER U WITH RING ABOVE
+U+0171                         # LATIN SMALL LETTER U WITH DOUBLE ACUTE
+U+0173                         # LATIN SMALL LETTER U WITH OGONEK
+U+016B                         # LATIN SMALL LETTER U WITH MACRON
+U+0076                         # LATIN SMALL LETTER V
+U+0077                         # LATIN SMALL LETTER W
+U+0175                         # LATIN SMALL LETTER W WITH CIRCUMFLEX
+U+0078                         # LATIN SMALL LETTER X
+U+0079                         # LATIN SMALL LETTER Y
+U+00FD                         # LATIN SMALL LETTER Y WITH ACUTE
+U+0177                         # LATIN SMALL LETTER Y WITH CIRCUMFLEX
+U+00FF                         # LATIN SMALL LETTER Y WITH DIAERESIS
+U+007A                         # LATIN SMALL LETTER Z
+U+017A                         # LATIN SMALL LETTER Z WITH ACUTE
+U+017E                         # LATIN SMALL LETTER Z WITH CARON
+U+0292                         # LATIN SMALL LETTER EZH
+U+01EF                         # LATIN SMALL LETTER EZH WITH CARON
+U+00FE                         # LATIN SMALL LETTER THORN

core/src/main/java/google/registry/tmch/NordnUploadAction.java

@@ -15,7 +15,6 @@
 package google.registry.tmch;
 
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.net.HttpHeaders.LOCATION;
 import static com.google.common.net.MediaType.CSV_UTF_8;
 import static google.registry.persistence.transaction.QueryComposer.Comparator.EQ;
@@ -26,23 +25,13 @@ import static google.registry.tmch.LordnTaskUtils.COLUMNS_SUNRISE;
 import static google.registry.tmch.LordnTaskUtils.getCsvLineForClaimsDomain;
 import static google.registry.tmch.LordnTaskUtils.getCsvLineForSunriseDomain;
 import static java.nio.charset.StandardCharsets.US_ASCII;
-import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 
 import com.google.api.client.http.HttpMethods;
-import com.google.appengine.api.taskqueue.LeaseOptions;
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.apphosting.api.DeadlineExceededException;
 import com.google.cloud.tasks.v2.Task;
 import com.google.common.base.Ascii;
-import com.google.common.base.Joiner;
-import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMultimap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Ordering;
 import com.google.common.flogger.FluentLogger;
 import google.registry.batch.CloudTasksUtils;
 import google.registry.config.RegistryConfig.Config;
@@ -65,9 +54,7 @@ import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 import java.util.List;
 import java.util.Random;
-import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
-import org.joda.time.DateTime;
 import org.joda.time.Duration;
 
 /**
@@ -88,9 +75,7 @@ public final class NordnUploadAction implements Runnable {
   static final String PATH = "/_dr/task/nordnUpload";
   static final String LORDN_PHASE_PARAM = "lordnPhase";
 
-  private static final int BATCH_SIZE = 1000;
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final Duration LEASE_PERIOD = Duration.standardHours(1);
 
   /**
    * A unique (enough) id that is outputted in log lines to make it clear which log lines are
@@ -192,46 +177,6 @@ public final class NordnUploadAction implements Runnable {
             });
   }
 
-  /**
-   * Converts a list of queue tasks, each containing a row of CSV data, into a single newline-
-   * delimited String.
-   */
-  static String convertTasksToCsv(List<TaskHandle> tasks, DateTime now, String columns) {
-    // Use a Set for deduping purposes, so we can be idempotent in case tasks happened to be
-    // enqueued multiple times for a given domain create.
-    ImmutableSortedSet.Builder<String> builder =
-        new ImmutableSortedSet.Builder<>(Ordering.natural());
-    for (TaskHandle task : checkNotNull(tasks)) {
-      String payload = new String(task.getPayload(), UTF_8);
-      if (!Strings.isNullOrEmpty(payload)) {
-        builder.add(payload + '\n');
-      }
-    }
-    ImmutableSortedSet<String> csvLines = builder.build();
-    String header = String.format("1,%s,%d\n%s\n", now, csvLines.size(), columns);
-    return header + Joiner.on("").join(csvLines);
-  }
-
-  /** Leases and returns all tasks from the queue with the specified tag tld, in batches. */
-  List<TaskHandle> loadAllTasks(Queue queue, String tld) {
-    ImmutableList.Builder<TaskHandle> allTasks = new ImmutableList.Builder<>();
-    while (true) {
-      List<TaskHandle> tasks =
-          retrier.callWithRetry(
-              () ->
-                  queue.leaseTasks(
-                      LeaseOptions.Builder.withTag(tld)
-                          .leasePeriod(LEASE_PERIOD.getMillis(), TimeUnit.MILLISECONDS)
-                          .countLimit(BATCH_SIZE)),
-              TransientFailureException.class,
-              DeadlineExceededException.class);
-      if (tasks.isEmpty()) {
-        return allTasks.build();
-      }
-      allTasks.addAll(tasks);
-    }
-  }
-
   /**
    * Upload LORDN file to MarksDB.
    *

core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java

@@ -15,6 +15,7 @@
 package google.registry.tools;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.tools.UpdateOrDeleteAllocationTokensCommand.getTokenKeys;
 import static google.registry.util.CollectionUtils.findDuplicates;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
@@ -34,9 +35,11 @@ import google.registry.model.tld.Registry.TldState;
 import google.registry.model.tld.Registry.TldType;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.PremiumListDao;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import google.registry.tools.params.OptionalStringParameter;
 import google.registry.tools.params.TransitionListParameter.BillingCostTransitions;
 import google.registry.tools.params.TransitionListParameter.TldStateTransitions;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -244,6 +247,15 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
               + " present default tokens.")
   List<String> defaultTokens;
 
+  @Nullable
+  @Parameter(
+      names = "--idn_tables",
+      description =
+          "A comma-separated list of the IDN tables to use for this TLD. Specify an empty list to"
+              + " remove any previously-set tables and to use the default. All elements must be"
+              + " IdnTableEnum values")
+  List<String> idnTables;
+
   /** Returns the existing registry (for update) or null (for creates). */
   @Nullable
   abstract Registry getOldRegistry(String tld);
@@ -392,6 +404,23 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
           builder.setDefaultPromoTokens(getTokenKeys(defaultTokens, null));
         }
       }
+      if (idnTables != null) {
+        if (idnTables.equals(ImmutableList.of(""))) {
+          builder.setIdnTables(ImmutableSet.of());
+        } else {
+          ImmutableSet<String> upperCaseIdnTables =
+              idnTables.stream().map(String::toUpperCase).collect(toImmutableSet());
+          ImmutableSet<String> validIdnStringValues =
+              Arrays.stream(IdnTableEnum.values()).map(Enum::name).collect(toImmutableSet());
+          checkArgument(
+              validIdnStringValues.containsAll(upperCaseIdnTables),
+              "IDN tables %s contained invalid value(s). Possible values: %s",
+              upperCaseIdnTables,
+              validIdnStringValues);
+          builder.setIdnTables(
+              upperCaseIdnTables.stream().map(IdnTableEnum::valueOf).collect(toImmutableSet()));
+        }
+      }
       // Update the Registry object.
       setCommandSpecificProperties(builder);
       stageEntityChange(oldRegistry, builder.build());

core/src/main/java/google/registry/tools/GenerateAllocationTokensCommand.java

@@ -39,6 +39,7 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Streams;
 import com.google.common.io.Files;
 import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.RegistrationBehavior;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
@@ -114,6 +115,11 @@ class GenerateAllocationTokensCommand implements Command {
       description = "Comma-separated list of allowed TLDs, or null if all are allowed")
   private List<String> allowedTlds;
 
+  @Parameter(
+      names = {"--allowed_epp_actions"},
+      description = "Comma-separated list of allowed EPP actions, or null if all are allowed")
+  private List<String> allowedEppActions;
+
   @Parameter(
       names = {"--discount_fraction"},
       description =
@@ -207,7 +213,13 @@ class GenerateAllocationTokensCommand implements Command {
                             .setTokenType(tokenType == null ? SINGLE_USE : tokenType)
                             .setAllowedRegistrarIds(
                                 ImmutableSet.copyOf(nullToEmpty(allowedClientIds)))
-                            .setAllowedTlds(ImmutableSet.copyOf(nullToEmpty(allowedTlds)));
+                            .setAllowedTlds(ImmutableSet.copyOf(nullToEmpty(allowedTlds)))
+                            .setAllowedEppActions(
+                                isNullOrEmpty(allowedEppActions)
+                                    ? ImmutableSet.of()
+                                    : allowedEppActions.stream()
+                                        .map(CommandName::parseKnownCommand)
+                                        .collect(toImmutableSet()));
                     Optional.ofNullable(discountFraction).ifPresent(token::setDiscountFraction);
                     Optional.ofNullable(discountPremiums).ifPresent(token::setDiscountPremiums);
                     Optional.ofNullable(discountYears).ifPresent(token::setDiscountYears);
@@ -255,6 +267,10 @@ class GenerateAllocationTokensCommand implements Command {
         !ImmutableList.of("").equals(allowedTlds),
         "Either omit --allowed_tlds if all TLDs are allowed, or include a comma-separated list");
 
+    if (ImmutableList.of("").equals(allowedEppActions)) {
+      allowedEppActions = ImmutableList.of();
+    }
+
     if (!isNullOrEmpty(tokenStatusTransitions)) {
       // Don't allow package tokens to be created with a scheduled end time since this could allow
       // future domains to be attributed to the package and never be billed. Package promotion

core/src/main/java/google/registry/tools/UpdateAllocationTokensCommand.java

@@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.RegistrationBehavior;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
@@ -63,6 +64,13 @@ final class UpdateAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
               + "existing list.")
   private List<String> allowedTlds;
 
+  @Parameter(
+      names = {"--allowed_epp_actions"},
+      description =
+          "Comma-separated list of allowed EPP actions. Use an empty string to clear the existing"
+              + " list.")
+  private List<String> allowedEppActions;
+
   @Parameter(
       names = {"--discount_fraction"},
       description =
@@ -128,6 +136,9 @@ final class UpdateAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
     if (ImmutableList.of("").equals(allowedTlds)) {
       allowedTlds = ImmutableList.of();
     }
+    if (ImmutableList.of("").equals(allowedEppActions)) {
+      allowedEppActions = ImmutableList.of();
+    }
 
     if (tokenStatusTransitions != null
         && (tokenStatusTransitions.containsValue(TokenStatus.ENDED)
@@ -184,6 +195,14 @@ final class UpdateAllocationTokensCommand extends UpdateOrDeleteAllocationTokens
         .ifPresent(clientIds -> builder.setAllowedRegistrarIds(ImmutableSet.copyOf(clientIds)));
     Optional.ofNullable(allowedTlds)
         .ifPresent(tlds -> builder.setAllowedTlds(ImmutableSet.copyOf(tlds)));
+    Optional.ofNullable(allowedEppActions)
+        .ifPresent(
+            eppActions -> {
+              builder.setAllowedEppActions(
+                  eppActions.stream()
+                      .map(CommandName::parseKnownCommand)
+                      .collect(toImmutableSet()));
+            });
     Optional.ofNullable(discountFraction).ifPresent(builder::setDiscountFraction);
     Optional.ofNullable(discountPremiums).ifPresent(builder::setDiscountPremiums);
     Optional.ofNullable(discountYears).ifPresent(builder::setDiscountYears);

core/src/main/resources/META-INF/persistence.xml

@@ -91,6 +91,7 @@
     <class>google.registry.persistence.converter.DatabaseMigrationScheduleTransitionConverter</class>
     <class>google.registry.persistence.converter.DateTimeConverter</class>
     <class>google.registry.persistence.converter.DurationConverter</class>
+    <class>google.registry.persistence.converter.IdnTableEnumSetConverter</class>
     <class>google.registry.persistence.converter.InetAddressSetConverter</class>
     <class>google.registry.persistence.converter.LocalDateConverter</class>
     <class>google.registry.persistence.converter.PostalInfoChoiceListConverter</class>

core/src/test/java/google/registry/dns/DnsUtilsTest.java

@@ -14,6 +14,7 @@
 
 package google.registry.dns;
 
+import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
@@ -26,6 +27,7 @@ import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Ordering;
@@ -36,6 +38,7 @@ import google.registry.model.common.DnsRefreshRequest;
 import google.registry.persistence.transaction.JpaTestExtensions;
 import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
 import google.registry.testing.FakeClock;
+import java.util.Comparator;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.Assertions;
@@ -53,8 +56,7 @@ public class DnsUtilsTest {
 
   private final DnsQueue dnsQueue = mock(DnsQueue.class);
   private final DnsUtils dnsUtils = new DnsUtils(dnsQueue);
-
-  FakeClock clock = new FakeClock(DateTime.parse("2020-02-02T01:23:45Z"));
+  private final FakeClock clock = new FakeClock(DateTime.parse("2020-02-02T01:23:45Z"));
 
   @RegisterExtension
   JpaIntegrationTestExtension jpa =
@@ -138,12 +140,133 @@ public class DnsUtilsTest {
     assertRequest(request, TargetType.DOMAIN, domainName, tld, clock.nowUtc().plusMinutes(3));
   }
 
+  @Test
+  void testSuccess_ProcessRequests() {
+    ImmutableList<DnsRefreshRequest> requests = processRequests();
+    DateTime processtime = clock.nowUtc();
+    assertThat(requests.size()).isEqualTo(4);
+    assertRequest(
+        requests.get(0),
+        TargetType.DOMAIN,
+        "test2.tld",
+        "tld",
+        clock.nowUtc().minusMinutes(4),
+        processtime);
+    assertRequest(
+        requests.get(1),
+        TargetType.DOMAIN,
+        "test1.tld",
+        "tld",
+        clock.nowUtc().minusMinutes(3),
+        processtime);
+    assertRequest(
+        requests.get(2),
+        TargetType.HOST,
+        "ns1.test2.tld",
+        "tld",
+        clock.nowUtc().minusMinutes(1),
+        processtime);
+    assertRequest(
+        requests.get(3),
+        TargetType.DOMAIN,
+        "test5.tld",
+        "tld",
+        clock.nowUtc().minusMinutes(1),
+        processtime);
+    requests = loadAllOf(DnsRefreshRequest.class);
+    assertThat(requests.size()).isEqualTo(7);
+    // The four processed records should have updated process time in SQL as well.
+    assertThat(requests.stream().filter(e -> e.getLastProcessTime().equals(processtime)).count())
+        .isEqualTo(4);
+    clock.advanceOneMilli();
+
+    // Requests within cooldown period not included.
+    requests =
+        dnsUtils.readAndUpdateRequestsWithLatestProcessTime("tld", Duration.standardMinutes(1), 4);
+    assertThat(requests.size()).isEqualTo(1);
+    assertRequest(
+        requests.get(0),
+        TargetType.DOMAIN,
+        "test6.tld",
+        "tld",
+        clock.nowUtc().minusMinutes(1).minusMillis(1),
+        clock.nowUtc());
+  }
+
+  @Test
+  void testSuccess_deleteRequests() {
+    dnsUtils.deleteRequests(processRequests());
+    ImmutableList<DnsRefreshRequest> remainingRequests =
+        loadAllOf(DnsRefreshRequest.class).stream()
+            .sorted(Comparator.comparing(DnsRefreshRequest::getRequestTime))
+            .collect(toImmutableList());
+    assertThat(remainingRequests.size()).isEqualTo(3);
+    assertRequest(
+        remainingRequests.get(0),
+        TargetType.DOMAIN,
+        "something.example",
+        "example",
+        clock.nowUtc().minusMinutes(2));
+    assertRequest(
+        remainingRequests.get(1),
+        TargetType.DOMAIN,
+        "test6.tld",
+        "tld",
+        clock.nowUtc().minusMinutes(1));
+    assertRequest(
+        remainingRequests.get(2),
+        TargetType.DOMAIN,
+        "test4.tld",
+        "tld",
+        clock.nowUtc().plusMinutes(1));
+    tm().transact(() -> tm().delete(remainingRequests.get(2)));
+    assertThat(loadAllOf(DnsRefreshRequest.class).size()).isEqualTo(2);
+    // Should not throw even though one of the request is already deleted.
+    dnsUtils.deleteRequests(remainingRequests);
+    assertThat(loadAllOf(DnsRefreshRequest.class).size()).isEqualTo(0);
+  }
+
+  private ImmutableList<DnsRefreshRequest> processRequests() {
+    useDnsSql();
+    createTld("example");
+    // Domain Included.
+    dnsUtils.requestDomainDnsRefresh("test1.tld", Duration.standardMinutes(1));
+    // This one should be returned before test1.tld, even though it's added later, because of
+    // the delay specified in test1.tld.
+    dnsUtils.requestDomainDnsRefresh("test2.tld");
+    // Not included because the TLD is not under management.
+    dnsUtils.requestDomainDnsRefresh("something.example", Duration.standardMinutes(2));
+    clock.advanceBy(Duration.standardMinutes(3));
+    // Host included.
+    dnsUtils.requestHostDnsRefresh("ns1.test2.tld");
+    // Not included because the request time is in the future
+    dnsUtils.requestDomainDnsRefresh("test4.tld", Duration.standardMinutes(2));
+    // Included after the previous one. Same request time, order by insertion order (i.e. ID);
+    dnsUtils.requestDomainDnsRefresh("test5.tld");
+    // Not included because batch size is exceeded;
+    dnsUtils.requestDomainDnsRefresh("test6.tld");
+    clock.advanceBy(Duration.standardMinutes(1));
+    return dnsUtils.readAndUpdateRequestsWithLatestProcessTime(
+        "tld", Duration.standardMinutes(1), 4);
+  }
+
   private static void assertRequest(
       DnsRefreshRequest request, TargetType type, String name, String tld, DateTime requestTime) {
+    assertRequest(request, type, name, tld, requestTime, START_OF_TIME);
+  }
+
+  private static void assertRequest(
+      DnsRefreshRequest request,
+      TargetType type,
+      String name,
+      String tld,
+      DateTime requestTime,
+      DateTime processTime) {
     assertThat(request.getType()).isEqualTo(type);
     assertThat(request.getName()).isEqualTo(name);
     assertThat(request.getTld()).isEqualTo(tld);
     assertThat(request.getRequestTime()).isEqualTo(requestTime);
+    assertThat(request.getLastProcessTime()).isEqualTo(processTime);
   }
 
   private void useDnsSql() {

core/src/test/java/google/registry/dns/PublishDnsUpdatesActionTest.java

@@ -22,7 +22,7 @@ import static google.registry.dns.DnsModule.PARAM_HOSTS;
 import static google.registry.dns.DnsModule.PARAM_LOCK_INDEX;
 import static google.registry.dns.DnsModule.PARAM_NUM_PUBLISH_LOCKS;
 import static google.registry.dns.DnsModule.PARAM_PUBLISH_TASK_ENQUEUED;
-import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_CREATED;
+import static google.registry.dns.DnsModule.PARAM_REFRESH_REQUEST_TIME;
 import static google.registry.dns.PublishDnsUpdatesAction.RETRIES_BEFORE_PERMANENT_FAILURE;
 import static google.registry.request.RequestParameters.PARAM_TLD;
 import static google.registry.testing.DatabaseHelper.createTld;
@@ -302,7 +302,7 @@ public class PublishDnsUpdatesActionTest {
             .param(PARAM_LOCK_INDEX, "1")
             .param(PARAM_NUM_PUBLISH_LOCKS, "1")
             .param(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-            .param(PARAM_REFRESH_REQUEST_CREATED, clock.nowUtc().minusHours(2).toString())
+            .param(PARAM_REFRESH_REQUEST_TIME, clock.nowUtc().minusHours(2).toString())
             .param(PARAM_DOMAINS, "example1.xn--q9jyb4c,example2.xn--q9jyb4c")
             .param(PARAM_HOSTS, "")
             .header("content-type", "application/x-www-form-urlencoded"),
@@ -313,7 +313,7 @@ public class PublishDnsUpdatesActionTest {
             .param(PARAM_LOCK_INDEX, "1")
             .param(PARAM_NUM_PUBLISH_LOCKS, "1")
             .param(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-            .param(PARAM_REFRESH_REQUEST_CREATED, clock.nowUtc().minusHours(2).toString())
+            .param(PARAM_REFRESH_REQUEST_TIME, clock.nowUtc().minusHours(2).toString())
             .param(PARAM_DOMAINS, "example3.xn--q9jyb4c,example4.xn--q9jyb4c")
             .param(PARAM_HOSTS, "ns1.example.xn--q9jyb4c")
             .header("content-type", "application/x-www-form-urlencoded"));
@@ -341,7 +341,7 @@ public class PublishDnsUpdatesActionTest {
             .param(PARAM_LOCK_INDEX, "1")
             .param(PARAM_NUM_PUBLISH_LOCKS, "1")
             .param(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-            .param(PARAM_REFRESH_REQUEST_CREATED, clock.nowUtc().minusHours(2).toString())
+            .param(PARAM_REFRESH_REQUEST_TIME, clock.nowUtc().minusHours(2).toString())
             .param(PARAM_DOMAINS, "example1.xn--q9jyb4c,example2.xn--q9jyb4c")
             .param(PARAM_HOSTS, "")
             .header("content-type", "application/x-www-form-urlencoded"),
@@ -352,7 +352,7 @@ public class PublishDnsUpdatesActionTest {
             .param(PARAM_LOCK_INDEX, "1")
             .param(PARAM_NUM_PUBLISH_LOCKS, "1")
             .param(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-            .param(PARAM_REFRESH_REQUEST_CREATED, clock.nowUtc().minusHours(2).toString())
+            .param(PARAM_REFRESH_REQUEST_TIME, clock.nowUtc().minusHours(2).toString())
             .param(PARAM_DOMAINS, "example3.xn--q9jyb4c,example4.xn--q9jyb4c,example5.xn--q9jyb4c")
             .param(PARAM_HOSTS, "ns1.example.xn--q9jyb4c")
             .header("content-type", "application/x-www-form-urlencoded"));
@@ -378,7 +378,7 @@ public class PublishDnsUpdatesActionTest {
             .param(PARAM_LOCK_INDEX, "1")
             .param(PARAM_NUM_PUBLISH_LOCKS, "1")
             .param(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-            .param(PARAM_REFRESH_REQUEST_CREATED, clock.nowUtc().minusHours(2).toString())
+            .param(PARAM_REFRESH_REQUEST_TIME, clock.nowUtc().minusHours(2).toString())
             .param(PARAM_DOMAINS, "example1.xn--q9jyb4c")
             .param(PARAM_HOSTS, "")
             .header("content-type", "application/x-www-form-urlencoded"),
@@ -389,7 +389,7 @@ public class PublishDnsUpdatesActionTest {
             .param(PARAM_LOCK_INDEX, "1")
             .param(PARAM_NUM_PUBLISH_LOCKS, "1")
             .param(PARAM_PUBLISH_TASK_ENQUEUED, clock.nowUtc().toString())
-            .param(PARAM_REFRESH_REQUEST_CREATED, clock.nowUtc().minusHours(2).toString())
+            .param(PARAM_REFRESH_REQUEST_TIME, clock.nowUtc().minusHours(2).toString())
             .param(PARAM_DOMAINS, "")
             .param(PARAM_HOSTS, "ns1.example.xn--q9jyb4c")
             .header("content-type", "application/x-www-form-urlencoded"));

core/src/test/java/google/registry/dns/ReadDnsQueueActionTest.java

@@ -144,7 +144,7 @@ public class ReadDnsQueueActionTest {
                     .url(PublishDnsUpdatesAction.PATH)
                     .param("tld", tldToDnsWriter.getKey())
                     .param("dnsWriter", tldToDnsWriter.getValue())
-                    .param("itemsCreated", "3000-01-01T00:00:00.000Z")
+                    .param("requestTime", "3000-01-01T00:00:00.000Z")
                     .param("enqueued", "3000-01-01T01:00:00.000Z")
                     // Single-lock TLDs should use lock 1 of 1 by default
                     .param("lockIndex", "1")
@@ -245,7 +245,7 @@ public class ReadDnsQueueActionTest {
         DNS_PUBLISH_PUSH_QUEUE_NAME,
         new TaskMatcher()
             .param("enqueued", "3000-02-05T01:00:00.000Z")
-            .param("itemsCreated", "3000-02-03T00:00:00.000Z")
+            .param("requestTime", "3000-02-03T00:00:00.000Z")
             .param("tld", "com")
             .param("dnsWriter", "comWriter")
             .param("domains", "domain1.com,domain2.com,domain3.com")
@@ -484,7 +484,7 @@ public class ReadDnsQueueActionTest {
             .url(PublishDnsUpdatesAction.PATH)
             .param("tld", "multilock.uk")
             .param("dnsWriter", "multilockWriter")
-            .param("itemsCreated", "3000-01-01T00:00:00.000Z")
+            .param("requestTime", "3000-01-01T00:00:00.000Z")
             .param("enqueued", "3000-01-01T01:00:00.000Z")
             .param("domains", "hello.multilock.uk")
             .param("hosts", "ns1.abc.hello.multilock.uk,ns2.hello.multilock.uk")
@@ -493,7 +493,7 @@ public class ReadDnsQueueActionTest {
             .url(PublishDnsUpdatesAction.PATH)
             .param("tld", "multilock.uk")
             .param("dnsWriter", "multilockWriter")
-            .param("itemsCreated", "3000-01-01T00:00:00.000Z")
+            .param("requestTime", "3000-01-01T00:00:00.000Z")
             .param("enqueued", "3000-01-01T01:00:00.000Z")
             .param("domains", "another.multilock.uk")
             .param("hosts", "ns3.def.another.multilock.uk,ns4.another.multilock.uk")

core/src/test/java/google/registry/dns/ReadDnsRefreshRequestsActionTest.java

@@ -0,0 +1,296 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.dns;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.common.DatabaseMigrationStateSchedule.MigrationState;
+import static google.registry.model.common.DatabaseMigrationStateSchedule.set;
+import static google.registry.model.common.DatabaseMigrationStateSchedule.useUncachedForTest;
+import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.loadAllOf;
+import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.testing.DatabaseHelper.persistResources;
+import static google.registry.util.DateTimeUtils.START_OF_TIME;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyCollection;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Ordering;
+import google.registry.dns.DnsConstants.TargetType;
+import google.registry.model.common.DnsRefreshRequest;
+import google.registry.model.common.DnsRefreshRequestTest;
+import google.registry.model.tld.Registry;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
+import google.registry.testing.CloudTasksHelper;
+import google.registry.testing.CloudTasksHelper.TaskMatcher;
+import google.registry.testing.FakeClock;
+import java.util.Collection;
+import java.util.Optional;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.Duration;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.mockito.ArgumentCaptor;
+
+/** Unit tests for {@link DnsRefreshRequestTest}. */
+public class ReadDnsRefreshRequestsActionTest {
+
+  private final FakeClock clock = new FakeClock(DateTime.parse("2020-02-02T01:23:45Z"));
+  private final CloudTasksHelper cloudTasksHelper = new CloudTasksHelper(clock);
+  private final DnsUtils dnsUtils = new DnsUtils(null);
+  private final Optional<Integer> jitterSeconds = Optional.of(5);
+
+  @RegisterExtension
+  JpaIntegrationTestExtension jpa =
+      new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension();
+
+  private final ReadDnsRefreshRequestsAction action =
+      spy(
+          new ReadDnsRefreshRequestsAction(
+              2,
+              Duration.standardSeconds(10),
+              jitterSeconds,
+              "tld",
+              clock,
+              dnsUtils,
+              null,
+              cloudTasksHelper.getTestCloudTasksUtils()));
+
+  private ImmutableList<DnsRefreshRequest> requests;
+
+  @BeforeAll
+  static void beforeAll() {
+    useUncachedForTest();
+  }
+
+  @BeforeEach
+  void beforeEach() {
+    useDnsSql();
+    persistResource(
+        createTld("tld")
+            .asBuilder()
+            .setDnsWriters(ImmutableSet.of("FooWriter", "BarWriter"))
+            .setNumDnsPublishLocks(2)
+            .build());
+    requests =
+        new ImmutableList.Builder<DnsRefreshRequest>()
+            .add(new DnsRefreshRequest(TargetType.DOMAIN, "domain.tld", "tld", clock.nowUtc()))
+            .add(
+                new DnsRefreshRequest(
+                    TargetType.HOST, "ns1.domain.tld", "tld", clock.nowUtc().minusMinutes(1)))
+            .add(
+                new DnsRefreshRequest(
+                    TargetType.DOMAIN, "future.tld", "tld", clock.nowUtc().plusMinutes(1)))
+            .build();
+    clock.advanceBy(Duration.standardMinutes(5));
+    persistResources(requests);
+    requests = loadAllOf(DnsRefreshRequest.class);
+  }
+
+  @Test
+  void testSuccess_runAction_pausedTld() {
+    persistResource(createTld("tld").asBuilder().setDnsPaused(true).build());
+    action.run();
+    verify(action, never()).enqueueUpdates(anyInt(), anyInt(), anyCollection());
+    verify(action, never()).processRequests(anyCollection());
+  }
+
+  @Test
+  void testSuccess_runAction_requestTimeInTheFuture() {
+    clock.setTo(DateTime.parse("2000-01-01T00:00:00Z"));
+    action.run();
+    verify(action, never()).enqueueUpdates(anyInt(), anyInt(), anyCollection());
+    verify(action, never()).processRequests(anyCollection());
+  }
+
+  @Test
+  void testSuccess_runAction_oneBatch() {
+    // The read batch size is 2 * 2 = 4. All requests will be in the same bucket, even though the
+    // bucket size should be roughly 2. But this is as expected because getLockIndex() only evenly
+    // distributes tasks in a statistical sense.
+    doReturn(2).when(action).getLockIndex(anyInt(), any(DnsRefreshRequest.class));
+    action.run();
+    @SuppressWarnings("unchecked")
+    ArgumentCaptor<Collection<DnsRefreshRequest>> captor =
+        ArgumentCaptor.forClass(Collection.class);
+    verify(action, times(1)).enqueueUpdates(eq(2), eq(2), captor.capture());
+    assertThat(captor.getValue().size()).isEqualTo(3);
+    verify(action, times(1)).processRequests(captor.capture());
+    assertThat(captor.getValue().size()).isEqualTo(3);
+    assertThat(loadAllOf(DnsRefreshRequest.class).isEmpty()).isTrue();
+  }
+
+  @Test
+  void testSuccess_runAction_twoBatches() {
+    // Make the read batch size 2 * 1 = 2.
+    persistResource(Registry.get("tld").asBuilder().setNumDnsPublishLocks(1).build());
+    doReturn(1).when(action).getLockIndex(anyInt(), any(DnsRefreshRequest.class));
+    doAnswer(
+            invocation -> {
+              @SuppressWarnings("unchecked")
+              ImmutableList<DnsRefreshRequest> ans =
+                  (ImmutableList<DnsRefreshRequest>) invocation.callRealMethod();
+              // The next read should not time out as we only increment by one millisecond, whereas
+              // the timeout is set to 10 seconds.
+              clock.advanceOneMilli();
+              return ans;
+            })
+        .when(action)
+        .processRequests(anyCollection());
+    action.run();
+    @SuppressWarnings("unchecked")
+    ArgumentCaptor<Collection<DnsRefreshRequest>> captor =
+        ArgumentCaptor.forClass(Collection.class);
+    verify(action, times(2)).enqueueUpdates(eq(1), eq(1), captor.capture());
+    assertThat(captor.getAllValues().get(0).size()).isEqualTo(2);
+    assertThat(captor.getAllValues().get(1).size()).isEqualTo(1);
+    verify(action, times(2)).processRequests(captor.capture());
+    assertThat(captor.getAllValues().get(0).size()).isEqualTo(2);
+    assertThat(captor.getAllValues().get(1).size()).isEqualTo(1);
+    assertThat(loadAllOf(DnsRefreshRequest.class).isEmpty()).isTrue();
+  }
+
+  @Test
+  void testSuccess_runAction_timeOutAfterFirstRead() {
+    // Make the process batch size 2 * 1 = 2.
+    persistResource(Registry.get("tld").asBuilder().setNumDnsPublishLocks(1).build());
+    // Both requests in the first batch will be bucketed to the same bucket.
+    doReturn(1).when(action).getLockIndex(anyInt(), any(DnsRefreshRequest.class));
+    doAnswer(
+            invocation -> {
+              @SuppressWarnings("unchecked")
+              ImmutableList<DnsRefreshRequest> ans =
+                  (ImmutableList<DnsRefreshRequest>) invocation.callRealMethod();
+              // After this function is called once, the loop in run() should top when it checks
+              // if the current time is before the request end time.
+              clock.advanceBy(Duration.standardHours(1));
+              return ans;
+            })
+        .when(action)
+        .processRequests(anyCollection());
+    action.run();
+    verify(action, times(1)).enqueueUpdates(anyInt(), anyInt(), anyCollection());
+    verify(action, times(1)).processRequests(anyCollection());
+    // The third request is left untouched because it is not read;
+    ImmutableList<DnsRefreshRequest> remainingRequests = loadAllOf(DnsRefreshRequest.class);
+    assertThat(remainingRequests.size()).isEqualTo(1);
+    assertThat(remainingRequests.get(0).getLastProcessTime()).isEqualTo(START_OF_TIME);
+  }
+
+  @Test
+  void testSuccess_processTasks() {
+    doReturn(2)
+        .doReturn(1)
+        .doReturn(2)
+        .when(action)
+        .getLockIndex(eq(2), any(DnsRefreshRequest.class));
+    action.processRequests(requests);
+    verify(action).enqueueUpdates(2, 2, ImmutableSet.of(requests.get(0), requests.get(2)));
+    verify(action).enqueueUpdates(1, 2, ImmutableSet.of(requests.get(1)));
+    assertThat(loadAllOf(DnsRefreshRequest.class)).isEmpty();
+  }
+
+  @Test
+  void testSuccess_processTasks_enqueueFailed_tasksNotDeleted() {
+    doReturn(2)
+        .doReturn(1)
+        .doReturn(2)
+        .when(action)
+        .getLockIndex(eq(2), any(DnsRefreshRequest.class));
+    doThrow(new RuntimeException("Something went wrong!"))
+        .when(action)
+        .enqueueUpdates(eq(2), eq(2), anyCollection());
+    action.processRequests(requests);
+    verify(action).enqueueUpdates(2, 2, ImmutableSet.of(requests.get(0), requests.get(2)));
+    verify(action).enqueueUpdates(1, 2, ImmutableSet.of(requests.get(1)));
+    assertThat(loadAllOf(DnsRefreshRequest.class).size()).isEqualTo(2);
+  }
+
+  @Test
+  void testSuccess_enqueueTasks() {
+    action.enqueueUpdates(2, 3, requests);
+    cloudTasksHelper.assertTasksEnqueued(
+        "dns-publish",
+        new TaskMatcher()
+            .url("/_dr/task/publishDnsUpdates")
+            .service("BACKEND")
+            .param("tld", "tld")
+            .param("dnsWriter", "FooWriter")
+            .param("lockIndex", "2")
+            .param("numPublishLocks", "3")
+            .param("enqueued", clock.nowUtc().toString())
+            .param("requestTime", clock.nowUtc().minusMinutes(6).toString())
+            .param("domains", "domain.tld,future.tld")
+            .param("hosts", "ns1.domain.tld"),
+        new TaskMatcher()
+            .url("/_dr/task/publishDnsUpdates")
+            .service("BACKEND")
+            .param("tld", "tld")
+            .param("dnsWriter", "BarWriter")
+            .param("lockIndex", "2")
+            .param("numPublishLocks", "3")
+            .param("enqueued", clock.nowUtc().toString())
+            .param("requestTime", clock.nowUtc().minusMinutes(6).toString())
+            .param("domains", "domain.tld,future.tld")
+            .param("hosts", "ns1.domain.tld"));
+    cloudTasksHelper
+        .getTestTasksFor("dns-publish")
+        .forEach(
+            task -> {
+              DateTime scheduledTime =
+                  new DateTime(task.getScheduleTime().getSeconds() * 1000, DateTimeZone.UTC);
+              assertThat(new Duration(clock.nowUtc(), scheduledTime))
+                  .isAtMost(Duration.standardSeconds(jitterSeconds.get()));
+            });
+  }
+
+  private void useDnsSql() {
+    DateTime currentTime = clock.nowUtc();
+    clock.setTo(START_OF_TIME);
+    tm().transact(
+            () ->
+                set(
+                    new ImmutableSortedMap.Builder<DateTime, MigrationState>(Ordering.natural())
+                        .put(START_OF_TIME, MigrationState.DATASTORE_ONLY)
+                        .put(START_OF_TIME.plusMillis(1), MigrationState.DATASTORE_PRIMARY)
+                        .put(START_OF_TIME.plusMillis(2), MigrationState.DATASTORE_PRIMARY_NO_ASYNC)
+                        .put(
+                            START_OF_TIME.plusMillis(3), MigrationState.DATASTORE_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusMillis(4), MigrationState.SQL_PRIMARY_READ_ONLY)
+                        .put(START_OF_TIME.plusMillis(5), MigrationState.SQL_PRIMARY)
+                        .put(START_OF_TIME.plusMillis(6), MigrationState.SQL_ONLY)
+                        .put(START_OF_TIME.plusMillis(7), MigrationState.SEQUENCE_BASED_ALLOCATE_ID)
+                        .put(START_OF_TIME.plusMillis(8), MigrationState.NORDN_SQL)
+                        .put(START_OF_TIME.plusMillis(9), MigrationState.DNS_SQL)
+                        .build()));
+    clock.setTo(currentTime);
+  }
+}

core/src/test/java/google/registry/flows/domain/DomainCheckFlowTest.java

@@ -75,6 +75,7 @@ import google.registry.model.billing.BillingEvent.Flag;
 import google.registry.model.billing.BillingEvent.Reason;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainHistory;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.eppcommon.StatusValue;
@@ -792,6 +793,7 @@ class DomainCheckFlowTest extends ResourceCheckFlowTestCase<DomainCheckFlow, Dom
                 .setTokenType(DEFAULT_PROMO)
                 .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar"))
                 .setAllowedTlds(ImmutableSet.of("tld"))
+                .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE))
                 .setDiscountFraction(0.5)
                 .build());
     persistResource(
@@ -875,6 +877,19 @@ class DomainCheckFlowTest extends ResourceCheckFlowTestCase<DomainCheckFlow, Dom
     runFlowAssertResponse(loadFile("domain_check_fee_multiple_commands_response_v06.xml"));
   }
 
+  @Test
+  void testFeeExtension_multipleCommands_tokenNotValidForSome_v06() throws Exception {
+    persistResource(
+        new AllocationToken.Builder()
+            .setToken("abc123")
+            .setTokenType(UNLIMITED_USE)
+            .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE, CommandName.TRANSFER))
+            .build());
+    setEppInput("domain_check_fee_multiple_commands_allocationtoken_v06.xml");
+    runFlowAssertResponse(
+        loadFile("domain_check_fee_multiple_commands_allocationtoken_response_v06.xml"));
+  }
+
   @Test
   void testFeeExtension_multipleCommands_defaultTokenOnlyOnCreate_v06() throws Exception {
     setUpDefaultToken();
@@ -891,6 +906,19 @@ class DomainCheckFlowTest extends ResourceCheckFlowTestCase<DomainCheckFlow, Dom
     runFlowAssertResponse(loadFile("domain_check_fee_multiple_commands_response_v12.xml"));
   }
 
+  @Test
+  void testFeeExtension_multipleCommands_tokenNotValidForSome_v12() throws Exception {
+    persistResource(
+        new AllocationToken.Builder()
+            .setToken("abc123")
+            .setTokenType(UNLIMITED_USE)
+            .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE, CommandName.TRANSFER))
+            .build());
+    setEppInput("domain_check_fee_multiple_commands_allocationtoken_v12.xml");
+    runFlowAssertResponse(
+        loadFile("domain_check_fee_multiple_commands_allocationtoken_response_v12.xml"));
+  }
+
   @Test
   void testFeeExtension_multipleCommands_defaultTokenOnlyOnCreate_v12() throws Exception {
     setUpDefaultToken();

core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java

@@ -1236,4 +1236,16 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
     EppException thrown = assertThrows(UnimplementedExtensionException.class, this::runFlow);
     assertAboutEppExceptions().that(thrown).marshalsToXml();
   }
+
+  @Test
+  void testSuccess_freeCreation_deletionDuringGracePeriod() throws Exception {
+    // Deletion during the add grace period should still work even if the credit is 0
+    setUpSuccessfulTest();
+    BillingEvent.OneTime graceBillingEvent =
+        persistResource(createBillingEvent(Reason.CREATE, Money.of(USD, 0)));
+    setUpGracePeriods(
+        GracePeriod.forBillingEvent(GracePeriodStatus.ADD, domain.getRepoId(), graceBillingEvent));
+    clock.advanceOneMilli();
+    runFlowAssertResponse(loadFile("domain_delete_response_fee_free_grace.xml"));
+  }
 }

core/src/test/java/google/registry/flows/domain/DomainRenewFlowTest.java

@@ -20,6 +20,7 @@ import static google.registry.flows.domain.DomainTransferFlowTestCase.persistWit
 import static google.registry.model.billing.BillingEvent.RenewalPriceBehavior.DEFAULT;
 import static google.registry.model.billing.BillingEvent.RenewalPriceBehavior.NONPREMIUM;
 import static google.registry.model.billing.BillingEvent.RenewalPriceBehavior.SPECIFIED;
+import static google.registry.model.domain.token.AllocationToken.TokenType.DEFAULT_PROMO;
 import static google.registry.model.domain.token.AllocationToken.TokenType.PACKAGE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.SINGLE_USE;
 import static google.registry.model.domain.token.AllocationToken.TokenType.UNLIMITED_USE;
@@ -46,6 +47,7 @@ import static org.joda.money.CurrencyUnit.JPY;
 import static org.joda.money.CurrencyUnit.USD;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
@@ -85,6 +87,7 @@ import google.registry.model.billing.BillingEvent.RenewalPriceBehavior;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainHistory;
 import google.registry.model.domain.GracePeriod;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.rgp.GracePeriodStatus;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
@@ -1374,4 +1377,416 @@ class DomainRenewFlowTest extends ResourceFlowTestCase<DomainRenewFlow, Domain>
             "domain_renew_response.xml",
             ImmutableMap.of("DOMAIN", "example.tld", "EXDATE", "2002-04-03T22:00:00.0Z")));
   }
+
+  @Test
+  void testSuccess_usesDefaultToken() throws Exception {
+    setEppInput("domain_renew.xml", ImmutableMap.of("DOMAIN", "example.tld", "YEARS", "2"));
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    AllocationToken defaultToken2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("bbbbb")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.75)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(
+                ImmutableList.of(defaultToken1.createVKey(), defaultToken2.createVKey()))
+            .build());
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response.xml",
+            ImmutableMap.of("DOMAIN", "example.tld", "EXDATE", "2002-04-03T22:00:00.0Z")));
+    BillingEvent.OneTime billingEvent =
+        Iterables.getOnlyElement(DatabaseHelper.loadAllOf(BillingEvent.OneTime.class));
+    assertThat(billingEvent.getTargetId()).isEqualTo("example.tld");
+    assertThat(billingEvent.getAllocationToken().get().getKey())
+        .isEqualTo(defaultToken1.getToken());
+    // Price is 50% off the first year only. Non-discounted price is $11.
+    assertThat(billingEvent.getCost()).isEqualTo(Money.of(USD, 16.5));
+  }
+
+  @Test
+  void testSuccess_doesNotUseDefaultTokenWhenTokenPassedIn() throws Exception {
+    setEppInput(
+        "domain_renew_allocationtoken.xml",
+        ImmutableMap.of("DOMAIN", "example.tld", "YEARS", "2", "TOKEN", "abc123"));
+    persistDomain();
+    AllocationToken allocationToken =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("abc123")
+                .setTokenType(SINGLE_USE)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .build());
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.25)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    AllocationToken defaultToken2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("bbbbb")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.75)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(
+                ImmutableList.of(defaultToken1.createVKey(), defaultToken2.createVKey()))
+            .build());
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response.xml",
+            ImmutableMap.of("DOMAIN", "example.tld", "EXDATE", "2002-04-03T22:00:00.0Z")));
+    BillingEvent.OneTime billingEvent =
+        Iterables.getOnlyElement(DatabaseHelper.loadAllOf(BillingEvent.OneTime.class));
+    assertThat(billingEvent.getTargetId()).isEqualTo("example.tld");
+    assertThat(billingEvent.getAllocationToken().get().getKey())
+        .isEqualTo(allocationToken.getToken());
+    // Price is 50% off the first year only. Non-discounted price is $11.
+    assertThat(billingEvent.getCost()).isEqualTo(Money.of(USD, 16.5));
+  }
+
+  @Test
+  void testSuccess_noValidDefaultToken() throws Exception {
+    setEppInput("domain_renew.xml", ImmutableMap.of("DOMAIN", "example.tld", "YEARS", "2"));
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedRegistrarIds(ImmutableSet.of("OtherRegistrar"))
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    AllocationToken defaultToken2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("bbbbb")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.75)
+                .setDiscountYears(1)
+                .setAllowedRegistrarIds(ImmutableSet.of("OtherRegistrar"))
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(
+                ImmutableList.of(defaultToken1.createVKey(), defaultToken2.createVKey()))
+            .build());
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response.xml",
+            ImmutableMap.of("DOMAIN", "example.tld", "EXDATE", "2002-04-03T22:00:00.0Z")));
+    BillingEvent.OneTime billingEvent =
+        Iterables.getOnlyElement(DatabaseHelper.loadAllOf(BillingEvent.OneTime.class));
+    assertThat(billingEvent.getTargetId()).isEqualTo("example.tld");
+    assertThat(billingEvent.getAllocationToken()).isEmpty();
+    assertThat(billingEvent.getCost()).isEqualTo(Money.of(USD, 22));
+  }
+
+  @Test
+  void testSuccess_onlyUsesFirstValidToken() throws Exception {
+    setEppInput("domain_renew.xml", ImmutableMap.of("DOMAIN", "example.tld", "YEARS", "2"));
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.25)
+                .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE))
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    AllocationToken defaultToken2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("bbbbb")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    AllocationToken defaultToken3 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("ccccc")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.75)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(
+                ImmutableList.of(
+                    defaultToken1.createVKey(),
+                    defaultToken2.createVKey(),
+                    defaultToken3.createVKey()))
+            .build());
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response.xml",
+            ImmutableMap.of("DOMAIN", "example.tld", "EXDATE", "2002-04-03T22:00:00.0Z")));
+    BillingEvent.OneTime billingEvent =
+        Iterables.getOnlyElement(DatabaseHelper.loadAllOf(BillingEvent.OneTime.class));
+    assertThat(billingEvent.getTargetId()).isEqualTo("example.tld");
+    assertThat(billingEvent.getAllocationToken().get().getKey())
+        .isEqualTo(defaultToken2.getToken());
+    // Price is 50% off the first year only. Non-discounted price is $11.
+    assertThat(billingEvent.getCost()).isEqualTo(Money.of(USD, 16.5));
+  }
+
+  @Test
+  void testSuccess_registryHasDeletedDefaultToken() throws Exception {
+    setEppInput("domain_renew.xml", ImmutableMap.of("DOMAIN", "example.tld", "YEARS", "2"));
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    AllocationToken defaultToken2 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("bbbbb")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.75)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(
+                ImmutableList.of(defaultToken1.createVKey(), defaultToken2.createVKey()))
+            .build());
+    DatabaseHelper.deleteResource(defaultToken1);
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response.xml",
+            ImmutableMap.of("DOMAIN", "example.tld", "EXDATE", "2002-04-03T22:00:00.0Z")));
+    BillingEvent.OneTime billingEvent =
+        Iterables.getOnlyElement(DatabaseHelper.loadAllOf(BillingEvent.OneTime.class));
+    assertThat(billingEvent.getTargetId()).isEqualTo("example.tld");
+    assertThat(billingEvent.getAllocationToken().get().getKey())
+        .isEqualTo(defaultToken2.getToken());
+    // Price is 75% off the first year only. Non-discounted price is $11.
+    assertThat(billingEvent.getCost()).isEqualTo(Money.of(USD, 13.75));
+  }
+
+  @Test
+  void testSuccess_wrongFeeAmountTooHigh_defaultToken_v06() throws Exception {
+    setEppInput("domain_renew_fee.xml", FEE_06_MAP);
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(ImmutableList.of(defaultToken1.createVKey()))
+            .build());
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response_fee.xml",
+            ImmutableMap.of(
+                "NAME",
+                "example.tld",
+                "PERIOD",
+                "5",
+                "EX_DATE",
+                "2005-04-03T22:00:00.0Z",
+                "FEE",
+                "49.50",
+                "CURRENCY",
+                "USD",
+                "FEE_VERSION",
+                "0.6",
+                "FEE_NS",
+                "fee")));
+  }
+
+  @Test
+  void testFailure_wrongFeeAmountTooLow_defaultToken_v06() throws Exception {
+    setEppInput("domain_renew_fee.xml", FEE_06_MAP);
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(ImmutableList.of(defaultToken1.createVKey()))
+            .setRenewBillingCostTransitions(ImmutableSortedMap.of(START_OF_TIME, Money.of(USD, 20)))
+            .build());
+    EppException thrown = assertThrows(FeesMismatchException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
+  @Test
+  void testSuccess_wrongFeeAmountTooHigh_defaultToken_v11() throws Exception {
+    setEppInput("domain_renew_fee.xml", FEE_11_MAP);
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(ImmutableList.of(defaultToken1.createVKey()))
+            .build());
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response_fee.xml",
+            ImmutableMap.of(
+                "NAME",
+                "example.tld",
+                "PERIOD",
+                "5",
+                "EX_DATE",
+                "2005-04-03T22:00:00.0Z",
+                "FEE",
+                "49.50",
+                "CURRENCY",
+                "USD",
+                "FEE_VERSION",
+                "0.11",
+                "FEE_NS",
+                "fee")));
+  }
+
+  @Test
+  void testFailure_wrongFeeAmountTooLow_defaultToken_v11() throws Exception {
+    setEppInput("domain_renew_fee.xml", FEE_11_MAP);
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(ImmutableList.of(defaultToken1.createVKey()))
+            .setRenewBillingCostTransitions(ImmutableSortedMap.of(START_OF_TIME, Money.of(USD, 20)))
+            .build());
+    EppException thrown = assertThrows(FeesMismatchException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
+
+  @Test
+  void testSuccess_wrongFeeAmountTooHigh_defaultToken_v12() throws Exception {
+    setEppInput("domain_renew_fee.xml", FEE_12_MAP);
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(ImmutableList.of(defaultToken1.createVKey()))
+            .build());
+    runFlowAssertResponse(
+        loadFile(
+            "domain_renew_response_fee.xml",
+            ImmutableMap.of(
+                "NAME",
+                "example.tld",
+                "PERIOD",
+                "5",
+                "EX_DATE",
+                "2005-04-03T22:00:00.0Z",
+                "FEE",
+                "49.50",
+                "CURRENCY",
+                "USD",
+                "FEE_VERSION",
+                "0.12",
+                "FEE_NS",
+                "fee")));
+  }
+
+  @Test
+  void testFailure_wrongFeeAmountTooLow_defaultToken_v12() throws Exception {
+    setEppInput("domain_renew_fee.xml", FEE_06_MAP);
+    persistDomain();
+    AllocationToken defaultToken1 =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("aaaaa")
+                .setTokenType(DEFAULT_PROMO)
+                .setDiscountFraction(0.5)
+                .setDiscountYears(1)
+                .setAllowedTlds(ImmutableSet.of("tld"))
+                .build());
+    persistResource(
+        Registry.get("tld")
+            .asBuilder()
+            .setDefaultPromoTokens(ImmutableList.of(defaultToken1.createVKey()))
+            .setRenewBillingCostTransitions(ImmutableSortedMap.of(START_OF_TIME, Money.of(USD, 20)))
+            .build());
+    EppException thrown = assertThrows(FeesMismatchException.class, this::runFlow);
+    assertAboutEppExceptions().that(thrown).marshalsToXml();
+  }
 }

core/src/test/java/google/registry/flows/domain/token/AllocationTokenFlowUtilsTest.java

@@ -39,11 +39,13 @@ import com.google.common.collect.Maps;
 import com.google.common.net.InternetDomainName;
 import google.registry.flows.EppException;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotInPromotionException;
+import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForCommandException;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForRegistrarException;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils.AllocationTokenNotValidForTldException;
 import google.registry.flows.domain.token.AllocationTokenFlowUtils.InvalidAllocationTokenException;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.DomainCommand;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.domain.token.AllocationTokenExtension;
@@ -80,7 +82,11 @@ class AllocationTokenFlowUtilsTest {
   void test_validateToken_successfullyVerifiesValidTokenOnCreate() throws Exception {
     AllocationToken token =
         persistResource(
-            new AllocationToken.Builder().setToken("tokeN").setTokenType(SINGLE_USE).build());
+            new AllocationToken.Builder()
+                .setToken("tokeN")
+                .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE, CommandName.RESTORE))
+                .setTokenType(SINGLE_USE)
+                .build());
     when(allocationTokenExtension.getAllocationToken()).thenReturn("tokeN");
     assertThat(
             flowUtils
@@ -96,6 +102,29 @@ class AllocationTokenFlowUtilsTest {
 
   @Test
   void test_validateToken_successfullyVerifiesValidTokenExistingDomain() throws Exception {
+    AllocationToken token =
+        persistResource(
+            new AllocationToken.Builder()
+                .setToken("tokeN")
+                .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE, CommandName.RENEW))
+                .setTokenType(SINGLE_USE)
+                .build());
+    when(allocationTokenExtension.getAllocationToken()).thenReturn("tokeN");
+    assertThat(
+            flowUtils
+                .verifyAllocationTokenIfPresent(
+                    DatabaseHelper.newDomain("blah.tld"),
+                    Registry.get("tld"),
+                    "TheRegistrar",
+                    DateTime.now(UTC),
+                    CommandName.RENEW,
+                    Optional.of(allocationTokenExtension))
+                .get())
+        .isEqualTo(token);
+  }
+
+  void test_validateToken_emptyAllowedEppActions_successfullyVerifiesValidTokenExistingDomain()
+      throws Exception {
     AllocationToken token =
         persistResource(
             new AllocationToken.Builder().setToken("tokeN").setTokenType(SINGLE_USE).build());
@@ -107,6 +136,7 @@ class AllocationTokenFlowUtilsTest {
                     Registry.get("tld"),
                     "TheRegistrar",
                     DateTime.now(UTC),
+                    CommandName.RENEW,
                     Optional.of(allocationTokenExtension))
                 .get())
         .isEqualTo(token);
@@ -150,6 +180,7 @@ class AllocationTokenFlowUtilsTest {
                         Registry.get("tld"),
                         "TheRegistrar",
                         DateTime.now(UTC),
+                        CommandName.RENEW,
                         Optional.of(allocationTokenExtension))))
         .marshalsToXml();
   }
@@ -190,6 +221,7 @@ class AllocationTokenFlowUtilsTest {
                     Registry.get("tld"),
                     "TheRegistrar",
                     DateTime.now(UTC),
+                    CommandName.RENEW,
                     Optional.of(allocationTokenExtension)));
     assertThat(thrown).hasMessageThat().isEqualTo("failed for tests");
   }
@@ -285,6 +317,25 @@ class AllocationTokenFlowUtilsTest {
     assertValidateExistingDomainThrowsEppException(AllocationTokenNotInPromotionException.class);
   }
 
+  @Test
+  void test_validateTokenCreate_invalidCommand() {
+    persistResource(
+        createOneMonthPromoTokenBuilder(DateTime.now(UTC).minusDays(1))
+            .setAllowedEppActions(ImmutableSet.of(CommandName.RENEW))
+            .build());
+    assertValidateCreateThrowsEppException(AllocationTokenNotValidForCommandException.class);
+  }
+
+  @Test
+  void test_validateTokenExistingDomain_invalidCommand() {
+    persistResource(
+        createOneMonthPromoTokenBuilder(DateTime.now(UTC).minusDays(1))
+            .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE))
+            .build());
+    assertValidateExistingDomainThrowsEppException(
+        AllocationTokenNotValidForCommandException.class);
+  }
+
   @Test
   void test_checkDomainsWithToken_successfullyVerifiesValidToken() {
     persistResource(
@@ -401,6 +452,7 @@ class AllocationTokenFlowUtilsTest {
                         Registry.get("tld"),
                         "TheRegistrar",
                         DateTime.now(UTC),
+                        CommandName.RENEW,
                         Optional.of(allocationTokenExtension))))
         .marshalsToXml();
   }

core/src/test/java/google/registry/persistence/transaction/JpaTransactionManagerImplTest.java

@@ -580,6 +580,32 @@ class JpaTransactionManagerImplTest {
                     .isFalse());
   }
 
+  @Test
+  void innerTransactions_noRetry() {
+    JpaTransactionManager spyJpaTm = spy(tm());
+    doThrow(OptimisticLockException.class).when(spyJpaTm).delete(any(VKey.class));
+    spyJpaTm.transact(() -> spyJpaTm.insert(theEntity));
+
+    Supplier<Runnable> supplier =
+        () -> {
+          Runnable work = () -> spyJpaTm.delete(theEntityKey);
+          work.run();
+          return null;
+        };
+
+    assertThrows(
+        OptimisticLockException.class,
+        () ->
+            spyJpaTm.transact(
+                () -> {
+                  spyJpaTm.exists(theEntity);
+                  spyJpaTm.transact(supplier);
+                }));
+
+    verify(spyJpaTm, times(3)).exists(theEntity);
+    verify(spyJpaTm, times(3)).delete(theEntityKey);
+  }
+
   private void insertPerson(int age) {
     tm().getEntityManager()
         .createNativeQuery(String.format("INSERT INTO Person (age) VALUES (%d)", age))

core/src/test/java/google/registry/tldconfig/idn/IdnLabelValidatorTest.java

@@ -15,17 +15,26 @@
 package google.registry.tldconfig.idn;
 
 import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.persistResource;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
 /** Unit tests for {@link IdnLabelValidator}. */
 class IdnLabelValidatorTest {
 
-  private IdnLabelValidator idnLabelValidator = IdnLabelValidator.createDefaultIdnLabelValidator();
+  @RegisterExtension
+  final JpaIntegrationTestExtension jpa =
+      new JpaTestExtensions.Builder().buildIntegrationTestExtension();
 
-  private void doJapaneseLanguageTests(String tld) {
+  private IdnLabelValidator idnLabelValidator = new IdnLabelValidator();
+
+  private void doJapaneseAndLatinLanguageTests(String tld) {
+    createTld(tld);
     assertThat(idnLabelValidator.findValidIdnTableForTld("foo", tld)).isPresent();
     assertThat(idnLabelValidator.findValidIdnTableForTld("12379foar", tld)).isPresent();
     assertThat(idnLabelValidator.findValidIdnTableForTld("みんな", tld)).isPresent();
@@ -84,26 +93,29 @@ class IdnLabelValidatorTest {
 
   @Test
   void testMinna() {
-    doJapaneseLanguageTests("xn--q9jyb4c");
+    doJapaneseAndLatinLanguageTests("xn--q9jyb4c");
   }
 
   @Test
   void testFoo() {
-    doJapaneseLanguageTests("foo");
+    doJapaneseAndLatinLanguageTests("foo");
   }
 
   @Test
   void testSoy() {
-    doJapaneseLanguageTests("soy");
+    doJapaneseAndLatinLanguageTests("soy");
   }
 
   @Test
-  void testOverridenTables() {
-    // Set .tld to have only the extended latin table and not japanese.
-    idnLabelValidator =
-        new IdnLabelValidator(
-            ImmutableMap.of("tld", ImmutableList.of(IdnTableEnum.EXTENDED_LATIN)));
+  void testPerTldConfig() {
+    persistResource(
+        createTld("tld")
+            .asBuilder()
+            .setIdnTables(ImmutableSet.of(IdnTableEnum.EXTENDED_LATIN))
+            .build());
     assertThat(idnLabelValidator.findValidIdnTableForTld("foo", "tld")).isPresent();
+    assertThat(idnLabelValidator.findValidIdnTableForTld("abcdefghæ", "tld")).isPresent();
+    // Extended Latin shouldn't include Japanese characters
     assertThat(idnLabelValidator.findValidIdnTableForTld("みんな", "tld")).isEmpty();
   }
 }

core/src/test/java/google/registry/tmch/NordnUploadActionTest.java

@@ -29,7 +29,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
 import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
 import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.startsWith;
@@ -39,15 +38,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
 
-import com.google.appengine.api.taskqueue.LeaseOptions;
-import com.google.appengine.api.taskqueue.Queue;
-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
-import com.google.appengine.api.taskqueue.TransientFailureException;
-import com.google.apphosting.api.DeadlineExceededException;
 import com.google.common.base.VerifyException;
-import com.google.common.collect.ImmutableList;
 import google.registry.batch.CloudTasksUtils;
 import google.registry.model.domain.Domain;
 import google.registry.model.domain.launch.LaunchNotice;
@@ -69,7 +60,6 @@ import java.io.ByteArrayOutputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
 import java.security.SecureRandom;
-import java.util.List;
 import java.util.Optional;
 import org.joda.time.DateTime;
 import org.joda.time.Duration;
@@ -139,63 +129,6 @@ class NordnUploadActionTest {
     action.retrier = new Retrier(new FakeSleeper(clock), 3);
   }
 
-  @Test
-  void test_convertTasksToCsv() {
-    List<TaskHandle> tasks =
-        ImmutableList.of(
-            makeTaskHandle("task2", "example", "csvLine2", "lordn-sunrise"),
-            makeTaskHandle("task1", "example", "csvLine1", "lordn-sunrise"),
-            makeTaskHandle("task3", "example", "ending", "lordn-sunrise"));
-    assertThat(NordnUploadAction.convertTasksToCsv(tasks, clock.nowUtc(), "col1,col2"))
-        .isEqualTo("1,2010-05-04T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
-  }
-
-  @Test
-  void test_convertTasksToCsv_dedupesDuplicates() {
-    List<TaskHandle> tasks =
-        ImmutableList.of(
-            makeTaskHandle("task2", "example", "csvLine2", "lordn-sunrise"),
-            makeTaskHandle("task1", "example", "csvLine1", "lordn-sunrise"),
-            makeTaskHandle("task3", "example", "ending", "lordn-sunrise"),
-            makeTaskHandle("task1", "example", "csvLine1", "lordn-sunrise"));
-    assertThat(NordnUploadAction.convertTasksToCsv(tasks, clock.nowUtc(), "col1,col2"))
-        .isEqualTo("1,2010-05-04T10:11:12.000Z,3\ncol1,col2\ncsvLine1\ncsvLine2\nending\n");
-  }
-
-  @Test
-  void test_convertTasksToCsv_doesntFailOnEmptyTasks() {
-    assertThat(NordnUploadAction.convertTasksToCsv(ImmutableList.of(), clock.nowUtc(), "col1,col2"))
-        .isEqualTo("1,2010-05-04T10:11:12.000Z,0\ncol1,col2\n");
-  }
-
-  @Test
-  void test_convertTasksToCsv_throwsNpeOnNullTasks() {
-    assertThrows(
-        NullPointerException.class,
-        () -> NordnUploadAction.convertTasksToCsv(null, clock.nowUtc(), "header"));
-  }
-
-  @Test
-  void test_loadAllTasks_retryLogic_thirdTrysTheCharm() {
-    Queue queue = mock(Queue.class);
-    TaskHandle task = new TaskHandle(TaskOptions.Builder.withTaskName("blah"), "blah");
-    when(queue.leaseTasks(any(LeaseOptions.class)))
-        .thenThrow(TransientFailureException.class)
-        .thenThrow(DeadlineExceededException.class)
-        .thenReturn(ImmutableList.of(task), ImmutableList.of());
-    assertThat(action.loadAllTasks(queue, "tld")).containsExactly(task);
-  }
-
-  @Test
-  void test_loadAllTasks_retryLogic_allFailures() {
-    Queue queue = mock(Queue.class);
-    when(queue.leaseTasks(any(LeaseOptions.class)))
-        .thenThrow(new TransientFailureException("some transient error"));
-    RuntimeException thrown =
-        assertThrows(TransientFailureException.class, () -> action.loadAllTasks(queue, "tld"));
-    assertThat(thrown).hasMessageThat().isEqualTo("some transient error");
-  }
-
   @Test
   void testSuccess_noPassword_doesntSendAuthorizationHeader() {
     action.lordnRequestInitializer = new LordnRequestInitializer(Optional.empty());
@@ -296,13 +229,6 @@ class NordnUploadActionTest {
             .build());
   }
 
-  private static TaskHandle makeTaskHandle(
-      String taskName, String tag, String payload, String queue) {
-    return new TaskHandle(
-        TaskOptions.Builder.withPayload(payload).method(Method.PULL).tag(tag).taskName(taskName),
-        queue);
-  }
-
   private void verifyColumnCleared(String domainName) {
     VKey<Domain> domainKey = load(Domain.class, domainName, clock.nowUtc());
     Domain domain = loadByKey(domainKey);

core/src/test/java/google/registry/tools/CreateTldCommandTest.java

@@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Range;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.tld.Registry;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import java.math.BigDecimal;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -544,6 +545,35 @@ class CreateTldCommandTest extends CommandTestCase<CreateTldCommand> {
     assertThat(Registry.get("xn--q9jyb4c").getDriveFolderId()).isNull();
   }
 
+  @Test
+  void testSuccess_setsIdnTables() throws Exception {
+    runCommandForced(
+        "--idn_tables=extended_latin,ja",
+        "--roid_suffix=ASDF",
+        "--dns_writers=VoidDnsWriter",
+        "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables())
+        .containsExactly(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA);
+  }
+
+  @Test
+  void testFailure_invalidIdnTable() throws Exception {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--idn_tables=extended_latin,bad_value",
+                    "--roid_suffix=ASDF",
+                    "--dns_writers=VoidDnsWriter",
+                    "xn--q9jyb4c"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "IDN tables [EXTENDED_LATIN, BAD_VALUE] contained invalid value(s). Possible values:"
+                + " [EXTENDED_LATIN, UNCONFUSABLE_LATIN, JA]");
+  }
+
   @Test
   void testFailure_setPremiumListThatDoesntExist() {
     IllegalArgumentException thrown =

core/src/test/java/google/registry/tools/GenerateAllocationTokensCommandTest.java

@@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import com.google.common.collect.Iterables;
 import com.google.common.io.Files;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
 import google.registry.model.reporting.HistoryEntry.HistoryEntryId;
@@ -136,6 +137,7 @@ class GenerateAllocationTokensCommandTest extends CommandTestCase<GenerateAlloca
         "--type", "UNLIMITED_USE",
         "--allowed_client_ids", "TheRegistrar,NewRegistrar",
         "--allowed_tlds", "tld,example",
+        "--allowed_epp_actions", "CREATE,RENEW",
         "--discount_fraction", "0.5",
         "--discount_premiums", "true",
         "--discount_years", "6",
@@ -148,6 +150,7 @@ class GenerateAllocationTokensCommandTest extends CommandTestCase<GenerateAlloca
             .setTokenType(UNLIMITED_USE)
             .setAllowedRegistrarIds(ImmutableSet.of("TheRegistrar", "NewRegistrar"))
             .setAllowedTlds(ImmutableSet.of("tld", "example"))
+            .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE, CommandName.RENEW))
             .setDiscountFraction(0.5)
             .setDiscountPremiums(true)
             .setDiscountYears(6)

core/src/test/java/google/registry/tools/UpdateAllocationTokensCommandTest.java

@@ -36,6 +36,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 import com.beust.jcommander.ParameterException;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
+import google.registry.model.domain.fee.FeeQueryCommandExtensionItem.CommandName;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.domain.token.AllocationToken.RegistrationBehavior;
 import google.registry.model.domain.token.AllocationToken.TokenStatus;
@@ -80,6 +81,57 @@ class UpdateAllocationTokensCommandTest extends CommandTestCase<UpdateAllocation
     assertThat(reloadResource(token).getAllowedRegistrarIds()).isEmpty();
   }
 
+  @Test
+  void testUpdateEppActions_setEppActions() throws Exception {
+    AllocationToken token =
+        persistResource(
+            builderWithPromo().setAllowedEppActions(ImmutableSet.of(CommandName.CREATE)).build());
+    runCommandForced("--prefix", "token", "--allowed_epp_actions", "RENEW,RESTORE");
+    assertThat(reloadResource(token).getAllowedEppActions())
+        .containsExactly(CommandName.RENEW, CommandName.RESTORE);
+  }
+
+  @Test
+  void testUpdateEppActions_clearEppActions() throws Exception {
+    AllocationToken token =
+        persistResource(
+            builderWithPromo()
+                .setAllowedEppActions(ImmutableSet.of(CommandName.CREATE, CommandName.RENEW))
+                .build());
+    runCommandForced("--prefix", "token", "--allowed_epp_actions", "");
+    assertThat(reloadResource(token).getAllowedEppActions()).isEmpty();
+  }
+
+  @Test
+  void testUpdateEppActions_invalidEppAction() throws Exception {
+    persistResource(
+        builderWithPromo().setAllowedEppActions(ImmutableSet.of(CommandName.CREATE)).build());
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> runCommandForced("--prefix", "token", "--allowed_epp_actions", "FAKE"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Invalid EPP action name. Valid actions are CREATE, RENEW, TRANSFER, RESTORE, and"
+                + " UPDATE");
+  }
+
+  @Test
+  void testUpdateEppActions_unknownEppAction() throws Exception {
+    persistResource(
+        builderWithPromo().setAllowedEppActions(ImmutableSet.of(CommandName.CREATE)).build());
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> runCommandForced("--prefix", "token", "--allowed_epp_actions", "UNKNOWN"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "Invalid EPP action name. Valid actions are CREATE, RENEW, TRANSFER, RESTORE, and"
+                + " UPDATE");
+  }
+
   @Test
   void testUpdateDiscountFraction() throws Exception {
     AllocationToken token = persistResource(builderWithPromo().setDiscountFraction(0.5).build());

core/src/test/java/google/registry/tools/UpdateTldCommandTest.java

@@ -39,6 +39,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.tld.Registry;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -1053,6 +1054,38 @@ class UpdateTldCommandTest extends CommandTestCase<UpdateTldCommand> {
     assertThat(Registry.get("xn--q9jyb4c").getDriveFolderId()).isNull();
   }
 
+  @Test
+  void testSuccess_setsIdnTables() throws Exception {
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables()).isEmpty();
+    runCommandForced("--idn_tables=extended_latin,ja", "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables())
+        .containsExactly(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA);
+  }
+
+  @Test
+  void testSuccess_removesIndTables() throws Exception {
+    persistResource(
+        Registry.get("xn--q9jyb4c")
+            .asBuilder()
+            .setIdnTables(ImmutableSet.of(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA))
+            .build());
+    runCommandForced("--idn_tables=", "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables()).isEmpty();
+  }
+
+  @Test
+  void testFailure_invalidIdnTable() throws Exception {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> runCommandForced("--idn_tables=extended_latin,bad_value", "xn--q9jyb4c"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "IDN tables [EXTENDED_LATIN, BAD_VALUE] contained invalid value(s). Possible values:"
+                + " [EXTENDED_LATIN, UNCONFUSABLE_LATIN, JA]");
+  }
+
   @Test
   void testFailure_setPremiumListThatDoesntExist() {
     IllegalArgumentException thrown =

core/src/test/resources/google/registry/beam/rde/reducer_rde.xml

@@ -21,6 +21,11 @@
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
 </rdeIDN:idnTableRef>
 
+<rdeIDN:idnTableRef id="unconfusable_latin">
+    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt</rdeIDN:url>
+    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
+</rdeIDN:idnTableRef>
+
 <rdeIDN:idnTableRef id="ja">
     <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_ja_1.0.txt</rdeIDN:url>
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
@@ -32,7 +37,7 @@
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">1</rdeHeader:count>
-    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
 </rdeHeader:header>
 
 </rde:contents>

core/src/test/resources/google/registry/beam/rde/reducer_rde_report.xml

@@ -14,6 +14,6 @@
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">1</rdeHeader:count>
-        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
     </rdeHeader:header>
 </rdeReport:report>

core/src/test/resources/google/registry/flows/domain/domain_check_allocationtoken_fee_anchor_response.xml

@@ -21,13 +21,6 @@
     </resData>
     <extension>
       <fee:chkData>
-        <fee:cd>
-          <fee:name>example2.example</fee:name>
-          <fee:currency>USD</fee:currency>
-          <fee:command>create</fee:command>
-          <fee:period unit="y">1</fee:period>
-          <fee:fee description="create">0.00</fee:fee>
-        </fee:cd>
         <fee:cd>
           <fee:name>example1.tld</fee:name>
           <fee:currency>USD</fee:currency>
@@ -35,12 +28,19 @@
           <fee:period unit="y">1</fee:period>
           <fee:fee description="create">0.00</fee:fee>
         </fee:cd>
+        <fee:cd>
+          <fee:name>example2.example</fee:name>
+          <fee:currency>USD</fee:currency>
+          <fee:command>create</fee:command>
+          <fee:period unit="y">1</fee:period>
+          <fee:class>token-not-supported</fee:class>
+        </fee:cd>
         <fee:cd>
           <fee:name>reserved.tld</fee:name>
           <fee:currency>USD</fee:currency>
           <fee:command>create</fee:command>
           <fee:period unit="y">1</fee:period>
-          <fee:class>reserved</fee:class>
+          <fee:class>token-not-supported</fee:class>
         </fee:cd>
       </fee:chkData>
     </extension>

core/src/test/resources/google/registry/flows/domain/domain_check_allocationtoken_fee_specificuse_response.xml

@@ -25,26 +25,26 @@
     </resData>
     <extension>
       <fee:chkData>
-        <fee:cd>
-          <fee:name>example2.example</fee:name>
-          <fee:currency>USD</fee:currency>
-          <fee:command>create</fee:command>
-          <fee:period unit="y">1</fee:period>
-          <fee:fee description="create">13.00</fee:fee>
-        </fee:cd>
         <fee:cd>
           <fee:name>example1.tld</fee:name>
           <fee:currency>USD</fee:currency>
           <fee:command>create</fee:command>
           <fee:period unit="y">1</fee:period>
-          <fee:fee description="create">13.00</fee:fee>
+          <fee:class>token-not-supported</fee:class>
+        </fee:cd>
+        <fee:cd>
+          <fee:name>example2.example</fee:name>
+          <fee:currency>USD</fee:currency>
+          <fee:command>create</fee:command>
+          <fee:period unit="y">1</fee:period>
+          <fee:class>token-not-supported</fee:class>
         </fee:cd>
         <fee:cd>
           <fee:name>reserved.tld</fee:name>
           <fee:currency>USD</fee:currency>
           <fee:command>create</fee:command>
           <fee:period unit="y">1</fee:period>
-          <fee:class>reserved</fee:class>
+          <fee:class>token-not-supported</fee:class>
         </fee:cd>
         <fee:cd>
           <fee:name>specificuse.tld</fee:name>

core/src/test/resources/google/registry/flows/domain/domain_check_fee_multiple_commands_allocationtoken_response_v06.xml

@@ -0,0 +1,57 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <domain:chkData xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:cd>
+          <domain:name avail="1">example1.tld</domain:name>
+        </domain:cd>
+      </domain:chkData>
+    </resData>
+    <extension>
+      <fee:chkData xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
+        <fee:cd xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
+          <fee:name>example1.tld</fee:name>
+          <fee:currency>USD</fee:currency>
+          <fee:command>create</fee:command>
+          <fee:period unit="y">1</fee:period>
+          <fee:fee description="create">13.00</fee:fee>
+        </fee:cd>
+        <fee:cd xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
+          <fee:name>example1.tld</fee:name>
+          <fee:currency>USD</fee:currency>
+          <fee:command>renew</fee:command>
+          <fee:period unit="y">1</fee:period>
+          <fee:class>token-not-supported</fee:class>
+        </fee:cd>
+        <fee:cd xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
+          <fee:name>example1.tld</fee:name>
+          <fee:currency>USD</fee:currency>
+          <fee:command>transfer</fee:command>
+          <fee:period unit="y">1</fee:period>
+          <fee:fee description="renew">11.00</fee:fee>
+        </fee:cd>
+        <fee:cd xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
+          <fee:name>example1.tld</fee:name>
+          <fee:currency>USD</fee:currency>
+          <fee:command>restore</fee:command>
+          <fee:period unit="y">1</fee:period>
+          <fee:class>token-not-supported</fee:class>
+        </fee:cd>
+        <fee:cd xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
+          <fee:name>example1.tld</fee:name>
+          <fee:currency>USD</fee:currency>
+          <fee:command>update</fee:command>
+          <fee:period unit="y">1</fee:period>
+          <fee:class>token-not-supported</fee:class>
+        </fee:cd>
+      </fee:chkData>
+    </extension>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</epp>

core/src/test/resources/google/registry/flows/domain/domain_check_fee_multiple_commands_allocationtoken_response_v12.xml

@@ -0,0 +1,68 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <resData>
+      <domain:chkData xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:cd>
+          <domain:name avail="1">example1.tld</domain:name>
+        </domain:cd>
+      </domain:chkData>
+    </resData>
+    <extension>
+      <fee:chkData xmlns:fee="urn:ietf:params:xml:ns:fee-0.12"
+          xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <fee:cd>
+          <fee:object>
+            <domain:name>example1.tld</domain:name>
+          </fee:object>
+          <fee:command name="create">
+            <fee:period unit="y">1</fee:period>
+            <fee:fee description="create">13.00</fee:fee>
+          </fee:command>
+        </fee:cd>
+        <fee:cd>
+          <fee:object>
+            <domain:name>example1.tld</domain:name>
+          </fee:object>
+          <fee:command name="renew">
+            <fee:period unit="y">1</fee:period>
+            <fee:class>token-not-supported</fee:class>
+          </fee:command>
+        </fee:cd>
+        <fee:cd>
+          <fee:object>
+            <domain:name>example1.tld</domain:name>
+          </fee:object>
+          <fee:command name="transfer">
+            <fee:period unit="y">1</fee:period>
+            <fee:fee description="renew">11.00</fee:fee>
+          </fee:command>
+        </fee:cd>
+        <fee:cd>
+          <fee:object>
+            <domain:name>example1.tld</domain:name>
+          </fee:object>
+          <fee:command name="restore">
+            <fee:period unit="y">1</fee:period>
+            <fee:class>token-not-supported</fee:class>
+          </fee:command>
+        </fee:cd>
+        <fee:cd>
+          <fee:object>
+            <domain:name>example1.tld</domain:name>
+          </fee:object>
+          <fee:command name="update">
+            <fee:period unit="y">1</fee:period>
+            <fee:class>token-not-supported</fee:class>
+          </fee:command>
+        </fee:cd>
+      </fee:chkData>
+    </extension>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</epp>

core/src/test/resources/google/registry/flows/domain/domain_check_fee_multiple_commands_allocationtoken_v06.xml

@@ -0,0 +1,39 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <check>
+      <domain:check xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>example1.tld</domain:name>
+      </domain:check>
+    </check>
+    <extension>
+      <allocationToken:allocationToken
+          xmlns:allocationToken=
+              "urn:ietf:params:xml:ns:allocationToken-1.0">
+        abc123
+      </allocationToken:allocationToken>
+      <fee:check xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
+        <fee:domain>
+          <fee:name>example1.tld</fee:name>
+          <fee:command>create</fee:command>
+        </fee:domain>
+        <fee:domain>
+          <fee:name>example1.tld</fee:name>
+          <fee:command>renew</fee:command>
+        </fee:domain>
+        <fee:domain>
+          <fee:name>example1.tld</fee:name>
+          <fee:command>transfer</fee:command>
+        </fee:domain>
+        <fee:domain>
+          <fee:name>example1.tld</fee:name>
+          <fee:command>restore</fee:command>
+        </fee:domain>
+        <fee:domain>
+          <fee:name>example1.tld</fee:name>
+          <fee:command>update</fee:command>
+        </fee:domain>
+      </fee:check>
+    </extension>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

core/src/test/resources/google/registry/flows/domain/domain_check_fee_multiple_commands_allocationtoken_v12.xml

@@ -0,0 +1,24 @@
+<epp xmlns="urn:ietf:params:xml:ns:epp-1.0">
+  <command>
+    <check>
+      <domain:check xmlns:domain="urn:ietf:params:xml:ns:domain-1.0">
+        <domain:name>example1.tld</domain:name>
+      </domain:check>
+    </check>
+    <extension>
+      <allocationToken:allocationToken
+          xmlns:allocationToken=
+              "urn:ietf:params:xml:ns:allocationToken-1.0">
+        abc123
+      </allocationToken:allocationToken>
+      <fee:check xmlns:fee="urn:ietf:params:xml:ns:fee-0.12">
+        <fee:command name="create" />
+        <fee:command name="renew" />
+        <fee:command name="transfer" />
+        <fee:command name="restore" />
+        <fee:command name="update" />
+      </fee:check>
+    </extension>
+    <clTRID>ABC-12345</clTRID>
+  </command>
+</epp>

core/src/test/resources/google/registry/flows/domain/domain_check_fee_multiple_commands_default_token_response_v06.xml

@@ -24,7 +24,7 @@
           <fee:currency>USD</fee:currency>
           <fee:command>renew</fee:command>
           <fee:period unit="y">1</fee:period>
-          <fee:fee description="renew">5.50</fee:fee>
+          <fee:fee description="renew">11.00</fee:fee>
         </fee:cd>
         <fee:cd xmlns:fee="urn:ietf:params:xml:ns:fee-0.6">
           <fee:name>example1.tld</fee:name>

core/src/test/resources/google/registry/flows/domain/domain_check_fee_multiple_commands_default_token_response_v12.xml

@@ -28,7 +28,7 @@
           </fee:object>
           <fee:command name="renew">
             <fee:period unit="y">1</fee:period>
-            <fee:fee description="renew">5.50</fee:fee>
+            <fee:fee description="renew">11.00</fee:fee>
           </fee:command>
         </fee:cd>
         <fee:cd>

core/src/test/resources/google/registry/flows/domain/domain_delete_response_fee_free_grace.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<epp xmlns:domain="urn:ietf:params:xml:ns:domain-1.0" xmlns:contact="urn:ietf:params:xml:ns:contact-1.0" xmlns:fee="urn:ietf:params:xml:ns:fee-0.6" xmlns:packageToken="urn:google:params:xml:ns:packageToken-1.0" xmlns="urn:ietf:params:xml:ns:epp-1.0" xmlns:rgp="urn:ietf:params:xml:ns:rgp-1.0" xmlns:fee11="urn:ietf:params:xml:ns:fee-0.11" xmlns:fee12="urn:ietf:params:xml:ns:fee-0.12" xmlns:launch="urn:ietf:params:xml:ns:launch-1.0" xmlns:secDNS="urn:ietf:params:xml:ns:secDNS-1.1" xmlns:host="urn:ietf:params:xml:ns:host-1.0">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <extension>
+      <fee12:delData>
+        <fee12:currency>USD</fee12:currency>
+        <fee12:credit description="addPeriod credit">0.00</fee12:credit>
+      </fee12:delData>
+    </extension>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</epp>

core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesExpectedXml.xml

@@ -245,6 +245,11 @@
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
 </rdeIDN:idnTableRef>
 
+<rdeIDN:idnTableRef id="unconfusable_latin">
+    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt</rdeIDN:url>
+    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
+</rdeIDN:idnTableRef>
+
 <rdeIDN:idnTableRef id="ja">
     <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_ja_1.0.txt</rdeIDN:url>
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
@@ -256,7 +261,7 @@
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">2</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
 </rdeHeader:header>
 
 </rde:contents>

core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesReportXml.xml

@@ -37,6 +37,6 @@
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">2</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
     </rdeHeader:header>
 </rdeReport:report>

core/src/test/resources/google/registry/tools/server/xn--q9jyb4c_2010-10-17_full_S1_R0.xml

@@ -119,6 +119,11 @@
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
 </rdeIDN:idnTableRef>
 
+<rdeIDN:idnTableRef id="unconfusable_latin">
+    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt</rdeIDN:url>
+    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
+</rdeIDN:idnTableRef>
+
 <rdeIDN:idnTableRef id="ja">
     <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_ja_1.0.txt</rdeIDN:url>
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
@@ -130,7 +135,7 @@
   <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeContact-1.0">0</rdeHeader:count>
   <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
   <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
 </rdeHeader:header>
 
 </rde:contents>

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -724,6 +724,7 @@
         drive_folder_id text,
         eap_fee_schedule hstore not null,
         escrow_enabled boolean not null,
+        idn_tables text[],
         invoicing_enabled boolean not null,
         lordn_username text,
         num_dns_publish_locks int4 not null,