Add Java code for storing and using IDN tables per-TLD (#1977 )

This includes changes to make sure that we use the proper per-TLD IDN tables as well as setting/updating/removing them via the Create/Update TLD commands.
Allow a credit of 0 when deleting a domain during a grace period (#1984 )
2026-05-28 02:30:37 +00:00 · 2023-04-06 17:33:23 -04:00 · 2023-04-06 15:58:53 -04:00 · 2023-04-06 15:49:36 -04:00 · 2023-04-06 13:41:51 -04:00 · 2023-04-06 11:30:07 -04:00
27 changed files with 386 additions and 67 deletions
--- a/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml
+++ b/core/src/main/java/google/registry/env/production/default/WEB-INF/cloud-scheduler-tasks.xml
@@ -270,6 +270,6 @@
      This job runs weekly to wipe out PII fields of ContactHistory entities
      that have been in the database for a certain period of time.
    </description>
-    <schedule>0 15 * * 1</schedule>
+    <schedule>0 15 * 12 1</schedule>
  </task>
 </taskentries>
--- a/core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml
+++ b/core/src/main/java/google/registry/env/sandbox/default/WEB-INF/cron.xml
@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--TODO: ptkach - Remove this file after cloud scheduler is fully up and running -->
-<cronentries>
-
-</cronentries>
--- a/core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java
+++ b/core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java
@@ -176,8 +176,7 @@ public class DomainFlowUtils {
      CharMatcher.inRange('a', 'z').or(CharMatcher.inRange('0', '9').or(CharMatcher.anyOf("-.")));

  /** Default validator used to determine if an IDN name can be provisioned on a TLD. */
-  private static final IdnLabelValidator IDN_LABEL_VALIDATOR =
-      IdnLabelValidator.createDefaultIdnLabelValidator();
+  private static final IdnLabelValidator IDN_LABEL_VALIDATOR = new IdnLabelValidator();

  /** The maximum number of DS records allowed on a domain. */
  private static final int MAX_DS_RECORDS_PER_DOMAIN = 8;
--- a/core/src/main/java/google/registry/idn/Latin-IDN.txt
+++ b/core/src/main/java/google/registry/idn/Latin-IDN.txt
@@ -68,6 +68,7 @@ U+011F                         # LATIN SMALL LETTER G WITH BREVE
 U+01E7                         # LATIN SMALL LETTER G WITH CARON
 U+0121                         # LATIN SMALL LETTER G WITH DOT ABOVE
 U+0123                         # LATIN SMALL LETTER G WITH CEDILLA
+U+01E5                         # LATIN SMALL LETTER G WITH STROKE
 U+0068                         # LATIN SMALL LETTER H
 U+0127                         # LATIN SMALL LETTER H WITH STROKE
 U+0069                         # LATIN SMALL LETTER I
--- a/core/src/main/java/google/registry/model/domain/fee/Credit.java
+++ b/core/src/main/java/google/registry/model/domain/fee/Credit.java
@@ -34,7 +34,7 @@ public class Credit extends BaseFee {
      BigDecimal cost, FeeType type, String description) {
    Credit instance = new Credit();
    instance.cost = checkNotNull(cost);
-    checkArgument(instance.cost.signum() < 0);
+    checkArgument(instance.cost.signum() <= 0, "A credit cannot have a positive cost");
    instance.type = checkNotNull(type);
    instance.description = description;
    return instance;
--- a/core/src/main/java/google/registry/model/tld/Registry.java
+++ b/core/src/main/java/google/registry/model/tld/Registry.java
@@ -52,6 +52,7 @@ import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.ReservedList;
 import google.registry.persistence.VKey;
 import google.registry.persistence.converter.JodaMoneyType;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import google.registry.util.Idn;
 import java.util.List;
 import java.util.Map;
@@ -487,6 +488,9 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
   */
  List<VKey<AllocationToken>> defaultPromoTokens;

+  /** A set of allowed {@link IdnTableEnum}s for this TLD, or empty if we should use the default. */
+  Set<IdnTableEnum> idnTables;
+
  public String getTldStr() {
    return tldStr;
  }
@@ -694,6 +698,10 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
    return nullToEmptyImmutableCopy(defaultPromoTokens);
  }

+  public ImmutableSet<IdnTableEnum> getIdnTables() {
+    return nullToEmptyImmutableCopy(idnTables);
+  }
+
  @Override
  public Builder asBuilder() {
    return new Builder(clone(this));
@@ -992,6 +1000,11 @@ public class Registry extends ImmutableObject implements Buildable, UnsafeSerial
      return this;
    }

+    public Builder setIdnTables(ImmutableSet<IdnTableEnum> idnTables) {
+      getInstance().idnTables = idnTables;
+      return this;
+    }
+
    @Override
    public Registry build() {
      final Registry instance = getInstance();
--- a/core/src/main/java/google/registry/persistence/converter/IdnTableEnumSetConverter.java
+++ b/core/src/main/java/google/registry/persistence/converter/IdnTableEnumSetConverter.java
@@ -0,0 +1,34 @@
+// Copyright 2023 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.persistence.converter;
+
+import google.registry.tldconfig.idn.IdnTableEnum;
+import javax.persistence.AttributeConverter;
+import javax.persistence.Converter;
+
+/** JPA {@link AttributeConverter} for storing/retrieving {@link IdnTableEnum}s. */
+@Converter(autoApply = true)
+public class IdnTableEnumSetConverter extends StringSetConverterBase<IdnTableEnum> {
+
+  @Override
+  String toString(IdnTableEnum element) {
+    return element.name();
+  }
+
+  @Override
+  IdnTableEnum fromString(String value) {
+    return IdnTableEnum.valueOf(value);
+  }
+}
--- a/core/src/main/java/google/registry/tldconfig/idn/IdnLabelValidator.java
+++ b/core/src/main/java/google/registry/tldconfig/idn/IdnLabelValidator.java
@@ -17,8 +17,8 @@ package google.registry.tldconfig.idn;
 import static google.registry.tldconfig.idn.IdnTableEnum.EXTENDED_LATIN;
 import static google.registry.tldconfig.idn.IdnTableEnum.JA;

-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.model.tld.Registry;
 import google.registry.util.Idn;
 import java.util.Optional;

@@ -26,23 +26,8 @@ import java.util.Optional;
 public final class IdnLabelValidator {

  /** Most TLDs will use this generic list of IDN tables. */
-  private static final ImmutableList<IdnTableEnum> DEFAULT_IDN_TABLES =
-      ImmutableList.of(EXTENDED_LATIN, JA);
-
-  private static final ImmutableMap<String, ImmutableList<IdnTableEnum>>
-      DEFAULT_IDN_TABLE_LISTS_PER_TLD =
-          ImmutableMap.of("xn--q9jyb4c", ImmutableList.of(EXTENDED_LATIN, JA));
-
-  /** Some TLDs have their own IDN tables, configured here. */
-  private ImmutableMap<String, ImmutableList<IdnTableEnum>> idnTableListsPerTld;
-
-  IdnLabelValidator(ImmutableMap<String, ImmutableList<IdnTableEnum>> indTableListsPerTld) {
-    this.idnTableListsPerTld = indTableListsPerTld;
-  }
-
-  public static IdnLabelValidator createDefaultIdnLabelValidator() {
-    return new IdnLabelValidator(DEFAULT_IDN_TABLE_LISTS_PER_TLD);
-  }
+  private static final ImmutableSet<IdnTableEnum> DEFAULT_IDN_TABLES =
+      ImmutableSet.of(EXTENDED_LATIN, JA);

  /**
   * Returns name of first matching {@link IdnTable} if domain label is valid for the given TLD.
@@ -50,10 +35,13 @@ public final class IdnLabelValidator {
   * <p>A label is valid if it is considered valid by at least one configured IDN table for that
   * TLD. If no match is found, an absent value is returned.
   */
-  public Optional<String> findValidIdnTableForTld(String label, String tld) {
+  public Optional<String> findValidIdnTableForTld(String label, String tldStr) {
    String unicodeString = Idn.toUnicode(label);
-    for (IdnTableEnum idnTable :
-        Optional.ofNullable(idnTableListsPerTld.get(tld)).orElse(DEFAULT_IDN_TABLES)) {
+    Registry tld = Registry.get(tldStr); // uses the cache
+    ImmutableSet<IdnTableEnum> idnTablesForTld = tld.getIdnTables();
+    ImmutableSet<IdnTableEnum> idnTables =
+        idnTablesForTld.isEmpty() ? DEFAULT_IDN_TABLES : idnTablesForTld;
+    for (IdnTableEnum idnTable : idnTables) {
      if (idnTable.getTable().isValidLabel(unicodeString)) {
        return Optional.of(idnTable.getTable().getName());
      }
--- a/core/src/main/java/google/registry/tldconfig/idn/IdnTableEnum.java
+++ b/core/src/main/java/google/registry/tldconfig/idn/IdnTableEnum.java
@@ -24,23 +24,48 @@ import java.net.URL;

 /** Wrapper enum that loads all {@link IdnTable} resources into memory. */
 public enum IdnTableEnum {
-  EXTENDED_LATIN,
-  JA;
+
+  /**
+   * Extended Latin, as used on our existing TLD launches prior to 2023.
+   *
+   * <p>As of 2023 this table is no longer conformant with ICANN's IDN policies for new launches, so
+   * it is retained solely for legacy compatibility with already-launched TLDs.
+   */
+  EXTENDED_LATIN("extended_latin.txt"),
+
+  /**
+   * Extended Latin, but with confusable characters removed.
+   *
+   * <p>This is compatible with ICANN's requirements as of 2023, and is used for the Dads and Grads
+   * TLDs and all subsequent TLD launches. Note that confusable characters consist of various
+   * letters with diacritic marks on them, e.g. U+00EF (LATIN SMALL LETTER I WITH DIAERESIS) is not
+   * allowed because it is confusable with the standard i.
+   */
+  UNCONFUSABLE_LATIN("unconfusable_latin.txt"),
+
+  /**
+   * Japanese, as used on our existing TLD launches prior to 2023.
+   *
+   * <p>As of 2023 this table is no longer conformant with ICANN's IDN policies for new launches, so
+   * it is retained solely for legacy compatibility with already-launched TLDs.
+   */
+  JA("japanese.txt");

  private final IdnTable table;

-  IdnTableEnum() {
-    this.table = load(Ascii.toLowerCase(name()));
+  IdnTableEnum(String filename) {
+    this.table = load(Ascii.toLowerCase(name()), filename);
  }

  public IdnTable getTable() {
    return table;
  }

-  private static IdnTable load(String name) {
+  private static IdnTable load(String tableName, String filename) {
    try {
-      URL resource = Resources.getResource(IdnTableEnum.class, name + ".txt");
-      return IdnTable.createFrom(name, readLines(resource, UTF_8), LanguageValidator.get(name));
+      URL resource = Resources.getResource(IdnTableEnum.class, filename);
+      return IdnTable.createFrom(
+          tableName, readLines(resource, UTF_8), LanguageValidator.get(tableName));
    } catch (IOException e) {
      throw new RuntimeException(e);  // should never happen
    }
--- a/core/src/main/java/google/registry/tldconfig/idn/extended_latin.txt
+++ b/core/src/main/java/google/registry/tldconfig/idn/extended_latin.txt
@@ -49,6 +49,7 @@ U+011F                         # LATIN SMALL LETTER G WITH BREVE
 U+01E7                         # LATIN SMALL LETTER G WITH CARON
 U+0121                         # LATIN SMALL LETTER G WITH DOT ABOVE
 U+0123                         # LATIN SMALL LETTER G WITH CEDILLA
+U+01E5                         # LATIN SMALL LETTER G WITH STROKE
 U+0068                         # LATIN SMALL LETTER H
 U+0127                         # LATIN SMALL LETTER H WITH STROKE
 U+0069                         # LATIN SMALL LETTER I
--- a/core/src/main/java/google/registry/tldconfig/idn/japanese.txt
+++ b/core/src/main/java/google/registry/tldconfig/idn/japanese.txt
--- a/core/src/main/java/google/registry/tldconfig/idn/unconfusable_latin.txt
+++ b/core/src/main/java/google/registry/tldconfig/idn/unconfusable_latin.txt
@@ -0,0 +1,124 @@
+# Registry: Charleston Road Registry Inc.
+# Script: Latn
+# Version: 2.0
+# Effective Date: 2023-04-04
+# URL: https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt
+# Policy: https://www.registry.google/about/policies/domainabuse/
+# Contact Name: CRR Tech
+# Email address: crr-tech@google.com
+# Telephone: +1 (650) 253-0000
+#
+# Code points requiring context rules
+#
+# Code point                        Description of rule/Reference
+#
+# U+002D                            Label must neither start nor end with U+002D. Label
+#    HYPHEN-MINUS                   must not have U+002D in both third and fourth
+#                                   position. RFC 5891 (sec 4.2.3.1)
+#
+
+U+002D                         # HYPHEN-MINUS
+U+0030                         # DIGIT ZERO
+U+0031                         # DIGIT ONE
+U+0032                         # DIGIT TWO
+U+0033                         # DIGIT THREE
+U+0034                         # DIGIT FOUR
+U+0035                         # DIGIT FIVE
+U+0036                         # DIGIT SIX
+U+0037                         # DIGIT SEVEN
+U+0038                         # DIGIT EIGHT
+U+0039                         # DIGIT NINE
+U+0061                         # LATIN SMALL LETTER A
+U+00E0                         # LATIN SMALL LETTER A WITH GRAVE
+U+0103                         # LATIN SMALL LETTER A WITH BREVE
+U+00E2                         # LATIN SMALL LETTER A WITH CIRCUMFLEX
+U+00E5                         # LATIN SMALL LETTER A WITH RING ABOVE
+U+00E4                         # LATIN SMALL LETTER A WITH DIAERESIS
+U+00E3                         # LATIN SMALL LETTER A WITH TILDE
+U+0105                         # LATIN SMALL LETTER A WITH OGONEK
+U+00E6                         # LATIN SMALL LETTER AE
+U+0062                         # LATIN SMALL LETTER B
+U+0063                         # LATIN SMALL LETTER C
+U+0107                         # LATIN SMALL LETTER C WITH ACUTE
+U+010D                         # LATIN SMALL LETTER C WITH CARON
+U+00E7                         # LATIN SMALL LETTER C WITH CEDILLA
+U+0064                         # LATIN SMALL LETTER D
+U+010F                         # LATIN SMALL LETTER D WITH CARON
+U+0111                         # LATIN SMALL LETTER D WITH STROKE
+U+00F0                         # LATIN SMALL LETTER ETH
+U+0065                         # LATIN SMALL LETTER E
+U+00E9                         # LATIN SMALL LETTER E WITH ACUTE
+U+00E8                         # LATIN SMALL LETTER E WITH GRAVE
+U+00EA                         # LATIN SMALL LETTER E WITH CIRCUMFLEX
+U+011B                         # LATIN SMALL LETTER E WITH CARON
+U+00EB                         # LATIN SMALL LETTER E WITH DIAERESIS
+U+0119                         # LATIN SMALL LETTER E WITH OGONEK
+U+0113                         # LATIN SMALL LETTER E WITH MACRON
+U+0117                         # LATIN SMALL LETTER E WITH DOT ABOVE
+U+0259                         # LATIN SMALL LETTER SCHWA
+U+0066                         # LATIN SMALL LETTER F
+U+0067                         # LATIN SMALL LETTER G
+U+011F                         # LATIN SMALL LETTER G WITH BREVE
+U+0121                         # LATIN SMALL LETTER G WITH DOT ABOVE
+U+0068                         # LATIN SMALL LETTER H
+U+0127                         # LATIN SMALL LETTER H WITH STROKE
+U+0069                         # LATIN SMALL LETTER I
+U+00EC                         # LATIN SMALL LETTER I WITH GRAVE
+U+00EE                         # LATIN SMALL LETTER I WITH CIRCUMFLEX
+U+012F                         # LATIN SMALL LETTER I WITH OGONEK
+U+012B                         # LATIN SMALL LETTER I WITH MACRON
+U+006A                         # LATIN SMALL LETTER J
+U+006B                         # LATIN SMALL LETTER K
+U+01E9                         # LATIN SMALL LETTER K WITH CARON
+U+0137                         # LATIN SMALL LETTER K WITH CEDILLA
+U+006C                         # LATIN SMALL LETTER L
+U+013A                         # LATIN SMALL LETTER L WITH ACUTE
+U+013E                         # LATIN SMALL LETTER L WITH CARON
+U+013C                         # LATIN SMALL LETTER L WITH CEDILLA
+U+0142                         # LATIN SMALL LETTER L WITH STROKE
+U+006D                         # LATIN SMALL LETTER M
+U+006E                         # LATIN SMALL LETTER N
+U+0148                         # LATIN SMALL LETTER N WITH CARON
+U+00F1                         # LATIN SMALL LETTER N WITH TILDE
+U+0146                         # LATIN SMALL LETTER N WITH CEDILLA
+U+006F                         # LATIN SMALL LETTER O
+U+00F2                         # LATIN SMALL LETTER O WITH GRAVE
+U+00F4                         # LATIN SMALL LETTER O WITH CIRCUMFLEX
+U+00F6                         # LATIN SMALL LETTER O WITH DIAERESIS
+U+0151                         # LATIN SMALL LETTER O WITH DOUBLE ACUTE
+U+00F5                         # LATIN SMALL LETTER O WITH TILDE
+U+00F8                         # LATIN SMALL LETTER O WITH STROKE
+U+0153                         # LATIN SMALL LIGATURE OE
+U+0070                         # LATIN SMALL LETTER P
+U+0071                         # LATIN SMALL LETTER Q
+U+0072                         # LATIN SMALL LETTER R
+U+0155                         # LATIN SMALL LETTER R WITH ACUTE
+U+0159                         # LATIN SMALL LETTER R WITH CARON
+U+0073                         # LATIN SMALL LETTER S
+U+015B                         # LATIN SMALL LETTER S WITH ACUTE
+U+0161                         # LATIN SMALL LETTER S WITH CARON
+U+015F                         # LATIN SMALL LETTER S WITH CEDILLA
+U+0074                         # LATIN SMALL LETTER T
+U+0165                         # LATIN SMALL LETTER T WITH CARON
+U+0167                         # LATIN SMALL LETTER T WITH STROKE
+U+0075                         # LATIN SMALL LETTER U
+U+00F9                         # LATIN SMALL LETTER U WITH GRAVE
+U+00FB                         # LATIN SMALL LETTER U WITH CIRCUMFLEX
+U+016F                         # LATIN SMALL LETTER U WITH RING ABOVE
+U+0171                         # LATIN SMALL LETTER U WITH DOUBLE ACUTE
+U+0173                         # LATIN SMALL LETTER U WITH OGONEK
+U+016B                         # LATIN SMALL LETTER U WITH MACRON
+U+0076                         # LATIN SMALL LETTER V
+U+0077                         # LATIN SMALL LETTER W
+U+0175                         # LATIN SMALL LETTER W WITH CIRCUMFLEX
+U+0078                         # LATIN SMALL LETTER X
+U+0079                         # LATIN SMALL LETTER Y
+U+00FD                         # LATIN SMALL LETTER Y WITH ACUTE
+U+0177                         # LATIN SMALL LETTER Y WITH CIRCUMFLEX
+U+00FF                         # LATIN SMALL LETTER Y WITH DIAERESIS
+U+007A                         # LATIN SMALL LETTER Z
+U+017A                         # LATIN SMALL LETTER Z WITH ACUTE
+U+017E                         # LATIN SMALL LETTER Z WITH CARON
+U+0292                         # LATIN SMALL LETTER EZH
+U+01EF                         # LATIN SMALL LETTER EZH WITH CARON
+U+00FE                         # LATIN SMALL LETTER THORN
--- a/core/src/main/java/google/registry/tmch/NordnUploadAction.java
+++ b/core/src/main/java/google/registry/tmch/NordnUploadAction.java
@@ -75,9 +75,7 @@ public final class NordnUploadAction implements Runnable {
  static final String PATH = "/_dr/task/nordnUpload";
  static final String LORDN_PHASE_PARAM = "lordnPhase";

-  private static final int BATCH_SIZE = 1000;
  private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private static final Duration LEASE_PERIOD = Duration.standardHours(1);

  /**
   * A unique (enough) id that is outputted in log lines to make it clear which log lines are
--- a/core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java
+++ b/core/src/main/java/google/registry/tools/CreateOrUpdateTldCommand.java
@@ -15,6 +15,7 @@
 package google.registry.tools;

 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static google.registry.tools.UpdateOrDeleteAllocationTokensCommand.getTokenKeys;
 import static google.registry.util.CollectionUtils.findDuplicates;
 import static google.registry.util.CollectionUtils.isNullOrEmpty;
@@ -34,9 +35,11 @@ import google.registry.model.tld.Registry.TldState;
 import google.registry.model.tld.Registry.TldType;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.PremiumListDao;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import google.registry.tools.params.OptionalStringParameter;
 import google.registry.tools.params.TransitionListParameter.BillingCostTransitions;
 import google.registry.tools.params.TransitionListParameter.TldStateTransitions;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -244,6 +247,15 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
              + " present default tokens.")
  List<String> defaultTokens;

+  @Nullable
+  @Parameter(
+      names = "--idn_tables",
+      description =
+          "A comma-separated list of the IDN tables to use for this TLD. Specify an empty list to"
+              + " remove any previously-set tables and to use the default. All elements must be"
+              + " IdnTableEnum values")
+  List<String> idnTables;
+
  /** Returns the existing registry (for update) or null (for creates). */
  @Nullable
  abstract Registry getOldRegistry(String tld);
@@ -392,6 +404,23 @@ abstract class CreateOrUpdateTldCommand extends MutatingCommand {
          builder.setDefaultPromoTokens(getTokenKeys(defaultTokens, null));
        }
      }
+      if (idnTables != null) {
+        if (idnTables.equals(ImmutableList.of(""))) {
+          builder.setIdnTables(ImmutableSet.of());
+        } else {
+          ImmutableSet<String> upperCaseIdnTables =
+              idnTables.stream().map(String::toUpperCase).collect(toImmutableSet());
+          ImmutableSet<String> validIdnStringValues =
+              Arrays.stream(IdnTableEnum.values()).map(Enum::name).collect(toImmutableSet());
+          checkArgument(
+              validIdnStringValues.containsAll(upperCaseIdnTables),
+              "IDN tables %s contained invalid value(s). Possible values: %s",
+              upperCaseIdnTables,
+              validIdnStringValues);
+          builder.setIdnTables(
+              upperCaseIdnTables.stream().map(IdnTableEnum::valueOf).collect(toImmutableSet()));
+        }
+      }
      // Update the Registry object.
      setCommandSpecificProperties(builder);
      stageEntityChange(oldRegistry, builder.build());
--- a/core/src/main/resources/META-INF/persistence.xml
+++ b/core/src/main/resources/META-INF/persistence.xml
@@ -91,6 +91,7 @@
    <class>google.registry.persistence.converter.DatabaseMigrationScheduleTransitionConverter</class>
    <class>google.registry.persistence.converter.DateTimeConverter</class>
    <class>google.registry.persistence.converter.DurationConverter</class>
+    <class>google.registry.persistence.converter.IdnTableEnumSetConverter</class>
    <class>google.registry.persistence.converter.InetAddressSetConverter</class>
    <class>google.registry.persistence.converter.LocalDateConverter</class>
    <class>google.registry.persistence.converter.PostalInfoChoiceListConverter</class>
--- a/core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java
+++ b/core/src/test/java/google/registry/flows/domain/DomainDeleteFlowTest.java
@@ -1236,4 +1236,16 @@ class DomainDeleteFlowTest extends ResourceFlowTestCase<DomainDeleteFlow, Domain
    EppException thrown = assertThrows(UnimplementedExtensionException.class, this::runFlow);
    assertAboutEppExceptions().that(thrown).marshalsToXml();
  }
+
+  @Test
+  void testSuccess_freeCreation_deletionDuringGracePeriod() throws Exception {
+    // Deletion during the add grace period should still work even if the credit is 0
+    setUpSuccessfulTest();
+    BillingEvent.OneTime graceBillingEvent =
+        persistResource(createBillingEvent(Reason.CREATE, Money.of(USD, 0)));
+    setUpGracePeriods(
+        GracePeriod.forBillingEvent(GracePeriodStatus.ADD, domain.getRepoId(), graceBillingEvent));
+    clock.advanceOneMilli();
+    runFlowAssertResponse(loadFile("domain_delete_response_fee_free_grace.xml"));
+  }
 }
--- a/core/src/test/java/google/registry/tldconfig/idn/IdnLabelValidatorTest.java
+++ b/core/src/test/java/google/registry/tldconfig/idn/IdnLabelValidatorTest.java
@@ -15,17 +15,26 @@
 package google.registry.tldconfig.idn;

 import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.persistResource;

-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;

 /** Unit tests for {@link IdnLabelValidator}. */
 class IdnLabelValidatorTest {

-  private IdnLabelValidator idnLabelValidator = IdnLabelValidator.createDefaultIdnLabelValidator();
+  @RegisterExtension
+  final JpaIntegrationTestExtension jpa =
+      new JpaTestExtensions.Builder().buildIntegrationTestExtension();

-  private void doJapaneseLanguageTests(String tld) {
+  private IdnLabelValidator idnLabelValidator = new IdnLabelValidator();
+
+  private void doJapaneseAndLatinLanguageTests(String tld) {
+    createTld(tld);
    assertThat(idnLabelValidator.findValidIdnTableForTld("foo", tld)).isPresent();
    assertThat(idnLabelValidator.findValidIdnTableForTld("12379foar", tld)).isPresent();
    assertThat(idnLabelValidator.findValidIdnTableForTld("みんな", tld)).isPresent();
@@ -84,26 +93,29 @@ class IdnLabelValidatorTest {

  @Test
  void testMinna() {
-    doJapaneseLanguageTests("xn--q9jyb4c");
+    doJapaneseAndLatinLanguageTests("xn--q9jyb4c");
  }

  @Test
  void testFoo() {
-    doJapaneseLanguageTests("foo");
+    doJapaneseAndLatinLanguageTests("foo");
  }

  @Test
  void testSoy() {
-    doJapaneseLanguageTests("soy");
+    doJapaneseAndLatinLanguageTests("soy");
  }

  @Test
-  void testOverridenTables() {
-    // Set .tld to have only the extended latin table and not japanese.
-    idnLabelValidator =
-        new IdnLabelValidator(
-            ImmutableMap.of("tld", ImmutableList.of(IdnTableEnum.EXTENDED_LATIN)));
+  void testPerTldConfig() {
+    persistResource(
+        createTld("tld")
+            .asBuilder()
+            .setIdnTables(ImmutableSet.of(IdnTableEnum.EXTENDED_LATIN))
+            .build());
    assertThat(idnLabelValidator.findValidIdnTableForTld("foo", "tld")).isPresent();
+    assertThat(idnLabelValidator.findValidIdnTableForTld("abcdefghæ", "tld")).isPresent();
+    // Extended Latin shouldn't include Japanese characters
    assertThat(idnLabelValidator.findValidIdnTableForTld("みんな", "tld")).isEmpty();
  }
 }
--- a/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java
+++ b/core/src/test/java/google/registry/tmch/NordnUploadActionTest.java
@@ -38,9 +38,6 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;

-import com.google.appengine.api.taskqueue.TaskHandle;
-import com.google.appengine.api.taskqueue.TaskOptions;
-import com.google.appengine.api.taskqueue.TaskOptions.Method;
 import com.google.common.base.VerifyException;
 import google.registry.batch.CloudTasksUtils;
 import google.registry.model.domain.Domain;
@@ -232,13 +229,6 @@ class NordnUploadActionTest {
            .build());
  }

-  private static TaskHandle makeTaskHandle(
-      String taskName, String tag, String payload, String queue) {
-    return new TaskHandle(
-        TaskOptions.Builder.withPayload(payload).method(Method.PULL).tag(tag).taskName(taskName),
-        queue);
-  }
-
  private void verifyColumnCleared(String domainName) {
    VKey<Domain> domainKey = load(Domain.class, domainName, clock.nowUtc());
    Domain domain = loadByKey(domainKey);
--- a/core/src/test/java/google/registry/tools/CreateTldCommandTest.java
+++ b/core/src/test/java/google/registry/tools/CreateTldCommandTest.java
@@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Range;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.tld.Registry;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import java.math.BigDecimal;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -544,6 +545,35 @@ class CreateTldCommandTest extends CommandTestCase<CreateTldCommand> {
    assertThat(Registry.get("xn--q9jyb4c").getDriveFolderId()).isNull();
  }

+  @Test
+  void testSuccess_setsIdnTables() throws Exception {
+    runCommandForced(
+        "--idn_tables=extended_latin,ja",
+        "--roid_suffix=ASDF",
+        "--dns_writers=VoidDnsWriter",
+        "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables())
+        .containsExactly(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA);
+  }
+
+  @Test
+  void testFailure_invalidIdnTable() throws Exception {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () ->
+                runCommandForced(
+                    "--idn_tables=extended_latin,bad_value",
+                    "--roid_suffix=ASDF",
+                    "--dns_writers=VoidDnsWriter",
+                    "xn--q9jyb4c"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "IDN tables [EXTENDED_LATIN, BAD_VALUE] contained invalid value(s). Possible values:"
+                + " [EXTENDED_LATIN, UNCONFUSABLE_LATIN, JA]");
+  }
+
  @Test
  void testFailure_setPremiumListThatDoesntExist() {
    IllegalArgumentException thrown =
--- a/core/src/test/java/google/registry/tools/UpdateTldCommandTest.java
+++ b/core/src/test/java/google/registry/tools/UpdateTldCommandTest.java
@@ -39,6 +39,7 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedMap;
 import google.registry.model.domain.token.AllocationToken;
 import google.registry.model.tld.Registry;
+import google.registry.tldconfig.idn.IdnTableEnum;
 import java.util.Optional;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -1053,6 +1054,38 @@ class UpdateTldCommandTest extends CommandTestCase<UpdateTldCommand> {
    assertThat(Registry.get("xn--q9jyb4c").getDriveFolderId()).isNull();
  }

+  @Test
+  void testSuccess_setsIdnTables() throws Exception {
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables()).isEmpty();
+    runCommandForced("--idn_tables=extended_latin,ja", "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables())
+        .containsExactly(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA);
+  }
+
+  @Test
+  void testSuccess_removesIndTables() throws Exception {
+    persistResource(
+        Registry.get("xn--q9jyb4c")
+            .asBuilder()
+            .setIdnTables(ImmutableSet.of(IdnTableEnum.EXTENDED_LATIN, IdnTableEnum.JA))
+            .build());
+    runCommandForced("--idn_tables=", "xn--q9jyb4c");
+    assertThat(Registry.get("xn--q9jyb4c").getIdnTables()).isEmpty();
+  }
+
+  @Test
+  void testFailure_invalidIdnTable() throws Exception {
+    IllegalArgumentException thrown =
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> runCommandForced("--idn_tables=extended_latin,bad_value", "xn--q9jyb4c"));
+    assertThat(thrown)
+        .hasMessageThat()
+        .isEqualTo(
+            "IDN tables [EXTENDED_LATIN, BAD_VALUE] contained invalid value(s). Possible values:"
+                + " [EXTENDED_LATIN, UNCONFUSABLE_LATIN, JA]");
+  }
+
  @Test
  void testFailure_setPremiumListThatDoesntExist() {
    IllegalArgumentException thrown =
--- a/core/src/test/resources/google/registry/beam/rde/reducer_rde.xml
+++ b/core/src/test/resources/google/registry/beam/rde/reducer_rde.xml
@@ -21,6 +21,11 @@
    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
 </rdeIDN:idnTableRef>

+<rdeIDN:idnTableRef id="unconfusable_latin">
+    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt</rdeIDN:url>
+    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
+</rdeIDN:idnTableRef>
+
 <rdeIDN:idnTableRef id="ja">
    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_ja_1.0.txt</rdeIDN:url>
    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
@@ -32,7 +37,7 @@
    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">1</rdeHeader:count>
-    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
 </rdeHeader:header>

 </rde:contents>
--- a/core/src/test/resources/google/registry/beam/rde/reducer_rde_report.xml
+++ b/core/src/test/resources/google/registry/beam/rde/reducer_rde_report.xml
@@ -14,6 +14,6 @@
        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">1</rdeHeader:count>
-        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
    </rdeHeader:header>
 </rdeReport:report>
--- a/core/src/test/resources/google/registry/flows/domain/domain_delete_response_fee_free_grace.xml
+++ b/core/src/test/resources/google/registry/flows/domain/domain_delete_response_fee_free_grace.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<epp xmlns:domain="urn:ietf:params:xml:ns:domain-1.0" xmlns:contact="urn:ietf:params:xml:ns:contact-1.0" xmlns:fee="urn:ietf:params:xml:ns:fee-0.6" xmlns:packageToken="urn:google:params:xml:ns:packageToken-1.0" xmlns="urn:ietf:params:xml:ns:epp-1.0" xmlns:rgp="urn:ietf:params:xml:ns:rgp-1.0" xmlns:fee11="urn:ietf:params:xml:ns:fee-0.11" xmlns:fee12="urn:ietf:params:xml:ns:fee-0.12" xmlns:launch="urn:ietf:params:xml:ns:launch-1.0" xmlns:secDNS="urn:ietf:params:xml:ns:secDNS-1.1" xmlns:host="urn:ietf:params:xml:ns:host-1.0">
+  <response>
+    <result code="1000">
+      <msg>Command completed successfully</msg>
+    </result>
+    <extension>
+      <fee12:delData>
+        <fee12:currency>USD</fee12:currency>
+        <fee12:credit description="addPeriod credit">0.00</fee12:credit>
+      </fee12:delData>
+    </extension>
+    <trID>
+      <clTRID>ABC-12345</clTRID>
+      <svTRID>server-trid</svTRID>
+    </trID>
+  </response>
+</epp>
--- a/core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesExpectedXml.xml
+++ b/core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesExpectedXml.xml
@@ -245,6 +245,11 @@
    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
 </rdeIDN:idnTableRef>

+<rdeIDN:idnTableRef id="unconfusable_latin">
+    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt</rdeIDN:url>
+    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
+</rdeIDN:idnTableRef>
+
 <rdeIDN:idnTableRef id="ja">
    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_ja_1.0.txt</rdeIDN:url>
    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
@@ -256,7 +261,7 @@
    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">2</rdeHeader:count>
    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
 </rdeHeader:header>

 </rde:contents>
--- a/core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesReportXml.xml
+++ b/core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesReportXml.xml
@@ -37,6 +37,6 @@
        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">2</rdeHeader:count>
        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
    </rdeHeader:header>
 </rdeReport:report>
--- a/core/src/test/resources/google/registry/tools/server/xn--q9jyb4c_2010-10-17_full_S1_R0.xml
+++ b/core/src/test/resources/google/registry/tools/server/xn--q9jyb4c_2010-10-17_full_S1_R0.xml
@@ -119,6 +119,11 @@
    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
 </rdeIDN:idnTableRef>

+<rdeIDN:idnTableRef id="unconfusable_latin">
+    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_latn_2.0.txt</rdeIDN:url>
+    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
+</rdeIDN:idnTableRef>
+
 <rdeIDN:idnTableRef id="ja">
    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_ja_1.0.txt</rdeIDN:url>
    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
@@ -130,7 +135,7 @@
  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeContact-1.0">0</rdeHeader:count>
  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">2</rdeHeader:count>
+  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
 </rdeHeader:header>

 </rde:contents>
--- a/db/src/main/resources/sql/schema/db-schema.sql.generated
+++ b/db/src/main/resources/sql/schema/db-schema.sql.generated
@@ -724,6 +724,7 @@
        drive_folder_id text,
        eap_fee_schedule hstore not null,
        escrow_enabled boolean not null,
+        idn_tables text[],
        invoicing_enabled boolean not null,
        lordn_username text,
        num_dns_publish_locks int4 not null,
Author	SHA1	Message	Date
gbrodman	33a771b13e	Add Java code for storing and using IDN tables per-TLD (#1977 ) This includes changes to make sure that we use the proper per-TLD IDN tables as well as setting/updating/removing them via the Create/Update TLD commands.	2023-04-06 17:33:23 -04:00
gbrodman	bd65c6eee6	Allow a credit of 0 when deleting a domain during a grace period (#1984 ) There can be situations (anchor tenants, test tokens, other ways of getting a domain to cost $0) where we may want to delete a domain during the add grace period but the credit applied is 0. We should not fail on those cases. See b/277115241 for an example.	2023-04-06 15:58:53 -04:00
Ben McIlwain	20c673840e	Add a new Unconfusable Latin table (#1981 ) This new table has just been approved by ICANN. It is the same as our existing Extended Latin table, except with the removal of some lesser-used characters with diacritic marks that are confusable variants. The filenames for the IDN tables are made explicit to improve code readability. And this reverses the removal of G with stroke from the existing Extended Latin table (see PR #1938), so that that table continues to accurately reflect the state of our previously launched TLDs. This is the full list of removed characters: U+00E1 # LATIN SMALL LETTER A WITH ACUTE U+0101 # LATIN SMALL LETTER A WITH MACRON U+01CE # LATIN SMALL LETTER A WITH CARON U+010B # LATIN SMALL LETTER C WITH DOT ABOVE U+01E7 # LATIN SMALL LETTER G WITH CARON U+0123 # LATIN SMALL LETTER G WITH CEDILLA U+01E5 # LATIN SMALL LETTER G WITH STROKE U+0131 # LATIN SMALL LETTER DOTLESS I U+00ED # LATIN SMALL LETTER I WITH ACUTE U+00EF # LATIN SMALL LETTER I WITH DIAERESIS U+01D0 # LATIN SMALL LETTER I WITH CARON U+0144 # LATIN SMALL LETTER N WITH ACUTE U+014B # LATIN SMALL LETTER ENG U+00F3 # LATIN SMALL LETTER O WITH ACUTE U+014D # LATIN SMALL LETTER O WITH MACRON U+01D2 # LATIN SMALL LETTER O WITH CARON U+0157 # LATIN SMALL LETTER R WITH CEDILLA U+0163 # LATIN SMALL LETTER T WITH CEDILLA U+00FA # LATIN SMALL LETTER U WITH ACUTE U+00FC # LATIN SMALL LETTER U WITH DIAERESIS U+01D4 # LATIN SMALL LETTER U WITH CARON U+1E83 # LATIN SMALL LETTER W WITH ACUTE U+1E81 # LATIN SMALL LETTER W WITH GRAVE U+1E85 # LATIN SMALL LETTER W WITH DIAERESIS U+1EF3 # LATIN SMALL LETTER Y WITH GRAVE U+017C # LATIN SMALL LETTER Z WITH DOT ABOVE	2023-04-06 15:49:36 -04:00
Lai Jiang	11c60b8c8f	Temporarily disable contact history wipeout (#1982 ) Makes the next run at the first Monday of December, which should give us plenty of time to fix the issue with it wiping out PII in the most recent contact history. <!-- Reviewable:start --> - - - This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1982) <!-- Reviewable:end -->	2023-04-06 13:41:51 -04:00
Lai Jiang	e330fd1c66	Remove cron.xml from sandbox (#1979 ) It is somehow missed in #1965.	2023-04-06 11:30:07 -04:00