Add Augmented Latin IDN table to IDN enums (#2914)

This was added in https://github.com/google/nomulus/pull/2884 , but now as of
this PR it can actually be configured and used on a TLD.

core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java

@@ -1240,49 +1240,49 @@ public class DomainFlowUtils {
   }
 
   /** Domain names can only contain a-z, 0-9, '.' and '-'. */
-  static class BadDomainNameCharacterException extends ParameterValuePolicyErrorException {
+  static class BadDomainNameCharacterException extends ParameterValueSyntaxErrorException {
     public BadDomainNameCharacterException() {
       super("Domain names can only contain a-z, 0-9, '.' and '-'");
     }
   }
 
   /** Non-IDN domain names cannot contain hyphens in the third or fourth position. */
-  static class DashesInThirdAndFourthException extends ParameterValuePolicyErrorException {
+  static class DashesInThirdAndFourthException extends ParameterValueSyntaxErrorException {
     public DashesInThirdAndFourthException() {
       super("Non-IDN domain names cannot contain dashes in the third or fourth position");
     }
   }
 
   /** Domain labels cannot begin with a dash. */
-  static class LeadingDashException extends ParameterValuePolicyErrorException {
+  static class LeadingDashException extends ParameterValueSyntaxErrorException {
     public LeadingDashException() {
       super("Domain labels cannot begin with a dash");
     }
   }
 
   /** Domain labels cannot end with a dash. */
-  static class TrailingDashException extends ParameterValuePolicyErrorException {
+  static class TrailingDashException extends ParameterValueSyntaxErrorException {
     public TrailingDashException() {
       super("Domain labels cannot end with a dash");
     }
   }
 
   /** Domain labels cannot be longer than 63 characters. */
-  static class DomainLabelTooLongException extends ParameterValuePolicyErrorException {
+  static class DomainLabelTooLongException extends ParameterValueSyntaxErrorException {
     public DomainLabelTooLongException() {
       super("Domain labels cannot be longer than 63 characters");
     }
   }
 
   /** No part of a domain name can be empty. */
-  static class EmptyDomainNamePartException extends ParameterValuePolicyErrorException {
+  static class EmptyDomainNamePartException extends ParameterValueSyntaxErrorException {
     public EmptyDomainNamePartException() {
       super("No part of a domain name can be empty");
     }
   }
 
   /** Domain name starts with xn-- but is not a valid IDN. */
-  static class InvalidPunycodeException extends ParameterValuePolicyErrorException {
+  static class InvalidPunycodeException extends ParameterValueSyntaxErrorException {
     public InvalidPunycodeException() {
       super("Domain name starts with xn-- but is not a valid IDN");
     }

core/src/main/java/google/registry/model/tld/Tld.java

@@ -1034,12 +1034,13 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
       return this;
     }
 
-    public static final Pattern ROID_SUFFIX_PATTERN = Pattern.compile("^[A-Z\\d_]{1,8}$");
+    public static final Pattern ROID_SUFFIX_PATTERN = Pattern.compile("^[A-Z\\d]{1,8}$");
 
     public Builder setRoidSuffix(String roidSuffix) {
       checkArgument(
           ROID_SUFFIX_PATTERN.matcher(roidSuffix).matches(),
-          "ROID suffix must be in format %s",
+          "ROID suffix %s must be in format %s",
+          roidSuffix,
           ROID_SUFFIX_PATTERN.pattern());
       getInstance().roidSuffix = roidSuffix;
       return this;

core/src/main/java/google/registry/tldconfig/idn/IdnTableEnum.java

@@ -43,6 +43,15 @@ public enum IdnTableEnum {
    */
   UNCONFUSABLE_LATIN("unconfusable_latin.txt"),
 
+  /**
+   * ICANN LGR 2025 Latin, but with confusable characters removed.
+   *
+   * <p>This is based on <a
+   * href="https://www.icann.org/sites/default/files/packages/lgr/lgr-second-level-latin-full-variant-script-24jan24-en.html">ICANN's
+   * LGR table</a>, but is simpler.
+   */
+  AUGMENTED_LATIN("augmented_latin.txt"),
+
   /**
    * Japanese, as used on our existing TLD launches prior to 2023.
    *

core/src/main/java/google/registry/tldconfig/idn/augmented_latin.txt

@@ -1,4 +1,4 @@
-# URL: https://www.iana.org/domains/idn-tables/tables/google_latn_1.0.txt
+# URL: https://www.iana.org/domains/idn-tables/tables/google_latn_3.0.txt
 # Policy: https://www.registry.google/about/policies/domainabuse/
 U+002D                         # HYPHEN-MINUS
 U+0030                         # DIGIT ZERO

core/src/test/java/google/registry/beam/billing/InvoicingPipelineTest.java

@@ -335,7 +335,7 @@ class InvoicingPipelineTest {
             .build();
     persistResource(registrar);
     Tld test =
-        newTld("test", "_TEST", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
+        newTld("test", "TEST", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
             .asBuilder()
             .setInvoicingEnabled(true)
             .build();
@@ -391,7 +391,7 @@ class InvoicingPipelineTest {
     // Test that comments are removed from the .sql file correctly
     assertThat(InvoicingPipeline.makeCloudSqlQuery("2017-10"))
         .isEqualTo(
-            """
+"""
 
 SELECT b, r FROM BillingEvent b
 JOIN Registrar r ON b.clientId = r.registrarId
@@ -449,13 +449,13 @@ AND cr.id IS NULL
     persistResource(registrar3);
 
     Tld test =
-        newTld("test", "_TEST", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
+        newTld("test", "TEST", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
             .asBuilder()
             .setInvoicingEnabled(true)
             .build();
     persistResource(test);
     Tld hello =
-        newTld("hello", "_HELLO", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
+        newTld("hello", "HELLO", ImmutableSortedMap.of(START_OF_TIME, GENERAL_AVAILABILITY))
             .asBuilder()
             .setInvoicingEnabled(true)
             .build();

core/src/test/java/google/registry/beam/common/RegistryJpaReadTest.java

@@ -163,7 +163,7 @@ public class RegistryJpaReadTest {
   }
 
   private void setupForJoinQuery() {
-    Tld registry = newTld("com", "ABCD_APP");
+    Tld registry = newTld("com", "ABCDAPP");
     Registrar registrar =
         makeRegistrar1()
             .asBuilder()

core/src/test/java/google/registry/bsa/IdnCheckerTest.java

@@ -17,6 +17,7 @@ package google.registry.bsa;
 import static com.google.common.truth.Truth.assertThat;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistResource;
+import static google.registry.tldconfig.idn.IdnTableEnum.AUGMENTED_LATIN;
 import static google.registry.tldconfig.idn.IdnTableEnum.EXTENDED_LATIN;
 import static google.registry.tldconfig.idn.IdnTableEnum.JA;
 import static google.registry.tldconfig.idn.IdnTableEnum.UNCONFUSABLE_LATIN;
@@ -43,6 +44,7 @@ public class IdnCheckerTest {
   Tld jaonly;
   Tld jandelatin;
   Tld strictlatin;
+  Tld auglatin;
   IdnChecker idnChecker;
 
   @BeforeEach
@@ -50,6 +52,7 @@ public class IdnCheckerTest {
     jaonly = createTld("jaonly");
     jandelatin = createTld("jandelatin");
     strictlatin = createTld("strictlatin");
+    auglatin = createTld("auglatin");
 
     jaonly =
         persistResource(
@@ -72,6 +75,13 @@ public class IdnCheckerTest {
                 .setBsaEnrollStartTime(Optional.of(fakeClock.nowUtc()))
                 .setIdnTables(ImmutableSet.of(UNCONFUSABLE_LATIN))
                 .build());
+    auglatin =
+        persistResource(
+            auglatin
+                .asBuilder()
+                .setBsaEnrollStartTime(Optional.of(fakeClock.nowUtc()))
+                .setIdnTables(ImmutableSet.of(AUGMENTED_LATIN))
+                .build());
     fakeClock.advanceOneMilli();
     idnChecker = new IdnChecker(fakeClock);
   }
@@ -79,12 +89,13 @@ public class IdnCheckerTest {
   @Test
   void getAllValidIdns_allTlds() {
     assertThat(idnChecker.getAllValidIdns("all"))
-        .containsExactly(EXTENDED_LATIN, JA, UNCONFUSABLE_LATIN);
+        .containsExactly(EXTENDED_LATIN, JA, UNCONFUSABLE_LATIN, AUGMENTED_LATIN);
   }
 
   @Test
   void getAllValidIdns_notJa() {
-    assertThat(idnChecker.getAllValidIdns("à")).containsExactly(EXTENDED_LATIN, UNCONFUSABLE_LATIN);
+    assertThat(idnChecker.getAllValidIdns("à"))
+        .containsExactly(EXTENDED_LATIN, UNCONFUSABLE_LATIN, AUGMENTED_LATIN);
   }
 
   @Test
@@ -116,6 +127,7 @@ public class IdnCheckerTest {
 
   @Test
   void getForbiddingTlds_success() {
-    assertThat(idnChecker.getForbiddingTlds(ImmutableSet.of("JA"))).containsExactly(strictlatin);
+    assertThat(idnChecker.getForbiddingTlds(ImmutableSet.of("JA")))
+        .containsExactly(strictlatin, auglatin);
   }
 }

core/src/test/java/google/registry/model/tld/TldTest.java

@@ -751,7 +751,9 @@ public final class TldTest extends EntityTestCase {
         assertThrows(
             IllegalArgumentException.class,
             () -> Tld.get("tld").asBuilder().setRoidSuffix("123456789"));
-    assertThat(e).hasMessageThat().isEqualTo("ROID suffix must be in format ^[A-Z\\d_]{1,8}$");
+    assertThat(e)
+        .hasMessageThat()
+        .isEqualTo("ROID suffix 123456789 must be in format ^[A-Z\\d]{1,8}$");
   }
 
   @Test
@@ -766,6 +768,12 @@ public final class TldTest extends EntityTestCase {
         IllegalArgumentException.class, () -> Tld.get("tld").asBuilder().setRoidSuffix("ABC-DEF"));
   }
 
+  @Test
+  void testFailure_roidSuffixContainsUnderscores() {
+    assertThrows(
+        IllegalArgumentException.class, () -> Tld.get("tld").asBuilder().setRoidSuffix("ABC_DEF"));
+  }
+
   @Test
   void testSuccess_setDefaultPromoTokens() {
     Tld registry = Tld.get("tld");

core/src/test/java/google/registry/rdap/RdapDomainActionTest.java

@@ -246,7 +246,7 @@ class RdapDomainActionTest extends RdapActionBaseTestCase<RdapDomainAction> {
         .isEqualTo(
             addDomainBoilerplateNotices(
                 jsonFileBuilder()
-                    .addDomain("cat.1.tld", "D-1_TLD")
+                    .addDomain("cat.1.tld", "D-1TLD")
                     .addNameserver("ns1.cat.lol", "2-ROID")
                     .addNameserver("ns2.cat.lol", "4-ROID")
                     .addRegistrar("Multilevel Registrar")

core/src/test/java/google/registry/rdap/RdapDomainSearchActionTest.java

@@ -732,7 +732,7 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
         RequestType.NAME,
         "cat.1.test",
         jsonFileBuilder()
-            .addDomain("cat.1.test", "1B-1_TEST")
+            .addDomain("cat.1.test", "1B-1TEST")
             .addRegistrar("1.test")
             .addNameserver("ns1.cat.1.test", "17-ROID")
             .addNameserver("ns2.cat.2.test", "19-ROID")
@@ -746,7 +746,7 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
         RequestType.NAME,
         "ca*.1.test",
         jsonFileBuilder()
-            .addDomain("cat.1.test", "1B-1_TEST")
+            .addDomain("cat.1.test", "1B-1TEST")
             .addRegistrar("1.test")
             .addNameserver("ns1.cat.1.test", "17-ROID")
             .addNameserver("ns2.cat.2.test", "19-ROID")
@@ -822,7 +822,7 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
         .that(generateActualJson(RequestType.NAME, "cat.*"))
         .isEqualTo(
             jsonFileBuilder()
-                .addDomain("cat.1.test", "1B-1_TEST")
+                .addDomain("cat.1.test", "1B-1TEST")
                 .addDomain("cat.example", "F-EXAMPLE")
                 .addDomain("cat.lol", "6-LOL")
                 .addDomain("cat.みんな", "15-Q9JYB4C")
@@ -860,7 +860,7 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
         .that(generateActualJson(RequestType.NAME, "cat*"))
         .isEqualTo(
             jsonFileBuilder()
-                .addDomain("cat.1.test", "1B-1_TEST")
+                .addDomain("cat.1.test", "1B-1TEST")
                 .addDomain("cat.example", "F-EXAMPLE")
                 .addDomain("cat.lol", "6-LOL")
                 .addDomain("cat.みんな", "15-Q9JYB4C")
@@ -1284,7 +1284,7 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
         RequestType.NS_LDH_NAME,
         "ns1.cat.1.test",
         jsonFileBuilder()
-            .addDomain("cat.1.test", "1B-1_TEST")
+            .addDomain("cat.1.test", "1B-1TEST")
             .addRegistrar("1.test")
             .addNameserver("ns1.cat.1.test", "17-ROID")
             .addNameserver("ns2.cat.2.test", "19-ROID")
@@ -1298,7 +1298,7 @@ class RdapDomainSearchActionTest extends RdapSearchActionTestCase<RdapDomainSear
         RequestType.NS_LDH_NAME,
         "ns*.cat.1.test",
         jsonFileBuilder()
-            .addDomain("cat.1.test", "1B-1_TEST")
+            .addDomain("cat.1.test", "1B-1TEST")
             .addRegistrar("1.test")
             .addNameserver("ns1.cat.1.test", "17-ROID")
             .addNameserver("ns2.cat.2.test", "19-ROID")

core/src/test/java/google/registry/testing/DatabaseHelper.java

@@ -420,8 +420,7 @@ public final class DatabaseHelper {
   public static Tld createTld(String tld, ImmutableSortedMap<DateTime, TldState> tldStates) {
     // Coerce the TLD string into a valid ROID suffix.
     String roidSuffix =
-        Ascii.toUpperCase(tld.replaceFirst(ACE_PREFIX_REGEX, "").replace('.', '_'))
-            .replace('-', '_');
+        Ascii.toUpperCase(tld.replaceFirst(ACE_PREFIX_REGEX, "").replace(".", "")).replace("-", "");
     return createTld(
         tld, roidSuffix.length() > 8 ? roidSuffix.substring(0, 8) : roidSuffix, tldStates);
   }

core/src/test/java/google/registry/tools/ConfigureTldCommandTest.java

@@ -368,7 +368,7 @@ public class ConfigureTldCommandTest extends CommandTestCase<ConfigureTldCommand
                     "TLDSTR", name, "TLDUNICODE", name, "ROIDSUFFIX", "TLLLLLLLLLLLLLLLLLLLLLLD")));
     IllegalArgumentException thrown =
         assertThrows(IllegalArgumentException.class, () -> runCommandForced("--input=" + tldFile));
-    assertThat(thrown.getMessage()).isEqualTo("ROID suffix must be in format ^[A-Z\\d_]{1,8}$");
+    assertThat(thrown.getMessage()).isEqualTo("ROID suffix must be in format ^[A-Z\\d]{1,8}$");
   }
 
   @Test

core/src/test/resources/google/registry/beam/rde/reducer_rde.xml

@@ -26,6 +26,11 @@
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
 </rdeIDN:idnTableRef>
 
+<rdeIDN:idnTableRef id="augmented_latin">
+    <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_latn_3.0.txt</rdeIDN:url>
+    <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
+</rdeIDN:idnTableRef>
+
 <rdeIDN:idnTableRef id="ja">
     <rdeIDN:url>https://www.iana.org/domains/idn-tables/tables/google_ja_1.0.txt</rdeIDN:url>
     <rdeIDN:urlPolicy>https://www.registry.google/about/policies/domainabuse/</rdeIDN:urlPolicy>
@@ -37,7 +42,7 @@
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">1</rdeHeader:count>
-    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
+    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">4</rdeHeader:count>
 </rdeHeader:header>
 
 </rde:contents>

core/src/test/resources/google/registry/beam/rde/reducer_rde_report.xml

@@ -14,6 +14,6 @@
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">1</rdeHeader:count>
-        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
+        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">4</rdeHeader:count>
     </rdeHeader:header>
 </rdeReport:report>

core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesExpectedXml.xml

@@ -261,7 +261,7 @@
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">2</rdeHeader:count>
     <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
+    <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">4</rdeHeader:count>
 </rdeHeader:header>
 
 </rde:contents>

core/src/test/resources/google/registry/rde/testMapReduce_withDomain_producesReportXml.xml

@@ -37,6 +37,6 @@
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeDomain-1.0">1</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">2</rdeHeader:count>
         <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
+        <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">4</rdeHeader:count>
     </rdeHeader:header>
 </rdeReport:report>

core/src/test/resources/google/registry/tools/server/xn--q9jyb4c_2010-10-17_full_S1_R0.xml

@@ -135,7 +135,7 @@
   <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeContact-1.0">0</rdeHeader:count>
   <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeHost-1.0">1</rdeHeader:count>
   <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeRegistrar-1.0">2</rdeHeader:count>
-  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">3</rdeHeader:count>
+  <rdeHeader:count uri="urn:ietf:params:xml:ns:rdeIDN-1.0">4</rdeHeader:count>
 </rdeHeader:header>
 
 </rde:contents>