Start using Tld's bsaEnrollStartTime field (#2239)

* Start using Tld's bsaEnrollStartTime field

    Longer-term change is tracked in b/309175410

core/src/main/java/google/registry/bsa/IdnChecker.java

@@ -16,8 +16,8 @@ package google.registry.bsa;
 
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static com.google.common.collect.Maps.transformValues;
+import static google.registry.model.tld.Tld.isEnrolledWithBsa;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.ImmutableSet;
@@ -50,14 +50,6 @@ public class IdnChecker {
     allTlds = idnToTlds.values().stream().flatMap(ImmutableSet::stream).collect(toImmutableSet());
   }
 
-  // TODO(11/30/2023): Remove below when new Tld schema is deployed and the `getBsaEnrollStartTime`
-  // method is no longer hardcoded.
-  @VisibleForTesting
-  IdnChecker(ImmutableMap<IdnTableEnum, ImmutableSet<Tld>> idnToTlds) {
-    this.idnToTlds = idnToTlds;
-    allTlds = idnToTlds.values().stream().flatMap(ImmutableSet::stream).collect(toImmutableSet());
-  }
-
   /** Returns all IDNs in which the {@code label} is valid. */
   ImmutableSet<IdnTableEnum> getAllValidIdns(String label) {
     return idnToTlds.keySet().stream()
@@ -88,11 +80,6 @@ public class IdnChecker {
     return Sets.difference(allTlds, getSupportingTlds(idnTables));
   }
 
-  private static boolean isEnrolledWithBsa(Tld tld, DateTime now) {
-    DateTime enrollTime = tld.getBsaEnrollStartTime();
-    return enrollTime != null && enrollTime.isBefore(now);
-  }
-
   private static ImmutableMap<IdnTableEnum, ImmutableSet<Tld>> getIdnToTldMap(DateTime now) {
     ImmutableMultimap.Builder<IdnTableEnum, Tld> idnToTldMap = new ImmutableMultimap.Builder();
     Tlds.getTldEntitiesOfType(TldType.REAL).stream()

core/src/main/java/google/registry/model/tld/Tld.java

@@ -256,6 +256,11 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
     return VKey.create(Tld.class, tldStr);
   }
 
+  /** Checks if {@code tld} is enrolled with BSA. */
+  public static boolean isEnrolledWithBsa(Tld tld, DateTime now) {
+    return tld.getBsaEnrollStartTime().orElse(END_OF_TIME).isBefore(now);
+  }
+
   /**
    * The name of the pricing engine that this TLD uses.
    *
@@ -550,9 +555,15 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
   @JsonSerialize(using = SortedEnumSetSerializer.class)
   Set<IdnTableEnum> idnTables;
 
-  // TODO(11/30/2023): uncomment below two lines
-  // /** The start time of this TLD's enrollment in the BSA program, if applicable. */
-  // @JsonIgnore @Nullable DateTime bsaEnrollStartTime;
+  /**
+   * The start time of this TLD's enrollment in the BSA program, if applicable.
+   *
+   * <p>This property is excluded from source-based configuration and is managed directly in the
+   * database.
+   */
+  // TODO(b/309175410): implement setup and cleanup procedure for joining or leaving BSA, and see
+  // if it can be integrated with the ConfigTldCommand.
+  @JsonIgnore @Nullable DateTime bsaEnrollStartTime;
 
   public String getTldStr() {
     return tldStr;
@@ -574,12 +585,9 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
   }
 
   /** Returns the time when this TLD was enrolled in the Brand Safety Alliance (BSA) program. */
-  @JsonIgnore // Annotation can be removed once we add the field and annotate it.
-  @Nullable
-  public DateTime getBsaEnrollStartTime() {
-    // TODO(11/30/2023): uncomment below.
-    // return this.bsaEnrollStartTime;
-    return null;
+  @JsonIgnore
+  public Optional<DateTime> getBsaEnrollStartTime() {
+    return Optional.ofNullable(this.bsaEnrollStartTime);
   }
 
   /** Retrieve whether invoicing is enabled. */
@@ -1101,10 +1109,9 @@ public class Tld extends ImmutableObject implements Buildable, UnsafeSerializabl
       return this;
     }
 
-    public Builder setBsaEnrollStartTime(DateTime enrollTime) {
+    public Builder setBsaEnrollStartTime(Optional<DateTime> enrollTime) {
       // TODO(b/309175133): forbid if enrolled with BSA
-      // TODO(11/30/2023): uncomment below line
-      // getInstance().bsaEnrollStartTime = enrollTime;
+      getInstance().bsaEnrollStartTime = enrollTime.orElse(null);
       return this;
     }
 

core/src/main/java/google/registry/request/UrlConnectionUtils.java

@@ -28,6 +28,7 @@ import com.google.common.net.MediaType;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.HttpURLConnection;
 import java.net.URLConnection;
 import java.util.Random;
 
@@ -36,26 +37,37 @@ public final class UrlConnectionUtils {
 
   private UrlConnectionUtils() {}
 
-  /** Retrieves the response from the given connection as a byte array. */
-  public static byte[] getResponseBytes(URLConnection connection) throws IOException {
-    try (InputStream is = connection.getInputStream()) {
+  /**
+   * Retrieves the response from the given connection as a byte array.
+   *
+   * <p>Note that in the event the response code is 4XX or 5XX, we use the error stream as any
+   * payload is included there.
+   *
+   * @see HttpURLConnection#getErrorStream()
+   */
+  public static byte[] getResponseBytes(HttpURLConnection connection) throws IOException {
+    int responseCode = connection.getResponseCode();
+    try (InputStream is =
+        responseCode < 400 ? connection.getInputStream() : connection.getErrorStream()) {
       return ByteStreams.toByteArray(is);
+    } catch (NullPointerException e) {
+      return new byte[] {};
     }
   }
 
   /** Sets auth on the given connection with the given username/password. */
-  public static void setBasicAuth(URLConnection connection, String username, String password) {
+  public static void setBasicAuth(HttpURLConnection connection, String username, String password) {
     setBasicAuth(connection, String.format("%s:%s", username, password));
   }
 
   /** Sets auth on the given connection with the given string, formatted "username:password". */
-  public static void setBasicAuth(URLConnection connection, String usernameAndPassword) {
+  public static void setBasicAuth(HttpURLConnection connection, String usernameAndPassword) {
     String token = base64().encode(usernameAndPassword.getBytes(UTF_8));
     connection.setRequestProperty(AUTHORIZATION, "Basic " + token);
   }
 
   /** Sets the given byte[] payload on the given connection with a particular content type. */
-  public static void setPayload(URLConnection connection, byte[] bytes, String contentType)
+  public static void setPayload(HttpURLConnection connection, byte[] bytes, String contentType)
       throws IOException {
     connection.setRequestProperty(CONTENT_TYPE, contentType);
     connection.setDoOutput(true);
@@ -72,7 +84,7 @@ public final class UrlConnectionUtils {
    * @see <a href="http://www.ietf.org/rfc/rfc2388.txt">RFC2388 - Returning Values from Forms</a>
    */
   public static void setPayloadMultipart(
-      URLConnection connection,
+      HttpURLConnection connection,
       String name,
       String filename,
       MediaType contentType,

core/src/main/java/google/registry/tools/ConfigureTldCommand.java

@@ -122,6 +122,13 @@ public class ConfigureTldCommand extends MutatingCommand {
     checkPremiumList(newTld);
     checkDnsWriters(newTld);
     checkCurrency(newTld);
+    // bsaEnrollStartTime only exists in DB. Need to carry it over to the updated copy. See Tld.java
+    // for more information.
+    Optional<DateTime> bsaEnrollTime =
+        Optional.ofNullable(oldTld).flatMap(Tld::getBsaEnrollStartTime);
+    if (bsaEnrollTime.isPresent()) {
+      newTld = newTld.asBuilder().setBsaEnrollStartTime(bsaEnrollTime).build();
+    }
     // Set the new TLD to breakglass mode if breakglass flag was used
     if (breakglass) {
       newTld = newTld.asBuilder().setBreakglassMode(true).build();

core/src/test/java/google/registry/bsa/IdnCheckerTest.java

@@ -15,38 +15,65 @@
 package google.registry.bsa;
 
 import static com.google.common.truth.Truth.assertThat;
+import static google.registry.testing.DatabaseHelper.createTld;
+import static google.registry.testing.DatabaseHelper.persistResource;
 import static google.registry.tldconfig.idn.IdnTableEnum.EXTENDED_LATIN;
 import static google.registry.tldconfig.idn.IdnTableEnum.JA;
 import static google.registry.tldconfig.idn.IdnTableEnum.UNCONFUSABLE_LATIN;
 
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import google.registry.model.tld.Tld;
+import google.registry.persistence.transaction.JpaTestExtensions;
+import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
+import google.registry.testing.FakeClock;
+import java.util.Optional;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.Mock;
-import org.mockito.junit.jupiter.MockitoExtension;
+import org.junit.jupiter.api.extension.RegisterExtension;
 
-@ExtendWith(MockitoExtension.class)
+/** Unit tests for {@link IdnChecker}. */
 public class IdnCheckerTest {
 
-  @Mock Tld jaonly;
-  @Mock Tld jandelatin;
-  @Mock Tld strictlatin;
+  FakeClock fakeClock = new FakeClock();
+
+  @RegisterExtension
+  final JpaIntegrationWithCoverageExtension jpa =
+      new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension();
+
+  Tld jaonly;
+  Tld jandelatin;
+  Tld strictlatin;
   IdnChecker idnChecker;
 
   @BeforeEach
   void setup() {
-    idnChecker =
-        new IdnChecker(
-            ImmutableMap.of(
-                JA,
-                ImmutableSet.of(jandelatin, jaonly),
-                EXTENDED_LATIN,
-                ImmutableSet.of(jandelatin),
-                UNCONFUSABLE_LATIN,
-                ImmutableSet.of(strictlatin)));
+    jaonly = createTld("jaonly");
+    jandelatin = createTld("jandelatin");
+    strictlatin = createTld("strictlatin");
+
+    jaonly =
+        persistResource(
+            jaonly
+                .asBuilder()
+                .setBsaEnrollStartTime(Optional.of(fakeClock.nowUtc()))
+                .setIdnTables(ImmutableSet.of(JA))
+                .build());
+    jandelatin =
+        persistResource(
+            jandelatin
+                .asBuilder()
+                .setBsaEnrollStartTime(Optional.of(fakeClock.nowUtc()))
+                .setIdnTables(ImmutableSet.of(JA, EXTENDED_LATIN))
+                .build());
+    strictlatin =
+        persistResource(
+            strictlatin
+                .asBuilder()
+                .setBsaEnrollStartTime(Optional.of(fakeClock.nowUtc()))
+                .setIdnTables(ImmutableSet.of(UNCONFUSABLE_LATIN))
+                .build());
+    fakeClock.advanceOneMilli();
+    idnChecker = new IdnChecker(fakeClock);
   }
 
   @Test

core/src/test/java/google/registry/rde/RdeReportActionTest.java

@@ -263,7 +263,7 @@ public class RdeReportActionTest {
   @Test
   void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
     when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST);
-    when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
+    when(httpUrlConnection.getErrorStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
     InternalServerErrorException thrown =
         assertThrows(
             InternalServerErrorException.class,

core/src/test/java/google/registry/reporting/icann/IcannHttpReporterTest.java

@@ -99,10 +99,10 @@ class IcannHttpReporterTest {
 
   @Test
   void testFail_BadIirdeaResponse() throws Exception {
-    when(connection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
     when(connection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST);
+    when(connection.getErrorStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
     assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse();
-    verify(connection).getInputStream();
+    verify(connection).getErrorStream();
   }
 
   @Test

core/src/test/java/google/registry/request/UrlConnectionUtilsTest.java

@@ -29,6 +29,7 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.util.Arrays;
 import java.util.List;
@@ -104,4 +105,21 @@ public class UrlConnectionUtilsTest {
             "Multipart data contains autogenerated boundary: "
                 + "------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
   }
+
+  @Test
+  void testErrorStream() throws Exception {
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
+    when(connection.getResponseCode()).thenReturn(400);
+    when(connection.getErrorStream())
+        .thenReturn(new ByteArrayInputStream("Failure".getBytes(UTF_8)));
+    assertThat(UrlConnectionUtils.getResponseBytes(connection))
+        .isEqualTo("Failure".getBytes(UTF_8));
+  }
+
+  @Test
+  void testErrorStream_null() throws Exception {
+    HttpsURLConnection connection = mock(HttpsURLConnection.class);
+    when(connection.getResponseCode()).thenReturn(400);
+    assertThat(UrlConnectionUtils.getResponseBytes(connection)).isEmpty();
+  }
 }

core/src/test/java/google/registry/tmch/NordnUploadActionTest.java

@@ -108,6 +108,8 @@ class NordnUploadActionTest {
   void beforeEach() throws Exception {
     when(httpUrlConnection.getInputStream())
         .thenReturn(new ByteArrayInputStream("Success".getBytes(UTF_8)));
+    when(httpUrlConnection.getErrorStream())
+        .thenReturn(new ByteArrayInputStream("Failure".getBytes(UTF_8)));
     when(httpUrlConnection.getResponseCode()).thenReturn(SC_ACCEPTED);
     when(httpUrlConnection.getHeaderField(LOCATION)).thenReturn("http://trololol");
     when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);

core/src/test/java/google/registry/tools/ConfigureTldCommandTest.java

@@ -14,9 +14,9 @@
 package google.registry.tools;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.Truth8.assertThat;
 import static google.registry.model.EntityYamlUtils.createObjectMapper;
 import static google.registry.model.domain.token.AllocationToken.TokenType.DEFAULT_PROMO;
-import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
 import static google.registry.testing.DatabaseHelper.createTld;
 import static google.registry.testing.DatabaseHelper.persistPremiumList;
 import static google.registry.testing.DatabaseHelper.persistResource;
@@ -46,6 +46,7 @@ import google.registry.model.tld.Tld.TldNotFoundException;
 import google.registry.model.tld.label.PremiumList;
 import google.registry.model.tld.label.PremiumListDao;
 import java.io.File;
+import java.util.Optional;
 import java.util.logging.Logger;
 import org.joda.money.Money;
 import org.joda.time.DateTime;
@@ -101,20 +102,19 @@ public class ConfigureTldCommandTest extends CommandTestCase<ConfigureTldCommand
     assertThat(updatedTld.getCreateBillingCost()).isEqualTo(Money.of(USD, 25));
     testTldConfiguredSuccessfully(updatedTld, "tld.yaml");
     assertThat(updatedTld.getBreakglassMode()).isFalse();
-    assertThat(tld.getBsaEnrollStartTime()).isNull();
+    assertThat(tld.getBsaEnrollStartTime()).isEmpty();
   }
 
   @Test
-  void testSuccess_updateTld_bsaTimeUnaffected() throws Exception {
+  void testSuccess_updateTld_existingBsaTimeCarriedOver() throws Exception {
     Tld tld = createTld("tld");
     DateTime bsaStartTime = DateTime.now(DateTimeZone.UTC);
-    tm().transact(() -> tm().put(tld.asBuilder().setBsaEnrollStartTime(bsaStartTime).build()));
+    persistResource(tld.asBuilder().setBsaEnrollStartTime(Optional.of(bsaStartTime)).build());
     File tldFile = tmpDir.resolve("tld.yaml").toFile();
     Files.asCharSink(tldFile, UTF_8).write(loadFile(getClass(), "tld.yaml"));
     runCommandForced("--input=" + tldFile);
-    // TODO(11/30/2023): uncomment below two lines
-    // Tld updatedTld = Tld.get("tld");
-    // assertThat(tld.getBsaEnrollStartTime()).isEqualTo(bsaStartTime);
+    Tld updatedTld = Tld.get("tld");
+    assertThat(updatedTld.getBsaEnrollStartTime()).hasValue(bsaStartTime);
   }
 
   @Test

db/src/main/resources/sql/flyway.txt

@@ -149,3 +149,4 @@ V148__add_bsa_download_and_label_tables.sql
 V149__add_bsa_domain_in_use_table.sql
 V150__add_tld_bsa_enroll_date.sql
 V151__add_bsa_unblockable_domain_table.sql
+V152__add_bsa_domain_refresh_table.sql

db/src/main/resources/sql/flyway/V152__add_bsa_domain_refresh_table.sql

@@ -0,0 +1,21 @@
+-- Copyright 2023 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE TABLE "BsaDomainRefresh" (
+   job_id  bigserial not null,
+   creation_time timestamptz not null,
+   stage text not null,
+   update_timestamp timestamptz,
+   primary key (job_id)
+);

db/src/main/resources/sql/schema/db-schema.sql.generated

@@ -728,6 +728,7 @@
         auto_renew_grace_period_length interval not null,
         automatic_transfer_length interval not null,
         breakglass_mode boolean not null,
+        bsa_enroll_start_time timestamptz,
         claims_period_end timestamptz not null,
         create_billing_cost_amount numeric(19, 2),
         create_billing_cost_currency text,

db/src/main/resources/sql/schema/nomulus.golden.sql

@@ -135,6 +135,37 @@ CREATE TABLE public."BsaDomainInUse" (
 );
 
 
+--
+-- Name: BsaDomainRefresh; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public."BsaDomainRefresh" (
+    job_id bigint NOT NULL,
+    creation_time timestamp with time zone NOT NULL,
+    stage text NOT NULL,
+    update_timestamp timestamp with time zone
+);
+
+
+--
+-- Name: BsaDomainRefresh_job_id_seq; Type: SEQUENCE; Schema: public; Owner: -
+--
+
+CREATE SEQUENCE public."BsaDomainRefresh_job_id_seq"
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: BsaDomainRefresh_job_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
+--
+
+ALTER SEQUENCE public."BsaDomainRefresh_job_id_seq" OWNED BY public."BsaDomainRefresh".job_id;
+
+
 --
 -- Name: BsaDownload; Type: TABLE; Schema: public; Owner: -
 --
@@ -1222,6 +1253,13 @@ CREATE SEQUENCE public.project_wide_unique_id_seq
     CACHE 10;
 
 
+--
+-- Name: BsaDomainRefresh job_id; Type: DEFAULT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."BsaDomainRefresh" ALTER COLUMN job_id SET DEFAULT nextval('public."BsaDomainRefresh_job_id_seq"'::regclass);
+
+
 --
 -- Name: BsaDownload job_id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -1339,6 +1377,14 @@ ALTER TABLE ONLY public."BsaDomainInUse"
     ADD CONSTRAINT "BsaDomainInUse_pkey" PRIMARY KEY (label, tld);
 
 
+--
+-- Name: BsaDomainRefresh BsaDomainRefresh_pkey; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public."BsaDomainRefresh"
+    ADD CONSTRAINT "BsaDomainRefresh_pkey" PRIMARY KEY (job_id);
+
+
 --
 -- Name: BsaDownload BsaDownload_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --

release/cloudbuild-deploy.yaml

@@ -78,8 +78,8 @@ steps:
       grep -e "^backend\|^default\|^bsa\|^pubapi\|^tools" |\
       while read line; do echo "${TAG_NAME},$line"; done | tee "$local_map"
     num_versions=$(cat "$local_map" | wc -l)
-    if [ "$num_versions" -ne 4 ]; then
-      echo "Expecting exactly four active services. Found $num_versions"
+    if [ "$num_versions" -ne 5 ]; then
+      echo "Expecting exactly five active services. Found $num_versions"
       exit 1
     fi
     gsutil cp "$local_map" gs://$PROJECT_ID-deployed-tags/nomulus.${_ENV}.tmp