Add indexes to DomainHistory sub tables (#1319 )

* Add indexes to DomainHistory sub tables Add indexes to DomainTransactionRecord and DomainDsDataHistory to speed up query for DomainHistory. Without these indexes, DomainHistory loading is extremely slow: 2 QPS with current production data.
Rename Spec11Pipeline's Subdomain -> DomainNameInfo (#1318 )
2026-07-16 13:02:29 +00:00 · 2021-09-14 14:54:21 -04:00 · 2021-09-14 14:07:26 -04:00 · 2021-09-13 18:02:41 -04:00 · 2021-09-13 16:55:20 -04:00 · 2021-09-13 15:12:52 -04:00
52 changed files with 473 additions and 328 deletions
@@ -22,6 +22,8 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
 import google.registry.backup.AppEngineEnvironment;
 import google.registry.beam.common.RegistryQuery.CriteriaQuerySupplier;
+import google.registry.model.UpdateAutoTimestamp;
+import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
 import google.registry.model.ofy.ObjectifyService;
 import google.registry.model.replay.SqlEntity;
 import google.registry.persistence.transaction.JpaTransactionManager;
@@ -274,6 +276,12 @@ public final class RegistryJpaIO {

    public abstract SerializableFunction<T, Object> jpaConverter();

+    /**
+     * Signal to the writer that the {@link UpdateAutoTimestamp} property should be allowed to
+     * manipulate its value before persistence. The default value is {@code true}.
+     */
+    abstract boolean withUpdateAutoTimestamp();
+
    public Write<T> withName(String name) {
      return toBuilder().name(name).build();
    }
@@ -294,6 +302,10 @@ public final class RegistryJpaIO {
      return toBuilder().jpaConverter(jpaConverter).build();
    }

+    public Write<T> disableUpdateAutoTimestamp() {
+      return toBuilder().withUpdateAutoTimestamp(false).build();
+    }
+
    abstract Builder<T> toBuilder();

    @Override
@@ -310,7 +322,7 @@ public final class RegistryJpaIO {
              GroupIntoBatches.<Integer, T>ofSize(batchSize()).withShardedKey())
          .apply(
              "Write in batch for " + name(),
-              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter())));
+              ParDo.of(new SqlBatchWriter<>(name(), jpaConverter(), withUpdateAutoTimestamp())));
    }

    static <T> Builder<T> builder() {
@@ -318,7 +330,8 @@ public final class RegistryJpaIO {
          .name(DEFAULT_NAME)
          .batchSize(DEFAULT_BATCH_SIZE)
          .shards(DEFAULT_SHARDS)
-          .jpaConverter(x -> x);
+          .jpaConverter(x -> x)
+          .withUpdateAutoTimestamp(true);
    }

    @AutoValue.Builder
@@ -332,6 +345,8 @@ public final class RegistryJpaIO {

      abstract Builder<T> jpaConverter(SerializableFunction<T, Object> jpaConverter);

+      abstract Builder<T> withUpdateAutoTimestamp(boolean withUpdateAutoTimestamp);
+
      abstract Write<T> build();
    }
  }
@@ -340,10 +355,13 @@ public final class RegistryJpaIO {
  private static class SqlBatchWriter<T> extends DoFn<KV<ShardedKey<Integer>, Iterable<T>>, Void> {
    private final Counter counter;
    private final SerializableFunction<T, Object> jpaConverter;
+    private final boolean withAutoTimestamp;

-    SqlBatchWriter(String type, SerializableFunction<T, Object> jpaConverter) {
+    SqlBatchWriter(
+        String type, SerializableFunction<T, Object> jpaConverter, boolean withAutoTimestamp) {
      counter = Metrics.counter("SQL_WRITE", type);
      this.jpaConverter = jpaConverter;
+      this.withAutoTimestamp = withAutoTimestamp;
    }

    @Setup
@@ -358,6 +376,16 @@ public final class RegistryJpaIO {

    @ProcessElement
    public void processElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
+      if (withAutoTimestamp) {
+        actuallyProcessElement(kv);
+        return;
+      }
+      try (DisableAutoUpdateResource disable = UpdateAutoTimestamp.disableAutoUpdate()) {
+        actuallyProcessElement(kv);
+      }
+    }
+
+    private void actuallyProcessElement(@Element KV<ShardedKey<Integer>, Iterable<T>> kv) {
      try (AppEngineEnvironment env = new AppEngineEnvironment()) {
        ImmutableList<Object> entities =
            Streams.stream(kv.getValue())
@@ -16,6 +16,7 @@ package google.registry.beam.common;

 import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;

+import google.registry.persistence.transaction.JpaTransactionManager;
 import java.io.Serializable;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -28,6 +29,15 @@ import javax.persistence.criteria.CriteriaQuery;

 /** Interface for query instances used by {@link RegistryJpaIO.Read}. */
 public interface RegistryQuery<T> extends Serializable {
+
+  /**
+   * Number of JPA entities to fetch in each batch during a query.
+   *
+   * <p>With Hibernate, for result streaming to work, a query's fetchSize property must be set to a
+   * non-zero value.
+   */
+  int QUERY_FETCH_SIZE = 1000;
+
  Stream<T> stream();

  interface CriteriaQuerySupplier<T> extends Supplier<CriteriaQuery<T>>, Serializable {}
@@ -49,6 +59,7 @@ public interface RegistryQuery<T> extends Serializable {
      if (parameters != null) {
        parameters.forEach(query::setParameter);
      }
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
      @SuppressWarnings("unchecked")
      Stream<T> resultStream = query.getResultStream();
      return nativeQuery ? resultStream : resultStream.map(e -> detach(entityManager, e));
@@ -64,11 +75,14 @@ public interface RegistryQuery<T> extends Serializable {
  static <T> RegistryQuery<T> createQuery(
      String jpql, @Nullable Map<String, Object> parameters, Class<T> clazz) {
    return () -> {
-      TypedQuery<T> query = jpaTm().query(jpql, clazz);
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(jpql, clazz);
      if (parameters != null) {
        parameters.forEach(query::setParameter);
      }
-      return query.getResultStream();
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
    };
  }

@@ -82,7 +96,13 @@ public interface RegistryQuery<T> extends Serializable {
   * @param <T> Type of each row in the result set.
   */
  static <T> RegistryQuery<T> createQuery(CriteriaQuerySupplier<T> criteriaQuery) {
-    return () -> jpaTm().query(criteriaQuery.get()).getResultStream();
+    return () -> {
+      // TODO(b/193662898): switch to jpaTm().query() when it can properly detach loaded entities.
+      EntityManager entityManager = jpaTm().getEntityManager();
+      TypedQuery<T> query = entityManager.createQuery(criteriaQuery.get());
+      JpaTransactionManager.setQueryFetchSize(query, QUERY_FETCH_SIZE);
+      return query.getResultStream().map(e -> detach(entityManager, e));
+    };
  }

  /**
@@ -108,7 +128,10 @@ public interface RegistryQuery<T> extends Serializable {
      return;
    }
    try {
-      entityManager.detach(object);
+      // TODO(b/193662898): choose detach() or clear() based on the type of transaction.
+      // For context, EntityManager.detach() does not remove all metadata about loaded entities.
+      // See b/193925312 or https://hibernate.atlassian.net/browse/HHH-14735 for details.
+      entityManager.clear();
    } catch (IllegalArgumentException e) {
      // Not an entity. Do nothing.
    }
@@ -219,7 +219,8 @@ public class InitSqlPipeline implements Serializable {
            .withName(transformId)
            .withBatchSize(options.getSqlWriteBatchSize())
            .withShards(options.getSqlWriteShards())
-            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity));
+            .withJpaConverter(Transforms::convertVersionedEntityToSqlEntity)
+            .disableUpdateAutoTimestamp());
  }

  private static ImmutableList<String> toKindStrings(Collection<Class<?>> entityClasses) {
@@ -25,7 +25,7 @@ import org.apache.avro.generic.GenericRecord;
 import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;

 /**
- * A POJO representing a single subdomain, parsed from a {@code SchemaAndRecord}.
+ * A POJO representing a domain name and associated info, parsed from a {@code SchemaAndRecord}.
 *
 * <p>This is a trivially serializable class that allows Beam to transform the results of a Bigquery
 * query into a standard Java representation, giving us the type guarantees and ease of manipulation
@@ -33,28 +33,31 @@ import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord;
 * function.
 */
@AutoValue
-public abstract class Subdomain implements Serializable {
+public abstract class DomainNameInfo implements Serializable {

  private static final ImmutableList<String> FIELD_NAMES =
      ImmutableList.of("domainName", "domainRepoId", "registrarId", "registrarEmailAddress");

  /** Returns the fully qualified domain name. */
  abstract String domainName();
+
  /** Returns the domain repo ID (the primary key of the domain table). */
  abstract String domainRepoId();
+
  /** Returns the registrar ID of the associated registrar for this domain. */
  abstract String registrarId();
+
  /** Returns the email address of the registrar associated with this domain. */
  abstract String registrarEmailAddress();

  /**
-   * Constructs a {@link Subdomain} from an Apache Avro {@code SchemaAndRecord}.
+   * Constructs a {@link DomainNameInfo} from an Apache Avro {@code SchemaAndRecord}.
   *
   * @see <a
   *     href=http://avro.apache.org/docs/1.7.7/api/java/org/apache/avro/generic/GenericData.Record.html>
   *     Apache AVRO GenericRecord</a>
   */
-  static Subdomain parseFromRecord(SchemaAndRecord schemaAndRecord) {
+  static DomainNameInfo parseFromRecord(SchemaAndRecord schemaAndRecord) {
    checkFieldsNotNull(FIELD_NAMES, schemaAndRecord);
    GenericRecord record = schemaAndRecord.getRecord();
    return create(
@@ -65,18 +68,15 @@ public abstract class Subdomain implements Serializable {
  }

  /**
-   * Creates a concrete {@link Subdomain}.
+   * Creates a concrete {@link DomainNameInfo}.
   *
-   * <p>This should only be used outside this class for testing- instances of {@link Subdomain}
+   * <p>This should only be used outside this class for testing- instances of {@link DomainNameInfo}
   * should otherwise come from {@link #parseFromRecord}.
   */
  @VisibleForTesting
-  static Subdomain create(
-      String domainName,
-      String domainRepoId,
-      String registrarId,
-      String registrarEmailAddress) {
-    return new AutoValue_Subdomain(
+  static DomainNameInfo create(
+      String domainName, String domainRepoId, String registrarId, String registrarEmailAddress) {
+    return new AutoValue_DomainNameInfo(
        domainName, domainRepoId, registrarId, registrarEmailAddress);
  }
 }
@@ -55,13 +55,14 @@ public class SafeBrowsingTransforms {
      "https://safebrowsing.googleapis.com/v4/threatMatches:find";

  /**
-   * {@link DoFn} mapping a {@link Subdomain} to its evaluation report from SafeBrowsing.
+   * {@link DoFn} mapping a {@link DomainNameInfo} to its evaluation report from SafeBrowsing.
   *
   * <p>Refer to the Lookup API documentation for the request/response format and other details.
   *
   * @see <a href=https://developers.google.com/safe-browsing/v4/lookup-api>Lookup API</a>
   */
-  static class EvaluateSafeBrowsingFn extends DoFn<Subdomain, KV<Subdomain, ThreatMatch>> {
+  static class EvaluateSafeBrowsingFn
+      extends DoFn<DomainNameInfo, KV<DomainNameInfo, ThreatMatch>> {

    /**
     * Max number of urls we can check in a single query.
@@ -74,10 +75,11 @@ public class SafeBrowsingTransforms {
    private final String apiKey;

    /**
-     * Maps a subdomain's {@code fullyQualifiedDomainName} to its corresponding {@link Subdomain} to
-     * facilitate batching SafeBrowsing API requests.
+     * Maps a domain name's {@code fullyQualifiedDomainName} to its corresponding {@link
+     * DomainNameInfo} to facilitate batching SafeBrowsing API requests.
     */
-    private final Map<String, Subdomain> subdomainBuffer = new LinkedHashMap<>(BATCH_SIZE);
+    private final Map<String, DomainNameInfo> domainNameInfoBuffer =
+        new LinkedHashMap<>(BATCH_SIZE);

    /**
     * Provides the HTTP client we use to interact with the SafeBrowsing API.
@@ -119,37 +121,38 @@ public class SafeBrowsingTransforms {
      closeableHttpClientSupplier = clientSupplier;
    }

-    /** Evaluates any buffered {@link Subdomain} objects upon completing the bundle. */
+    /** Evaluates any buffered {@link DomainNameInfo} objects upon completing the bundle. */
    @FinishBundle
    public void finishBundle(FinishBundleContext context) {
-      if (!subdomainBuffer.isEmpty()) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      if (!domainNameInfoBuffer.isEmpty()) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
        results.forEach((kv) -> context.output(kv, Instant.now(), GlobalWindow.INSTANCE));
      }
    }

    /**
-     * Buffers {@link Subdomain} objects until we reach the batch size, then bulk-evaluate the URLs
-     * with the SafeBrowsing API.
+     * Buffers {@link DomainNameInfo} objects until we reach the batch size, then bulk-evaluate the
+     * URLs with the SafeBrowsing API.
     */
    @ProcessElement
    public void processElement(ProcessContext context) {
-      Subdomain subdomain = context.element();
-      subdomainBuffer.put(subdomain.domainName(), subdomain);
-      if (subdomainBuffer.size() >= BATCH_SIZE) {
-        ImmutableSet<KV<Subdomain, ThreatMatch>> results = evaluateAndFlush();
+      DomainNameInfo domainNameInfo = context.element();
+      domainNameInfoBuffer.put(domainNameInfo.domainName(), domainNameInfo);
+      if (domainNameInfoBuffer.size() >= BATCH_SIZE) {
+        ImmutableSet<KV<DomainNameInfo, ThreatMatch>> results = evaluateAndFlush();
        results.forEach(context::output);
      }
    }

    /**
-     * Evaluates all {@link Subdomain} objects in the buffer and returns a list of key-value pairs
-     * from {@link Subdomain} to its SafeBrowsing report.
+     * Evaluates all {@link DomainNameInfo} objects in the buffer and returns a list of key-value
+     * pairs from {@link DomainNameInfo} to its SafeBrowsing report.
     *
-     * <p>If a {@link Subdomain} is safe according to the API, it will not emit a report.
+     * <p>If a {@link DomainNameInfo} is safe according to the API, it will not emit a report.
     */
-    private ImmutableSet<KV<Subdomain, ThreatMatch>> evaluateAndFlush() {
-      ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder = new ImmutableSet.Builder<>();
+    private ImmutableSet<KV<DomainNameInfo, ThreatMatch>> evaluateAndFlush() {
+      ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder =
+          new ImmutableSet.Builder<>();
      try {
        URIBuilder uriBuilder = new URIBuilder(SAFE_BROWSING_URL);
        // Add the API key param
@@ -174,7 +177,7 @@ public class SafeBrowsingTransforms {
        throw new RuntimeException("Caught parsing exception, failing pipeline.", e);
      } finally {
        // Flush the buffer
-        subdomainBuffer.clear();
+        domainNameInfoBuffer.clear();
      }
      return resultBuilder.build();
    }
@@ -183,7 +186,7 @@ public class SafeBrowsingTransforms {
    private JSONObject createRequestBody() throws JSONException {
      // Accumulate all domain names to evaluate.
      JSONArray threatArray = new JSONArray();
-      for (String fullyQualifiedDomainName : subdomainBuffer.keySet()) {
+      for (String fullyQualifiedDomainName : domainNameInfoBuffer.keySet()) {
        threatArray.put(new JSONObject().put("url", fullyQualifiedDomainName));
      }
      // Construct the JSON request body
@@ -211,7 +214,7 @@ public class SafeBrowsingTransforms {
     */
    private void processResponse(
        CloseableHttpResponse response,
-        ImmutableSet.Builder<KV<Subdomain, ThreatMatch>> resultBuilder)
+        ImmutableSet.Builder<KV<DomainNameInfo, ThreatMatch>> resultBuilder)
        throws JSONException, IOException {
      int statusCode = response.getStatusLine().getStatusCode();
      if (statusCode != SC_OK) {
@@ -226,16 +229,17 @@ public class SafeBrowsingTransforms {
        if (responseBody.length() == 0) {
          logger.atInfo().log("Response was empty, no threats detected");
        } else {
-          // Emit all Subdomains with their API results.
+          // Emit all DomainNameInfos with their API results.
          JSONArray threatMatches = responseBody.getJSONArray("matches");
          for (int i = 0; i < threatMatches.length(); i++) {
            JSONObject match = threatMatches.getJSONObject(i);
            String url = match.getJSONObject("threat").getString("url");
-            Subdomain subdomain = subdomainBuffer.get(url);
+            DomainNameInfo domainNameInfo = domainNameInfoBuffer.get(url);
            resultBuilder.add(
                KV.of(
-                    subdomain,
-                    ThreatMatch.create(match.getString("threatType"), subdomain.domainName())));
+                    domainNameInfo,
+                    ThreatMatch.create(
+                        match.getString("threatType"), domainNameInfo.domainName())));
          }
        }
      }
@@ -100,20 +100,20 @@ public class Spec11Pipeline implements Serializable {

  void setupPipeline(Pipeline pipeline) {
    options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
-    PCollection<Subdomain> domains =
+    PCollection<DomainNameInfo> domains =
        options.getDatabase().equals("DATASTORE")
            ? readFromBigQuery(options, pipeline)
            : readFromCloudSql(pipeline);

-    PCollection<KV<Subdomain, ThreatMatch>> threatMatches =
+    PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches =
        domains.apply("Run through SafeBrowsing API", ParDo.of(safeBrowsingFn));

    saveToSql(threatMatches, options);
    saveToGcs(threatMatches, options);
  }

-  static PCollection<Subdomain> readFromCloudSql(Pipeline pipeline) {
-    Read<Object[], Subdomain> read =
+  static PCollection<DomainNameInfo> readFromCloudSql(Pipeline pipeline) {
+    Read<Object[], DomainNameInfo> read =
        RegistryJpaIO.read(
            "select d, r.emailAddress from Domain d join Registrar r on"
                + " d.currentSponsorClientId = r.clientIdentifier where r.type = 'REAL'"
@@ -124,30 +124,31 @@ public class Spec11Pipeline implements Serializable {
    return pipeline.apply("Read active domains from Cloud SQL", read);
  }

-  static PCollection<Subdomain> readFromBigQuery(Spec11PipelineOptions options, Pipeline pipeline) {
+  static PCollection<DomainNameInfo> readFromBigQuery(
+      Spec11PipelineOptions options, Pipeline pipeline) {
    return pipeline.apply(
        "Read active domains from BigQuery",
-        BigQueryIO.read(Subdomain::parseFromRecord)
+        BigQueryIO.read(DomainNameInfo::parseFromRecord)
            .fromQuery(
-                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "subdomains.sql"))
+                SqlTemplate.create(getQueryFromFile(Spec11Pipeline.class, "domain_name_infos.sql"))
                    .put("PROJECT_ID", options.getProject())
                    .put("DATASTORE_EXPORT_DATASET", "latest_datastore_export")
                    .put("REGISTRAR_TABLE", "Registrar")
                    .put("DOMAIN_BASE_TABLE", "DomainBase")
                    .build())
-            .withCoder(SerializableCoder.of(Subdomain.class))
+            .withCoder(SerializableCoder.of(DomainNameInfo.class))
            .usingStandardSql()
            .withoutValidation()
            .withTemplateCompatibility());
  }

-  private static Subdomain parseRow(Object[] row) {
+  private static DomainNameInfo parseRow(Object[] row) {
    DomainBase domainBase = (DomainBase) row[0];
    String emailAddress = (String) row[1];
    if (emailAddress == null) {
      emailAddress = "";
    }
-    return Subdomain.create(
+    return DomainNameInfo.create(
        domainBase.getDomainName(),
        domainBase.getRepoId(),
        domainBase.getCurrentSponsorClientId(),
@@ -155,31 +156,31 @@ public class Spec11Pipeline implements Serializable {
  }

  static void saveToSql(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
    String transformId = "Spec11 Threat Matches";
    LocalDate date = LocalDate.parse(options.getDate(), ISODateTimeFormat.date());
    threatMatches.apply(
        "Write to Sql: " + transformId,
-        RegistryJpaIO.<KV<Subdomain, ThreatMatch>>write()
+        RegistryJpaIO.<KV<DomainNameInfo, ThreatMatch>>write()
            .withName(transformId)
            .withBatchSize(options.getSqlWriteBatchSize())
            .withShards(options.getSqlWriteShards())
            .withJpaConverter(
                (kv) -> {
-                  Subdomain subdomain = kv.getKey();
+                  DomainNameInfo domainNameInfo = kv.getKey();
                  return new Spec11ThreatMatch.Builder()
                      .setThreatTypes(
                          ImmutableSet.of(ThreatType.valueOf(kv.getValue().threatType())))
                      .setCheckDate(date)
-                      .setDomainName(subdomain.domainName())
-                      .setDomainRepoId(subdomain.domainRepoId())
-                      .setRegistrarId(subdomain.registrarId())
+                      .setDomainName(domainNameInfo.domainName())
+                      .setDomainRepoId(domainNameInfo.domainRepoId())
+                      .setRegistrarId(domainNameInfo.registrarId())
                      .build();
                }));
  }

  static void saveToGcs(
-      PCollection<KV<Subdomain, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
+      PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches, Spec11PipelineOptions options) {
    threatMatches
        .apply(
            "Map registrar ID to email/ThreatMatch pair",
@@ -187,7 +188,7 @@ public class Spec11Pipeline implements Serializable {
                    TypeDescriptors.kvs(
                        TypeDescriptors.strings(), TypeDescriptor.of(EmailAndThreatMatch.class)))
                .via(
-                    (KV<Subdomain, ThreatMatch> kv) ->
+                    (KV<DomainNameInfo, ThreatMatch> kv) ->
                        KV.of(
                            kv.getKey().registrarId(),
                            EmailAndThreatMatch.create(
@@ -230,7 +231,7 @@ public class Spec11Pipeline implements Serializable {
                        options.getReportingBucketUrl(),
                        getSpec11ReportFilePath(LocalDate.parse(options.getDate()))))
                .withoutSharding()
-                .withHeader("Map from registrar email / name to detected subdomain threats:"));
+                .withHeader("Map from registrar email / name to detected domain name threats:"));
  }

  public static void main(String[] args) {
@@ -459,46 +459,29 @@ sslCertificateValidation:
  expirationWarningEmailSubjectText: "[Important] Expiring SSL certificate for Google Registry EPP connection"
  # Text for expiring certificate notification email body that accepts 3 parameters:
  # registrar name, certificate type, and expiration date, respectively.
-  expirationWarningEmailBodyText: >
+  expirationWarningEmailBodyText: |
    Dear %1$s,

-    We would like to inform you that your %2$s SSL certificate will expire at
-    %3$s. Please take note that using expired certificates will prevent
-    successful Registry login.
+    We would like to inform you that your %2$s SSL certificate will expire at %3$s. Please take note that using expired certificates will prevent successful Registry login.

-    Kindly update your production account certificate within the support
-    console using the following steps:
+    Kindly update your production account certificate within the support console using the following steps:

-      1. Navigate to support.registry.google and login using your
-    %4$s@registry.google credentials.
-          * If this is your first time logging in, you will be prompted to
-          reset your password, so please keep your new password safe.
-          * If you are already logged in with some other Google account(s) but
-          not your %4$s@registry.google account, you need to click on
+      1. Navigate to support.registry.google and login using your %4$s@registry.google credentials.
+          * If this is your first time logging in, you will be prompted to reset your password, so please keep your new password safe.
+          * If you are already logged in with some other Google account(s) but not your %4$s@registry.google account, you need to click on
          “Add Account” and login using your %4$s@registry.google credentials.
      2. Select “Settings > Security” from the left navigation bar.
      3. Click “Edit” on the top left corner.
-      4. Enter your full certificate string
-         (including lines -----BEGIN CERTIFICATE----- and
-         -----END CERTIFICATE-----) in the box.
-      5. Click “Save”. If there are validation issues with the form, you will
-         be prompted to fix them and click “Save” again.
+      4. Enter your full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----) in the box.
+      5. Click “Save”. If there are validation issues with the form, you will be prompted to fix them and click “Save” again.

-    A failover SSL certificate can also be added in order to prevent connection
-    issues once your main certificate expires. Connecting with either of the
-    certificates will work with our production EPP server.
+    A failover SSL certificate can also be added in order to prevent connection issues once your main certificate expires. Connecting with either of the certificates will work with our production EPP server.

-    Further information about our EPP connection requirements can be found in
-    section 9.2 in the updated Technical Guide in your Google Drive folder.
+    Further information about our EPP connection requirements can be found in section 9.2 in the updated Technical Guide in your Google Drive folder.

-    Note that account certificate changes take a few minutes to become
-    effective and that the existing connections will remain unaffected by
-    the change.
+    Note that account certificate changes take a few minutes to become effective and that the existing connections will remain unaffected by the change.

-    If you also would like to update your OT&E account certificate, please send
-    an email from your primary or technical contact to
-    registry-support@google.com and include the full certificate string
-    (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).
+    If you also would like to update your OT&E account certificate, please send an email from your primary or technical contact to registry-support@google.com and include the full certificate string (including lines -----BEGIN CERTIFICATE----- and -----END CERTIFICATE-----).

    Regards,
    Google Registry
@@ -31,12 +31,12 @@ package google.registry.dns.writer;
 public interface DnsWriter {

  /**
-   * Loads {@code domainName} from Datastore and publishes its NS/DS records to the DNS server.
+   * Loads {@code domainName} from the database and publishes its NS/DS records to the DNS server.
   * Replaces existing records for the exact name supplied with an NS record for each name server
   * and a DS record for each delegation signer stored in the registry for the supplied domain name.
   * If the domain is deleted or is in a "non-publish" state then any existing records are deleted.
   *
-   * This must NOT actually perform any action, instead it should stage the action so that it's
+   * <p>This must NOT actually perform any action, instead it should stage the action so that it's
   * performed when {@link #commit()} is called.
   *
   * @param domainName the fully qualified domain name, with no trailing dot
@@ -44,14 +44,14 @@ public interface DnsWriter {
  void publishDomain(String domainName);

  /**
-   * Loads {@code hostName} from Datastore and publishes its A/AAAA glue records to the DNS server,
-   * if it is used as an in-bailiwick nameserver. Orphaned glue records are prohibited. Replaces
-   * existing records for the exact name supplied, with an A or AAAA record (as appropriate) for
-   * each address stored in the registry, for the supplied host name. If the host is deleted then
-   * the existing records are deleted. Assumes that this method will only be called for in-bailiwick
-   * hosts. The registry does not have addresses for other hosts.
+   * Loads {@code hostName} from the database and publishes its A/AAAA glue records to the DNS
+   * server, if it is used as an in-bailiwick nameserver. Orphaned glue records are prohibited.
+   * Replaces existing records for the exact name supplied, with an A or AAAA record (as
+   * appropriate) for each address stored in the registry, for the supplied host name. If the host
+   * is deleted then the existing records are deleted. Assumes that this method will only be called
+   * for in-bailiwick hosts. The registry does not have addresses for other hosts.
   *
-   * This must NOT actually perform any action, instead it should stage the action so that it's
+   * <p>This must NOT actually perform any action, instead it should stage the action so that it's
   * performed when {@link #commit()} is called.
   *
   * @param hostName the fully qualified host name, with no trailing dot
@@ -193,6 +193,15 @@
    <target>backend</target>
  </cron>

+  <cron>
+    <url><![CDATA[/_dr/task/sendExpiringCertificateNotificationEmail]]></url>
+    <description>
+      This job runs an action that sends emails to partners if their certificates are expiring soon.
+    </description>
+    <schedule>every day 04:30</schedule>
+    <target>backend</target>
+  </cron>
+
  <cron>
    <url><![CDATA[/_dr/cron/fanout?queue=export-snapshot&endpoint=/_dr/task/backupDatastore&runInEmpty]]></url>
    <description>
@@ -150,7 +150,8 @@ public final class SyncGroupMembersAction implements Runnable {

  /**
   * Parses the results from Google Groups for each registrar, setting the dirty flag to false in
-   * Datastore for the calls that succeeded and accumulating the errors for the calls that failed.
+   * the database for the calls that succeeded and accumulating the errors for the calls that
+   * failed.
   */
  private static List<Throwable> getErrorsAndUpdateFlagsForSuccesses(
      ImmutableMap<Registrar, Optional<Throwable>> results) {
@@ -47,7 +47,7 @@ import javax.inject.Inject;
 import org.joda.time.DateTime;

 /**
- * Class for synchronizing all {@link Registrar} Datastore objects to a Google Spreadsheet.
+ * Class for synchronizing all {@link Registrar} objects to a Google Spreadsheet.
 *
 * @see SyncRegistrarsSheetAction
 */
@@ -56,10 +56,10 @@ public class DomainCreateFlowCustomLogic extends BaseFlowCustomLogic {
  /**
   * A hook that runs before new entities are persisted, allowing them to be changed.
   *
-   * <p>It returns the actual entity changes that should be persisted to Datastore. It is important
-   * to be careful when changing the flow behavior for existing entities, because the core logic
-   * across many different flows expects the existence of these entities and many of the fields on
-   * them.
+   * <p>It returns the actual entity changes that should be persisted to the database. It is
+   * important to be careful when changing the flow behavior for existing entities, because the core
+   * logic across many different flows expects the existence of these entities and many of the
+   * fields on them.
   */
  @SuppressWarnings("unused")
  public EntityChanges beforeSave(BeforeSaveParameters parameters) throws EppException {
@@ -54,10 +54,10 @@ public class DomainDeleteFlowCustomLogic extends BaseFlowCustomLogic {
  /**
   * A hook that runs before new entities are persisted, allowing them to be changed.
   *
-   * <p>It returns the actual entity changes that should be persisted to Datastore. It is important
-   * to be careful when changing the flow behavior for existing entities, because the core logic
-   * across many different flows expects the existence of these entities and many of the fields on
-   * them.
+   * <p>It returns the actual entity changes that should be persisted to the database. It is
+   * important to be careful when changing the flow behavior for existing entities, because the core
+   * logic across many different flows expects the existence of these entities and many of the
+   * fields on them.
   */
  @SuppressWarnings("unused")
  public EntityChanges beforeSave(BeforeSaveParameters parameters) throws EppException {
@@ -55,10 +55,10 @@ public class DomainRenewFlowCustomLogic extends BaseFlowCustomLogic {
  /**
   * A hook that runs before new entities are persisted, allowing them to be changed.
   *
-   * <p>It returns the actual entity changes that should be persisted to Datastore. It is important
-   * to be careful when changing the flow behavior for existing entities, because the core logic
-   * across many different flows expects the existence of these entities and many of the fields on
-   * them.
+   * <p>It returns the actual entity changes that should be persisted to the database. It is
+   * important to be careful when changing the flow behavior for existing entities, because the core
+   * logic across many different flows expects the existence of these entities and many of the
+   * fields on them.
   */
  @SuppressWarnings("unused")
  public EntityChanges beforeSave(BeforeSaveParameters parameters) throws EppException {
@@ -51,10 +51,10 @@ public class DomainUpdateFlowCustomLogic extends BaseFlowCustomLogic {
  /**
   * A hook that runs before new entities are persisted, allowing them to be changed.
   *
-   * <p>It returns the actual entity changes that should be persisted to Datastore. It is important
-   * to be careful when changing the flow behavior for existing entities, because the core logic
-   * across many different flows expects the existence of these entities and many of the fields on
-   * them.
+   * <p>It returns the actual entity changes that should be persisted to the database. It is
+   * important to be careful when changing the flow behavior for existing entities, because the core
+   * logic across many different flows expects the existence of these entities and many of the
+   * fields on them.
   */
  @SuppressWarnings("unused")
  public EntityChanges beforeSave(BeforeSaveParameters parameters) throws EppException {
@@ -19,7 +19,7 @@ import com.google.common.collect.ImmutableSet;
 import google.registry.model.ImmutableObject;
 import google.registry.persistence.VKey;

-/** A wrapper class that encapsulates Datastore entities to both save and delete. */
+/** A wrapper class that encapsulates database entities to both save and delete. */
@AutoValue
 public abstract class EntityChanges {

@@ -149,7 +149,7 @@ public class AllocationTokenFlowUtils {
  private AllocationToken loadToken(String token) throws EppException {
    if (Strings.isNullOrEmpty(token)) {
      // We load the token directly from the input XML. If it's null or empty we should throw
-      // an InvalidAllocationTokenException before the Datastore load attempt fails.
+      // an InvalidAllocationTokenException before the database load attempt fails.
      // See https://tools.ietf.org/html/draft-ietf-regext-allocation-token-04#section-2.1
      throw new InvalidAllocationTokenException();
    }
@@ -39,11 +39,11 @@ import org.joda.time.DateTime;
 /**
 * An EPP flow for requesting {@link PollMessage}s.
 *
- * <p>This flow uses an eventually consistent Datastore query to return the oldest poll message for
- * the registrar, as well as the total number of pending messages. Note that poll messages whose
- * event time is in the future (i.e. they are speculative and could still be changed or rescinded)
- * are ignored. The externally visible id for the poll message that the registrar sees is generated
- * by {@link PollMessageExternalKeyConverter}.
+ * <p>This flow uses an eventually consistent query to return the oldest poll message for the
+ * registrar, as well as the total number of pending messages. Note that poll messages whose event
+ * time is in the future (i.e. they are speculative and could still be changed or rescinded) are
+ * ignored. The externally visible id for the poll message that the registrar sees is generated by
+ * {@link PollMessageExternalKeyConverter}.
 *
 * @error {@link PollRequestFlow.UnexpectedMessageIdException}
 */
@@ -65,7 +65,8 @@ import org.joda.time.DateTime;
      @Index(columnList = "domainName"),
      @Index(columnList = "techContact"),
      @Index(columnList = "tld"),
-      @Index(columnList = "registrantContact")
+      @Index(columnList = "registrantContact"),
+      @Index(columnList = "dnsRefreshRequestTime")
    })
@WithStringVKey
@ExternalMessagingName("domain")
@@ -213,7 +214,8 @@ public class DomainBase extends DomainContent
          .setSmdId(domainContent.getSmdId())
          .setSubordinateHosts(domainContent.getSubordinateHosts())
          .setStatusValues(domainContent.getStatusValues())
-          .setTransferData(domainContent.getTransferData());
+          .setTransferData(domainContent.getTransferData())
+          .setDnsRefreshRequestTime(domainContent.getDnsRefreshRequestTime());
    }
  }
 }
@@ -50,6 +50,7 @@ import com.googlecode.objectify.annotation.IgnoreSave;
 import com.googlecode.objectify.annotation.Index;
 import com.googlecode.objectify.annotation.OnLoad;
 import com.googlecode.objectify.condition.IfNull;
+import google.registry.dns.RefreshDnsAction;
 import google.registry.flows.ResourceFlowUtils;
 import google.registry.model.EppResource;
 import google.registry.model.EppResource.ResourceWithTransferData;
@@ -303,6 +304,35 @@ public class DomainContent extends EppResource
   */
  @Index DateTime autorenewEndTime;

+  /**
+   * When this domain's DNS was requested to be refreshed, or null if its DNS is up-to-date.
+   *
+   * <p>This will almost always be null except in the couple minutes' interval between when a
+   * DNS-affecting create or update operation takes place and when the {@link RefreshDnsAction}
+   * runs, which resets this back to null upon completion of the DNS refresh task. This is a {@link
+   * DateTime} rather than a simple dirty boolean so that the DNS refresh action can order by the
+   * DNS refresh request time and take action on the oldest ones first.
+   *
+   * <p>Note that this is a Cloud SQL-based replacement for the {@code dns-pull} task queue. The
+   * domains that have a non-null value for this field should be exactly the same as the tasks that
+   * would be in the {@code dns-pull} queue. Because this is Cloud SQL-specific, it is omitted from
+   * Datastore.
+   *
+   * <p>Note that in the {@link DomainHistory} table this value means something slightly different:
+   * It means that the given domain action requested a DNS update. Unlike on the {@code Domain}
+   * table, this value is not then subsequently nulled out once the DNS refresh is complete; rather,
+   * it remains as a permanent record of which actions were DNS-affecting and which were not.
+   */
+  // TODO(mcilwain): Start using this field once we are further along in the DB migration.
+  @Ignore DateTime dnsRefreshRequestTime;
+
+  /**
+   * Returns the DNS refresh request time iff this domain's DNS needs refreshing, otherwise absent.
+   */
+  public Optional<DateTime> getDnsRefreshRequestTime() {
+    return Optional.ofNullable(dnsRefreshRequestTime);
+  }
+
  @OnLoad
  void load() {
    // Reconstitute all of the contacts so that they have VKeys.
@@ -967,6 +997,11 @@ public class DomainContent extends EppResource
      return thisCastToDerived();
    }

+    public B setDnsRefreshRequestTime(Optional<DateTime> dnsRefreshRequestTime) {
+      getInstance().dnsRefreshRequestTime = dnsRefreshRequestTime.orElse(null);
+      return thisCastToDerived();
+    }
+
    public B setSmdId(String smdId) {
      getInstance().smdId = smdId;
      return thisCastToDerived();
@@ -92,11 +92,6 @@ public class VKeyTranslatorFactory extends AbstractSimpleTranslatorFactory<VKey,
    }
  }

-  /** Create a VKey from a URL-safe string representation. */
-  public static VKey<?> createVKey(String urlSafe) {
-    return createVKey(com.googlecode.objectify.Key.create(urlSafe));
-  }
-
  @VisibleForTesting
  public static void addTestEntityClass(Class<?> clazz) {
    CLASS_REGISTRY.put(com.googlecode.objectify.Key.getKind(clazz), clazz);
@@ -21,6 +21,7 @@ import java.util.Date;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Stream;
 import javax.persistence.FlushModeType;
 import javax.persistence.LockModeType;
 import javax.persistence.Parameter;
@@ -41,6 +42,11 @@ class ReadOnlyCheckingTypedQuery<T> implements TypedQuery<T> {
    return delegate.getResultList();
  }

+  @Override
+  public Stream<T> getResultStream() {
+    return delegate.getResultStream();
+  }
+
  @Override
  public T getSingleResult() {
    return delegate.getSingleResult();
@@ -339,7 +339,7 @@ abstract class AbstractJsonableObject implements Jsonable {
      return new JsonPrimitive((Boolean) object);
    }
    if (object instanceof DateTime) {
-      // According to RFC7483 section 3, the syntax of dates and times is defined in RFC3339.
+      // According to RFC 9083 section 3, the syntax of dates and times is defined in RFC3339.
      //
      // According to RFC3339, we should use ISO8601, which is what DateTime.toString does!
      return new JsonPrimitive(((DateTime) object).toString());
@@ -50,8 +50,8 @@ import org.joda.time.DateTime;
 /**
 * Base RDAP (new WHOIS) action for all requests.
 *
- * @see <a href="https://tools.ietf.org/html/rfc7482">
- *        RFC 7482: Registration Data Access Protocol (RDAP) Query Format</a>
+ * @see <a href="https://tools.ietf.org/html/rfc9082">RFC 9082: Registration Data Access Protocol
+ *     (RDAP) Query Format</a>
 */
 public abstract class RdapActionBase implements Runnable {

@@ -22,16 +22,12 @@ import google.registry.rdap.AbstractJsonableObject.RestrictJsonNames;
 import java.util.Optional;
 import org.joda.time.DateTime;

-/**
- * Data Structures defined in RFC7483 section 4.
- */
+/** Data Structures defined in RFC 9083 section 4. */
 final class RdapDataStructures {

  private RdapDataStructures() {}

-  /**
-   * RDAP conformance defined in 4.1 of RFC7483.
-   */
+  /** RDAP conformance defined in 4.1 of RFC 9083. */
  @RestrictJsonNames("rdapConformance")
  static final class RdapConformance implements Jsonable {

@@ -42,7 +38,7 @@ final class RdapDataStructures {
    @Override
    public JsonArray toJson() {
      JsonArray jsonArray = new JsonArray();
-      // Conformance to RFC7483
+      // Conformance to RFC 9083
      jsonArray.add("rdap_level_0");

      // Conformance to the RDAP Response Profile V2.1
@@ -57,9 +53,7 @@ final class RdapDataStructures {
    }
  }

-  /**
-   * Links defined in 4.2 of RFC7483.
-   */
+  /** Links defined in 4.2 of RFC 9083. */
  @RestrictJsonNames("links[]")
  @AutoValue
  abstract static class Link extends AbstractJsonableObject {
@@ -91,7 +85,7 @@ final class RdapDataStructures {
  }

  /**
-   * Notices and Remarks defined in 4.3 of RFC7483.
+   * Notices and Remarks defined in 4.3 of RFC 9083.
   *
   * <p>Each has an optional "type" denoting a registered type string defined in 10.2.1. The type is
   * defined as common to both Notices and Remarks, but each item is only appropriate to one of
@@ -118,7 +112,7 @@ final class RdapDataStructures {
  }

  /**
-   * Notices defined in 4.3 of RFC7483.
+   * Notices defined in 4.3 of RFC 9083.
   *
   * <p>A notice denotes information about the service itself or the entire response, and hence will
   * only be in the top-most object.
@@ -128,7 +122,7 @@ final class RdapDataStructures {
  abstract static class Notice extends NoticeOrRemark {

    /**
-     * Notice and Remark Type are defined in 10.2.1 of RFC7483.
+     * Notice and Remark Type are defined in 10.2.1 of RFC 9083.
     *
     * <p>We only keep the "service or entire response" values for Notice.Type.
     */
@@ -138,16 +132,15 @@ final class RdapDataStructures {
      RESULT_TRUNCATED_LOAD("result set truncated due to excessive load"),
      RESULT_TRUNCATED_UNEXPLAINABLE("result set truncated due to unexplainable reasons");

+      private final String rfc9083String;

-      private final String rfc7483String;
-
-      Type(String rfc7483String) {
-        this.rfc7483String = rfc7483String;
+      Type(String rfc9083String) {
+        this.rfc9083String = rfc9083String;
      }

      @Override
      public JsonPrimitive toJson() {
-        return new JsonPrimitive(rfc7483String);
+        return new JsonPrimitive(rfc9083String);
      }
    }

@@ -167,7 +160,7 @@ final class RdapDataStructures {
  }

  /**
-   * Remarks defined in 4.3 of RFC7483.
+   * Remarks defined in 4.3 of RFC 9083.
   *
   * <p>A remark denotes information about the specific object, and hence each object has its own
   * "remarks" array.
@@ -177,7 +170,7 @@ final class RdapDataStructures {
  abstract static class Remark extends NoticeOrRemark {

    /**
-     * Notice and Remark Type are defined in 10.2.1 of RFC7483.
+     * Notice and Remark Type are defined in 10.2.1 of RFC 9083.
     *
     * <p>We only keep the "specific object" values for Remark.Type.
     */
@@ -190,15 +183,15 @@ final class RdapDataStructures {
      // so I'm adding it here, but we have to ask them about it...
      OBJECT_REDACTED_AUTHORIZATION("object redacted due to authorization");

-      private final String rfc7483String;
+      private final String rfc9083String;

-      Type(String rfc7483String) {
-        this.rfc7483String = rfc7483String;
+      Type(String rfc9083String) {
+        this.rfc9083String = rfc9083String;
      }

      @Override
      public JsonPrimitive toJson() {
-        return new JsonPrimitive(rfc7483String);
+        return new JsonPrimitive(rfc9083String);
      }
    }

@@ -218,9 +211,9 @@ final class RdapDataStructures {
  }

  /**
-   * Language Identifier defined in 4.4 of RFC7483.
+   * Language Identifier defined in 4.4 of RFC 9083.
   *
-   * The allowed values are described in RFC5646.
+   * <p>The allowed values are described in RFC5646.
   */
  @RestrictJsonNames("lang")
  enum LanguageIdentifier implements Jsonable {
@@ -239,7 +232,7 @@ final class RdapDataStructures {
  }

  /**
-   * Events defined in 4.5 of RFC7483.
+   * Events defined in 4.5 of RFC 9083.
   *
   * <p>There's a type of Event that must not have the "eventActor" (see 5.1), so we create 2
   * versions - one with and one without.
@@ -263,7 +256,7 @@ final class RdapDataStructures {
    }
  }

-  /** Status values for events specified in RFC 7483 § 10.2.3. */
+  /** Status values for events specified in RFC 9083 § 10.2.3. */
  enum EventAction implements Jsonable {
    REGISTRATION("registration"),
    REREGISTRATION("reregistration"),
@@ -277,25 +270,24 @@ final class RdapDataStructures {
    LAST_UPDATE_OF_RDAP_DATABASE("last update of RDAP database");

    /** Value as it appears in RDAP messages. */
-    private final String rfc7483String;
+    private final String rfc9083String;

-    EventAction(String rfc7483String) {
-      this.rfc7483String = rfc7483String;
+    EventAction(String rfc9083String) {
+      this.rfc9083String = rfc9083String;
    }

    String getDisplayName() {
-      return rfc7483String;
+      return rfc9083String;
    }

    @Override
    public JsonPrimitive toJson() {
-      return new JsonPrimitive(rfc7483String);
+      return new JsonPrimitive(rfc9083String);
    }
  }

-
  /**
-   * Events defined in 4.5 of RFC7483.
+   * Events defined in 4.5 of RFC 9083.
   *
   * <p>There's a type of Event that MUST NOT have the "eventActor" (see 5.1), so we have this
   * object to enforce that.
@@ -315,9 +307,7 @@ final class RdapDataStructures {
    }
  }

-  /**
-   * Events defined in 4.5 of RFC7483.
-   */
+  /** Events defined in 4.5 of RFC 9083. */
  @RestrictJsonNames("events[]")
  @AutoValue
  abstract static class Event extends EventBase {
@@ -336,7 +326,7 @@ final class RdapDataStructures {
  }

  /**
-   * Status defined in 4.6 of RFC7483.
+   * Status defined in 4.6 of RFC 9083.
   *
   * <p>This indicates the state of the registered object.
   *
@@ -345,7 +335,7 @@ final class RdapDataStructures {
  @RestrictJsonNames("status[]")
  enum RdapStatus implements Jsonable {

-    // Status values specified in RFC 7483 § 10.2.2.
+    // Status values specified in RFC 9083 § 10.2.2.
    VALIDATED("validated"),
    RENEW_PROHIBITED("renew prohibited"),
    UPDATE_PROHIBITED("update prohibited"),
@@ -385,24 +375,24 @@ final class RdapDataStructures {
    TRANSFER_PERIOD("transfer period");

    /** Value as it appears in RDAP messages. */
-    private final String rfc7483String;
+    private final String rfc9083String;

-    RdapStatus(String rfc7483String) {
-      this.rfc7483String = rfc7483String;
+    RdapStatus(String rfc9083String) {
+      this.rfc9083String = rfc9083String;
    }

    String getDisplayName() {
-      return rfc7483String;
+      return rfc9083String;
    }

    @Override
    public JsonPrimitive toJson() {
-      return new JsonPrimitive(rfc7483String);
+      return new JsonPrimitive(rfc9083String);
    }
  }

  /**
-   * Port 43 WHOIS Server defined in 4.7 of RFC7483.
+   * Port 43 WHOIS Server defined in 4.7 of RFC 9083.
   *
   * <p>This contains the fully qualifies host name of IP address of the WHOIS RFC3912 server where
   * the containing object instance may be found.
@@ -423,7 +413,7 @@ final class RdapDataStructures {
  }

  /**
-   * Public IDs defined in 4.8 of RFC7483.
+   * Public IDs defined in 4.8 of RFC 9083.
   *
   * <p>Maps a public identifier to an object class.
   */
@@ -434,15 +424,15 @@ final class RdapDataStructures {
    enum Type implements Jsonable {
      IANA_REGISTRAR_ID("IANA Registrar ID");

-      private final String rfc7483String;
+      private final String rfc9083String;

-      Type(String rfc7483String) {
-        this.rfc7483String = rfc7483String;
+      Type(String rfc9083String) {
+        this.rfc9083String = rfc9083String;
      }

      @Override
      public JsonPrimitive toJson() {
-        return new JsonPrimitive(rfc7483String);
+        return new JsonPrimitive(rfc9083String);
      }
    }

@@ -457,7 +447,7 @@ final class RdapDataStructures {
  }

  /**
-   * Object Class Name defined in 4.7 of RFC7483.
+   * Object Class Name defined in 4.7 of RFC 9083.
   *
   * <p>Identifies the type of the object being processed. Is REQUIRED in all RDAP response objects,
   * but not so for internal objects whose type can be inferred by their key name in the enclosing
@@ -465,15 +455,15 @@ final class RdapDataStructures {
   */
  @RestrictJsonNames("objectClassName")
  enum ObjectClassName implements Jsonable {
-    /** Defined in 5.1 of RFC7483. */
+    /** Defined in 5.1 of RFC 9083. */
    ENTITY("entity"),
-    /** Defined in 5.2 of RFC7483. */
+    /** Defined in 5.2 of RFC 9083. */
    NAMESERVER("nameserver"),
-    /** Defined in 5.3 of RFC7483. */
+    /** Defined in 5.3 of RFC 9083. */
    DOMAIN("domain"),
-    /** Defined in 5.4 of RFC7483. Only relevant for Registrars, so isn't implemented here. */
+    /** Defined in 5.4 of RFC 9083. Only relevant for Registrars, so isn't implemented here. */
    IP_NETWORK("ip network"),
-    /** Defined in 5.5 of RFC7483. Only relevant for Registrars, so isn't implemented here. */
+    /** Defined in 5.5 of RFC 9083. Only relevant for Registrars, so isn't implemented here. */
    AUTONOMUS_SYSTEM("autnum");

    private final String className;
@@ -68,9 +68,9 @@ import org.hibernate.Hibernate;
 *
 * <p>All commands and responses conform to the RDAP spec as defined in RFCs 7480 through 7485.
 *
- * @see <a href="http://tools.ietf.org/html/rfc7482">RFC 7482: Registration Data Access Protocol
+ * @see <a href="http://tools.ietf.org/html/rfc9082">RFC 9082: Registration Data Access Protocol
 *     (RDAP) Query Format</a>
- * @see <a href="http://tools.ietf.org/html/rfc7483">RFC 7483: JSON Responses for the Registration
+ * @see <a href="http://tools.ietf.org/html/rfc9083">RFC 9083: JSON Responses for the Registration
 *     Data Access Protocol (RDAP)</a>
 */
 // TODO: This isn't required by the RDAP Technical Implementation Guide, and hence should be
@@ -119,7 +119,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
    } else if (nsLdhNameParam.isPresent()) {
      metricInformationBuilder.setSearchType(SearchType.BY_NAMESERVER_NAME);
      // syntax: /rdap/domains?nsLdhName=ns1.exam*.com
-      // RFC 7482 appears to say that Unicode domains must be specified using punycode when
+      // RFC 9082 appears to say that Unicode domains must be specified using punycode when
      // passed to nsLdhName, so IDN.toASCII is not called here.
      results =
          searchByNameserverLdhName(
@@ -41,7 +41,7 @@ import javax.inject.Inject;
 * profile dictates that the "handle" for registrars is to be the IANA registrar ID:
 *
 * <p>2.8.3. Registries MUST support lookup for entities with the registrar role within other
- * objects using the handle (as described in 3.1.5 of RFC7482). The handle of the entity with the
+ * objects using the handle (as described in 3.1.5 of RFC 9082). The handle of the entity with the
 * registrar role MUST be equal to IANA Registrar ID. The entity with the registrar role in the RDAP
 * response MUST contain a publicIDs member to identify the IANA Registrar ID from the IANA’s
 * Registrar ID registry. The type value of the publicID object MUST be equal to IANA Registrar ID.
@@ -73,9 +73,9 @@ import javax.inject.Inject;
 * that we can skip the contact search altogether (because we returned a registrar, and all
 * registrars come after all contacts).
 *
- * @see <a href="http://tools.ietf.org/html/rfc7482">RFC 7482: Registration Data Access Protocol
+ * @see <a href="http://tools.ietf.org/html/rfc9082">RFC 9082: Registration Data Access Protocol
 *     (RDAP) Query Format</a>
- * @see <a href="http://tools.ietf.org/html/rfc7483">RFC 7483: JSON Responses for the Registration
+ * @see <a href="http://tools.ietf.org/html/rfc9083">RFC 9083: JSON Responses for the Registration
 *     Data Access Protocol (RDAP)</a>
 */
 // TODO: This isn't required by the RDAP Technical Implementation Guide, and hence should be
@@ -195,7 +195,7 @@ public class RdapEntitySearchAction extends RdapSearchActionBase {
   * <p>The search is by registrar name only. The profile is supporting the functionality defined in
   * the Base Registry Agreement.
   *
-   * <p>According to RFC 7482 section 6.1, punycode is only used for domain name labels, so we can
+   * <p>According to RFC 9082 section 6.1, punycode is only used for domain name labels, so we can
   * assume that entity names are regular unicode.
   *
   * <p>The includeDeleted flag is ignored when searching for contacts, because contact names are
@@ -91,8 +91,8 @@ import org.joda.time.DateTime;
 * of the methods, is used as the first part of the link URL. For instance, if linkBase is
 * "http://rdap.org/dir/", the link URLs will look like "http://rdap.org/dir/domain/XXXX", etc.
 *
- * @see <a href="https://tools.ietf.org/html/rfc7483">
- *        RFC 7483: JSON Responses for the Registration Data Access Protocol (RDAP)</a>
+ * @see <a href="https://tools.ietf.org/html/rfc9083">RFC 9083: JSON Responses for the Registration
+ *     Data Access Protocol (RDAP)</a>
 */
 public class RdapJsonFormatter {

@@ -253,9 +253,9 @@ public class RdapJsonFormatter {
  /**
   * Creates a JSON object for a {@link DomainBase}.
   *
-   * <p>NOTE that domain searches aren't in the spec yet - they're in the RFC7482 that describes the
-   * query format, but they aren't in the RDAP Technical Implementation Guide 15feb19, meaning we
-   * don't have to implement them yet and the RDAP Response Profile doesn't apply to them.
+   * <p>NOTE that domain searches aren't in the spec yet - they're in the RFC 9082 that describes
+   * the query format, but they aren't in the RDAP Technical Implementation Guide 15feb19, meaning
+   * we don't have to implement them yet and the RDAP Response Profile doesn't apply to them.
   *
   * <p>We're implementing domain searches anyway, BUT we won't have the response for searches
   * conform to the RDAP Response Profile.
@@ -811,7 +811,7 @@ public class RdapJsonFormatter {
    return Optional.of(builder.build());
  }

-  /** Converts a domain registry contact type into a role as defined by RFC 7483. */
+  /** Converts a domain registry contact type into a role as defined by RFC 9083. */
  private static RdapEntity.Role convertContactTypeToRdapRole(DesignatedContact.Type contactType) {
    switch (contactType) {
      case REGISTRANT:
@@ -928,7 +928,7 @@ public class RdapJsonFormatter {
    return eventsBuilder.build();
  }

-  /** Creates an RDAP event object as defined by RFC 7483. */
+  /** Creates an RDAP event object as defined by RFC 9083. */
  private static Event makeEvent(
      EventAction eventAction, @Nullable String eventActor, DateTime eventDate) {
    Event.Builder builder = Event.builder()
@@ -1069,7 +1069,7 @@ public class RdapJsonFormatter {
  /**
   * Creates a self link as directed by the spec.
   *
-   * @see <a href="https://tools.ietf.org/html/rfc7483">RFC 7483: JSON Responses for the
+   * @see <a href="https://tools.ietf.org/html/rfc9083">RFC 9083: JSON Responses for the
   *     Registration Data Access Protocol (RDAP)</a>
   */
  private Link makeSelfLink(String type, String name) {
@@ -52,9 +52,9 @@ import javax.inject.Inject;
 *
 * <p>All commands and responses conform to the RDAP spec as defined in RFCs 7480 through 7485.
 *
- * @see <a href="http://tools.ietf.org/html/rfc7482">RFC 7482: Registration Data Access Protocol
+ * @see <a href="http://tools.ietf.org/html/rfc9082">RFC 9082: Registration Data Access Protocol
 *     (RDAP) Query Format</a>
- * @see <a href="http://tools.ietf.org/html/rfc7483">RFC 7483: JSON Responses for the Registration
+ * @see <a href="http://tools.ietf.org/html/rfc9083">RFC 9083: JSON Responses for the Registration
 *     Data Access Protocol (RDAP)</a>
 */
@Action(
@@ -91,7 +91,7 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase {
    NameserverSearchResponse results;
    if (nameParam.isPresent()) {
      // RDAP Technical Implementation Guilde 2.2.3 - we MAY support nameserver search queries based
-      // on a "nameserver search pattern" as defined in RFC7482
+      // on a "nameserver search pattern" as defined in RFC 9082
      //
      // syntax: /rdap/nameservers?name=exam*.com
      metricInformationBuilder.setSearchType(SearchType.BY_NAMESERVER_NAME);
@@ -101,7 +101,7 @@ public class RdapNameserverSearchAction extends RdapSearchActionBase {
                  RdapSearchPattern.createFromLdhOrUnicodeDomainName(nameParam.get())));
    } else {
      // RDAP Technical Implementation Guide 2.2.3 - we MUST support nameserver search queries based
-      // on IP address as defined in RFC7482 3.2.2. Doesn't require pattern matching
+      // on IP address as defined in RFC 9082 3.2.2. Doesn't require pattern matching
      //
      // syntax: /rdap/nameservers?ip=1.2.3.4
      metricInformationBuilder.setSearchType(SearchType.BY_NAMESERVER_ADDRESS);
@@ -41,9 +41,7 @@ import google.registry.rdap.RdapDataStructures.Remark;
 import google.registry.util.Idn;
 import java.util.Optional;

-/**
- * Object Classes defined in RFC7483 section 5.
- */
+/** Object Classes defined in RFC 9083 section 5. */
 final class RdapObjectClasses {

  /**
@@ -161,7 +159,7 @@ final class RdapObjectClasses {
  /**
   * The Top Level JSON reply, Adds the required top-level boilerplate to a ReplyPayloadBase.
   *
-   * <p>RFC 7483 specifies that the top-level object should include an entry indicating the
+   * <p>RFC 9083 specifies that the top-level object should include an entry indicating the
   * conformance level. ICANN RDAP spec for 15feb19 mandates several additional entries, in sections
   * 2.6.3, 2.11 of the Response Profile and 3.3, 3.5, of the Technical Implementation Guide.
   */
@@ -250,7 +248,7 @@ final class RdapObjectClasses {
  }

  /**
-   * The Entity Object Class defined in 5.1 of RFC7483.
+   * The Entity Object Class defined in 5.1 of RFC 9083.
   *
   * <p>Entities are used both for Contacts and for Registrars. We will create different subobjects
   * for each one for type safety.
@@ -260,7 +258,7 @@ final class RdapObjectClasses {
  @RestrictJsonNames({"entities[]", "entitySearchResults[]"})
  abstract static class RdapEntity extends RdapObjectBase {

-    /** Role values specified in RFC 7483 § 10.2.4. */
+    /** Role values specified in RFC 9083 § 10.2.4. */
    @RestrictJsonNames("roles[]")
    enum Role implements Jsonable {
      REGISTRANT("registrant"),
@@ -276,15 +274,15 @@ final class RdapObjectClasses {
      NOC("noc");

      /** Value as it appears in RDAP messages. */
-      final String rfc7483String;
+      final String rfc9083String;

-      Role(String rfc7483String) {
-        this.rfc7483String = rfc7483String;
+      Role(String rfc9083String) {
+        this.rfc9083String = rfc9083String;
      }

      @Override
      public JsonPrimitive toJson() {
-        return new JsonPrimitive(rfc7483String);
+        return new JsonPrimitive(rfc9083String);
      }
    }

@@ -306,7 +304,7 @@ final class RdapObjectClasses {
  }

  /**
-   * Registrar version of the Entity Object Class defined in 5.1 of RFC7483.
+   * Registrar version of the Entity Object Class defined in 5.1 of RFC 9083.
   *
   * <p>Entities are used both for Contacts and for Registrars. We will create different subobjects
   * for each one for type safety.
@@ -325,7 +323,7 @@ final class RdapObjectClasses {
  }

  /**
-   * Contact version of the Entity Object Class defined in 5.1 of RFC7483.
+   * Contact version of the Entity Object Class defined in 5.1 of RFC 9083.
   *
   * <p>Entities are used both for Contacts and for Registrars. We will create different subobjects
   * for each one for type safety.
@@ -385,9 +383,7 @@ final class RdapObjectClasses {
    }
  }

-  /**
-   * The Nameserver Object Class defined in 5.2 of RFC7483.
-   */
+  /** The Nameserver Object Class defined in 5.2 of RFC 9083. */
  @RestrictJsonNames({"nameservers[]", "nameserverSearchResults[]"})
  @AutoValue
  abstract static class RdapNameserver extends RdapNamedObjectBase {
@@ -429,7 +425,7 @@ final class RdapObjectClasses {
    }
  }

-  /** Object defined in RFC7483 section 5.3, only used for RdapDomain. */
+  /** Object defined in RFC 9083 section 5.3, only used for RdapDomain. */
  @RestrictJsonNames("secureDNS")
  @AutoValue
  abstract static class SecureDns extends AbstractJsonableObject {
@@ -471,7 +467,7 @@ final class RdapObjectClasses {
     * an integer representing the signature lifetime in seconds to be used when creating the RRSIG
     * DS record in the parent zone [RFC5910].
     *
-     * <p>Note that although it isn't given as optional in RFC7483, in RFC5910 it's mentioned as
+     * <p>Note that although it isn't given as optional in RFC 9083, in RFC5910 it's mentioned as
     * optional. Also, our code doesn't support it at all - so it's set to always be empty.
     */
    @JsonableElement
@@ -504,9 +500,9 @@ final class RdapObjectClasses {
  }

  /**
-   * The Domain Object Class defined in 5.3 of RFC7483.
+   * The Domain Object Class defined in 5.3 of RFC 9083.
   *
-   * We're missing the "variants", "secureDNS", "network" fields
+   * <p>We're missing the "variants", "secureDNS", "network" fields
   */
  @RestrictJsonNames("domainSearchResults[]")
  @AutoValue
@@ -535,9 +531,7 @@ final class RdapObjectClasses {
    }
  }

-  /**
-   * Error Response Body defined in 6 of RFC7483.
-   */
+  /** Error Response Body defined in 6 of RFC 9083. */
  @RestrictJsonNames({})
  @AutoValue
  abstract static class ErrorResponse extends ReplyPayloadBase {
@@ -559,7 +553,7 @@ final class RdapObjectClasses {
  }

  /**
-   * Help Response defined in 7 of RFC7483.
+   * Help Response defined in 7 of RFC 9083.
   *
   * <p>The helpNotice field is optional, because if the user requests the TOS - that's already
   * given by the boilerplate of TopLevelReplyObject so we don't want to give it again.
@@ -50,8 +50,8 @@ import javax.persistence.criteria.CriteriaBuilder;
 /**
 * Base RDAP (new WHOIS) action for domain, nameserver and entity search requests.
 *
- * @see <a href="https://tools.ietf.org/html/rfc7482">
- *        RFC 7482: Registration Data Access Protocol (RDAP) Query Format</a>
+ * @see <a href="https://tools.ietf.org/html/rfc9082">RFC 9082: Registration Data Access Protocol
+ *     (RDAP) Query Format</a>
 */
 public abstract class RdapSearchActionBase extends RdapActionBase {

@@ -29,8 +29,8 @@ import javax.annotation.Nullable;
 * must be at the end, except for a possible suffix string on the end to restrict the search to a
 * particular TLD (for domains) or domain (for nameservers).
 *
- * @see <a href="http://www.ietf.org/rfc/rfc7482.txt">
- *        RFC 7482: Registration Data Access Protocol (RDAP) Query Format</a>
+ * @see <a href="http://www.ietf.org/rfc/rfc9082.txt">RFC 9082: Registration Data Access Protocol
+ *     (RDAP) Query Format</a>
 */
 public final class RdapSearchPattern {

@@ -39,7 +39,7 @@ public final class RdapSearchPattern {
  /**
   * Pattern for allowed LDH searches.
   *
-   * <p>Based on RFC7482 4.1. Must contains only alphanumeric plus dots and hyphens. A single
+   * <p>Based on RFC 9082 4.1. Must contains only alphanumeric plus dots and hyphens. A single
   * whildcard asterix is allowed - but if exists must be the last character of a domain name label
   * (so exam* and exam*.com are allowed, but exam*le.com isn't allowd)
   *
@@ -57,11 +57,10 @@ public final class RdapSearchPattern {

  /**
   * Terminating suffix after the wildcard, or null if none was specified; for domains, it should be
-   * a TLD, for nameservers, a domain. RFC 7482 requires only that it be a sequence of domain
+   * a TLD, for nameservers, a domain. RFC 9082 requires only that it be a sequence of domain
   * labels, but this definition is stricter for efficiency purposes.
   */
-  @Nullable
-  private final String suffix;
+  @Nullable private final String suffix;

  private RdapSearchPattern(
      final String initialString, final boolean hasWildcard, @Nullable final String suffix) {
@@ -153,7 +152,7 @@ public final class RdapSearchPattern {
   * <p>The domain search pattern can have a single wildcard asterix that can match 0 or more
   * charecters. If such an asterix exists - it must be at the end of a domain label.
   *
-   * <p>In theory, according to RFC7482 4.1 - we should make some checks about partial matching in
+   * <p>In theory, according to RFC 9082 4.1 - we should make some checks about partial matching in
   * unicode queries. We don't, but we might want to just disable partial matches for unicode inputs
   * (meaning if it doesn't match LDH_PATTERN, then don't allow wildcard at all).
   *
@@ -39,9 +39,7 @@ import java.util.Optional;
@AutoValue
 abstract class RdapSearchResults {

-  /**
-   * Responding To Searches defined in 8 of RFC7483.
-   */
+  /** Responding To Searches defined in 8 of RFC 9083. */
  abstract static class BaseSearchResponse extends ReplyPayloadBase {
    abstract IncompletenessWarningType incompletenessWarningType();
    abstract ImmutableMap<String, URI> navigationLinks();
@@ -26,7 +26,6 @@ import com.beust.jcommander.Parameters;
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Streams;
 import com.google.common.flogger.FluentLogger;
 import google.registry.config.RegistryConfig.Config;
 import google.registry.model.domain.DomainBase;
@@ -134,10 +133,10 @@ public class BackfillRegistryLocksCommand extends ConfirmingCommand
  private DateTime getLockCompletionTimestamp(DomainBase domainBase, DateTime now) {
    // Best-effort, if a domain was URS-locked we should use that time
    // If we can't find that, return now.
-    return Streams.stream(HistoryEntryDao.loadHistoryObjectsForResource(domainBase.createVKey()))
+    return HistoryEntryDao.loadHistoryObjectsForResource(domainBase.createVKey()).stream()
        // sort by modification time descending so we get the most recent one if it was locked twice
        .sorted(Comparator.comparing(HistoryEntry::getModificationTime).reversed())
-        .filter(entry -> entry.getReason().equals("Uniform Rapid Suspension"))
+        .filter(entry -> "Uniform Rapid Suspension".equals(entry.getReason()))
        .findFirst()
        .map(HistoryEntry::getModificationTime)
        .orElse(now);
@@ -306,6 +306,7 @@ class InitSqlPipelineTest {
              .build());
      exportDir = store.export(exportRootDir.getAbsolutePath(), ALL_KINDS, ImmutableSet.of());
      commitLogDir = Files.createDirectory(tmpDir.resolve("commits")).toFile();
+      fakeClock.advanceOneMilli();
    }
  }

@@ -362,6 +363,7 @@ class InitSqlPipelineTest {
        .isEqualTo(expected.getAutorenewPollMessage().getOfyKey());
    assertThat(actual.getDeletePollMessage().getOfyKey())
        .isEqualTo(expected.getDeletePollMessage().getOfyKey());
+    assertThat(actual.getUpdateTimestamp()).isEqualTo(expected.getUpdateTimestamp());
    // TODO(weiminyu): check gracePeriods and transferData when it is easier to do
  }
 }
@@ -80,7 +80,7 @@ class SafeBrowsingTransformsTest {
  private static final String REGISTRAR_ID = "registrarID";
  private static final String REGISTRAR_EMAIL = "email@registrar.net";

-  private static ImmutableMap<Subdomain, ThreatMatch> THREAT_MATCH_MAP;
+  private static ImmutableMap<DomainNameInfo, ThreatMatch> THREAT_MATCH_MAP;

  private final CloseableHttpClient mockHttpClient =
      mock(CloseableHttpClient.class, withSettings().serializable());
@@ -95,24 +95,25 @@ class SafeBrowsingTransformsTest {
  final TestPipelineExtension pipeline =
      TestPipelineExtension.create().enableAbandonedNodeEnforcement(true);

-  private static Subdomain createSubdomain(String url) {
-    return Subdomain.create(url, REPO_ID, REGISTRAR_ID, REGISTRAR_EMAIL);
+  private static DomainNameInfo createDomainNameInfo(String url) {
+    return DomainNameInfo.create(url, REPO_ID, REGISTRAR_ID, REGISTRAR_EMAIL);
  }

-  private KV<Subdomain, ThreatMatch> getKv(String url) {
-    Subdomain subdomain = createSubdomain(url);
-    return KV.of(subdomain, THREAT_MATCH_MAP.get(subdomain));
+  private KV<DomainNameInfo, ThreatMatch> getKv(String url) {
+    DomainNameInfo domainNameInfo = createDomainNameInfo(url);
+    return KV.of(domainNameInfo, THREAT_MATCH_MAP.get(domainNameInfo));
  }

  @BeforeAll
  static void beforeAll() {
-    ImmutableMap.Builder<Subdomain, ThreatMatch> builder = new ImmutableMap.Builder<>();
+    ImmutableMap.Builder<DomainNameInfo, ThreatMatch> builder = new ImmutableMap.Builder<>();
    THREAT_MAP
        .entrySet()
        .forEach(
            kv ->
                builder.put(
-                    createSubdomain(kv.getKey()), ThreatMatch.create(kv.getValue(), kv.getKey())));
+                    createDomainNameInfo(kv.getKey()),
+                    ThreatMatch.create(kv.getValue(), kv.getKey())));
    THREAT_MATCH_MAP = builder.build();
  }

@@ -123,16 +124,16 @@ class SafeBrowsingTransformsTest {

  @Test
  void testSuccess_someBadDomains() throws Exception {
-    ImmutableList<Subdomain> subdomains =
+    ImmutableList<DomainNameInfo> domainNameInfos =
        ImmutableList.of(
-            createSubdomain("111.com"),
-            createSubdomain("hooli.com"),
-            createSubdomain("party-night.net"),
-            createSubdomain("anti-anti-anti-virus.dev"),
-            createSubdomain("no-email.com"));
-    PCollection<KV<Subdomain, ThreatMatch>> threats =
+            createDomainNameInfo("111.com"),
+            createDomainNameInfo("hooli.com"),
+            createDomainNameInfo("party-night.net"),
+            createDomainNameInfo("anti-anti-anti-virus.dev"),
+            createDomainNameInfo("no-email.com"));
+    PCollection<KV<DomainNameInfo, ThreatMatch>> threats =
        pipeline
-            .apply(Create.of(subdomains).withCoder(SerializableCoder.of(Subdomain.class)))
+            .apply(Create.of(domainNameInfos).withCoder(SerializableCoder.of(DomainNameInfo.class)))
            .apply(ParDo.of(safeBrowsingFn));

    PAssert.that(threats)
@@ -146,14 +147,14 @@ class SafeBrowsingTransformsTest {

  @Test
  void testSuccess_noBadDomains() throws Exception {
-    ImmutableList<Subdomain> subdomains =
+    ImmutableList<DomainNameInfo> domainNameInfos =
        ImmutableList.of(
-            createSubdomain("hello_kitty.dev"),
-            createSubdomain("555.com"),
-            createSubdomain("goodboy.net"));
-    PCollection<KV<Subdomain, ThreatMatch>> threats =
+            createDomainNameInfo("hello_kitty.dev"),
+            createDomainNameInfo("555.com"),
+            createDomainNameInfo("goodboy.net"));
+    PCollection<KV<DomainNameInfo, ThreatMatch>> threats =
        pipeline
-            .apply(Create.of(subdomains).withCoder(SerializableCoder.of(Subdomain.class)))
+            .apply(Create.of(domainNameInfos).withCoder(SerializableCoder.of(DomainNameInfo.class)))
            .apply(ParDo.of(safeBrowsingFn));

    PAssert.that(threats).empty();
@@ -94,13 +94,16 @@ class Spec11PipelineTest {
  private final CloseableHttpClient mockHttpClient =
      mock(CloseableHttpClient.class, withSettings().serializable());

-  private static final ImmutableList<Subdomain> SUBDOMAINS =
+  private static final ImmutableList<DomainNameInfo> DOMAIN_NAME_INFOS =
      ImmutableList.of(
-          Subdomain.create("111.com", "123456789-COM", "hello-registrar", "email@hello.net"),
-          Subdomain.create("party-night.net", "2244AABBC-NET", "kitty-registrar", "contact@kit.ty"),
-          Subdomain.create("bitcoin.bank", "1C3D5E7F9-BANK", "hello-registrar", "email@hello.net"),
-          Subdomain.create("no-email.com", "2A4BA9BBC-COM", "kitty-registrar", "contact@kit.ty"),
-          Subdomain.create(
+          DomainNameInfo.create("111.com", "123456789-COM", "hello-registrar", "email@hello.net"),
+          DomainNameInfo.create(
+              "party-night.net", "2244AABBC-NET", "kitty-registrar", "contact@kit.ty"),
+          DomainNameInfo.create(
+              "bitcoin.bank", "1C3D5E7F9-BANK", "hello-registrar", "email@hello.net"),
+          DomainNameInfo.create(
+              "no-email.com", "2A4BA9BBC-COM", "kitty-registrar", "contact@kit.ty"),
+          DomainNameInfo.create(
              "anti-anti-anti-virus.dev", "555666888-DEV", "cool-registrar", "cool@aid.net"));

  private static final ImmutableList<ThreatMatch> THREAT_MATCHES =
@@ -129,7 +132,7 @@ class Spec11PipelineTest {
      PipelineOptionsFactory.create().as(Spec11PipelineOptions.class);

  private File reportingBucketUrl;
-  private PCollection<KV<Subdomain, ThreatMatch>> threatMatches;
+  private PCollection<KV<DomainNameInfo, ThreatMatch>> threatMatches;

  ImmutableSet<Spec11ThreatMatch> sqlThreatMatches;

@@ -143,11 +146,11 @@ class Spec11PipelineTest {
    threatMatches =
        pipeline.apply(
            Create.of(
-                    Streams.zip(SUBDOMAINS.stream(), THREAT_MATCHES.stream(), KV::of)
+                    Streams.zip(DOMAIN_NAME_INFOS.stream(), THREAT_MATCHES.stream(), KV::of)
                        .collect(toImmutableList()))
                .withCoder(
                    KvCoder.of(
-                        SerializableCoder.of(Subdomain.class),
+                        SerializableCoder.of(DomainNameInfo.class),
                        SerializableCoder.of(ThreatMatch.class))));

    sqlThreatMatches =
@@ -223,8 +226,8 @@ class Spec11PipelineTest {
  @Test
  void testSuccess_readFromCloudSql() throws Exception {
    setupCloudSql();
-    PCollection<Subdomain> subdomains = Spec11Pipeline.readFromCloudSql(pipeline);
-    PAssert.that(subdomains).containsInAnyOrder(SUBDOMAINS);
+    PCollection<DomainNameInfo> domainNameInfos = Spec11Pipeline.readFromCloudSql(pipeline);
+    PAssert.that(domainNameInfos).containsInAnyOrder(DOMAIN_NAME_INFOS);
    pipeline.run().waitUntilFinish();
  }

@@ -173,6 +173,7 @@ public class DomainBaseTest extends EntityTestCase {
                            "registrar",
                            null))
                    .setAutorenewEndTime(Optional.of(fakeClock.nowUtc().plusYears(2)))
+                    .setDnsRefreshRequestTime(Optional.of(fakeClock.nowUtc()))
                    .build()));
  }

@@ -53,6 +53,8 @@ import google.registry.testing.DatabaseHelper;
 import google.registry.testing.DualDatabaseTest;
 import google.registry.testing.TestOfyOnly;
 import google.registry.testing.TestSqlOnly;
+import java.util.Optional;
+import org.joda.time.DateTime;
 import org.junit.jupiter.api.BeforeEach;

 /** Tests for {@link DomainHistory}. */
@@ -245,6 +247,7 @@ public class DomainHistoryTest extends EntityTestCase {
            .asBuilder()
            .setNameservers(host.createVKey())
            .setDsData(ImmutableSet.of(DelegationSignerData.create(1, 2, 3, new byte[] {0, 1, 2})))
+            .setDnsRefreshRequestTime(Optional.of(DateTime.parse("2020-03-09T16:40:00Z")))
            .build();
    jpaTm().transact(() -> jpaTm().insert(domain));
    return domain;
@@ -88,18 +88,6 @@ public class VKeyTranslatorFactoryTest {
    assertThat(vkey.getSqlKey()).isEqualTo(200L);
  }

-  @Test
-  void testUrlSafeKey() {
-    // Creating an objectify key instead of a datastore key as this should get a correctly formatted
-    // key path.
-    DomainBase domain = newDomainBase("example.com", "ROID-1", persistActiveContact("contact-1"));
-    Key<DomainBase> key = Key.create(domain);
-    VKey<DomainBase> vkey = (VKey<DomainBase>) VKeyTranslatorFactory.createVKey(key.getString());
-    assertThat(vkey.getKind()).isEqualTo(DomainBase.class);
-    assertThat(vkey.getOfyKey()).isEqualTo(key);
-    assertThat(vkey.getSqlKey()).isEqualTo("ROID-1");
-  }
-
  @Test
  void testExtraEntityClass() {
    TestObject testObject = TestObject.create("id", "field");
@@ -15,11 +15,14 @@ package google.registry.persistence;

 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
+import static google.registry.testing.DatabaseHelper.newDomainBase;
+import static google.registry.testing.DatabaseHelper.persistActiveContact;
 import static org.junit.jupiter.api.Assertions.assertThrows;

 import com.googlecode.objectify.Key;
 import com.googlecode.objectify.annotation.Entity;
 import google.registry.model.billing.BillingEvent.OneTime;
+import google.registry.model.domain.DomainBase;
 import google.registry.model.registrar.RegistrarContact;
 import google.registry.testing.AppEngineExtension;
 import google.registry.testing.TestObject;
@@ -114,6 +117,19 @@ class VKeyTest {
        .contains("Missing value for last key of type class google.registry.testing.TestObject");
  }

+  @Test
+  void testFromWebsafeKey() {
+    // Creating an objectify key instead of a datastore key as this should get a correctly formatted
+    // key path.  We have to one of our actual model object classes for this, TestObject can not be
+    // reconstructed by the VKeyTranslatorFactory.
+    DomainBase domain = newDomainBase("example.com", "ROID-1", persistActiveContact("contact-1"));
+    Key<DomainBase> key = Key.create(domain);
+    VKey<DomainBase> vkey = VKey.fromWebsafeKey(key.getString());
+    assertThat(vkey.getKind()).isEqualTo(DomainBase.class);
+    assertThat(vkey.getOfyKey()).isEqualTo(key);
+    assertThat(vkey.getSqlKey()).isEqualTo("ROID-1");
+  }
+
  @Entity
  static class OtherObject {}
 }
@@ -127,7 +127,7 @@ class BackfillRegistryLocksCommandTest extends CommandTestCase<BackfillRegistryL
  void testBackfill_usesUrsTime_ifExists() throws Exception {
    DateTime ursTime = fakeClock.nowUtc();
    DomainBase ursDomain = persistLockedDomain("urs.tld");
-    HistoryEntry historyEntry =
+    persistResource(
        new DomainHistory.Builder()
            .setBySuperuser(true)
            .setClientId("adminreg")
@@ -136,9 +136,17 @@ class BackfillRegistryLocksCommandTest extends CommandTestCase<BackfillRegistryL
            .setReason("Uniform Rapid Suspension")
            .setType(HistoryEntry.Type.DOMAIN_UPDATE)
            .setRequestedByRegistrar(false)
-            .build();
-    persistResource(historyEntry);
+            .build());
    DomainBase nonUrsDomain = persistLockedDomain("nonurs.tld");
+    persistResource(
+        new DomainHistory.Builder()
+            .setBySuperuser(true)
+            .setClientId("adminreg")
+            .setDomain(nonUrsDomain)
+            .setType(HistoryEntry.Type.DOMAIN_UPDATE)
+            .setRequestedByRegistrar(false)
+            .setModificationTime(ursTime)
+            .build());

    fakeClock.advanceBy(Duration.standardDays(10));
    runCommandForced(
@@ -1,4 +1,4 @@
-Map from registrar email / name to detected subdomain threats:
+Map from registrar email / name to detected domain name threats:
 {"threatMatches":[{"threatType":"UNWANTED_SOFTWARE","fullyQualifiedDomainName":"anti-anti-anti-virus.dev"}],"registrarClientId":"cool-registrar","registrarEmailAddress":"cool@aid.net"}
 {"threatMatches":[{"threatType":"MALWARE","fullyQualifiedDomainName":"111.com"},{"threatType":"POTENTIALLY_HARMFUL_APPLICATION","fullyQualifiedDomainName":"bitcoin.bank"}],"registrarClientId":"hello-registrar","registrarEmailAddress":"email@hello.net"}
 {"threatMatches":[{"threatType":"THREAT_TYPE_UNSPECIFIED","fullyQualifiedDomainName":"no-eamil.com"},{"threatType":"SOCIAL_ENGINEERING","fullyQualifiedDomainName":"party-night.net"}],"registrarClientId":"kitty-registrar","registrarEmailAddress":"contact@kit.ty"}
@@ -261,15 +261,11 @@ td.section {
    </tr> 
    <tr> 
     <td class="property_name">generated on</td> 
-<<<<<<< HEAD
-     <td class="property_value">2021-08-10 20:50:09.993881</td> 
-=======
-     <td class="property_value">2021-08-11 23:54:56.914311</td> 
->>>>>>> 475839855 (Add the domain DNS refresh request time field to the DB schema)
+     <td class="property_value">2021-09-14 16:11:21.688911</td> 
    </tr> 
    <tr>
     <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V101__domain_add_dns_refresh_request_time.sql</td>
+     <td id="lastFlywayFile" class="property_value">V102__add_indexes_to_domain_history_sub_tables.sql</td>
    </tr>
   </tbody>
  </table> 
@@ -288,11 +284,7 @@ td.section {
     generated on
    </text> 
    <text text-anchor="start" x="4055.5" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-<<<<<<< HEAD
-     2021-08-10 20:50:09.993881
-=======
-     2021-08-11 23:54:56.914311
->>>>>>> 475839855 (Add the domain DNS refresh request time field to the DB schema)
+     2021-09-14 16:11:21.688911
    </text> 
    <polygon fill="none" stroke="#888888" points="3968,-4 3968,-44 4233,-44 4233,-4 3968,-4" /> <!-- allocationtoken_a08ccbef --> 
    <g id="node1" class="node"> 
@@ -261,15 +261,11 @@ td.section {
    </tr> 
    <tr> 
     <td class="property_name">generated on</td> 
-<<<<<<< HEAD
-     <td class="property_value">2021-08-10 20:50:07.996141</td> 
-=======
-     <td class="property_value">2021-08-11 23:54:54.724849</td> 
->>>>>>> 475839855 (Add the domain DNS refresh request time field to the DB schema)
+     <td class="property_value">2021-09-14 16:11:19.580097</td> 
    </tr> 
    <tr>
     <td class="property_name">last flyway file</td>
-     <td id="lastFlywayFile" class="property_value">V101__domain_add_dns_refresh_request_time.sql</td>
+     <td id="lastFlywayFile" class="property_value">V102__add_indexes_to_domain_history_sub_tables.sql</td>
    </tr>
   </tbody>
  </table> 
@@ -288,11 +284,7 @@ td.section {
     generated on
    </text> 
    <text text-anchor="start" x="4755.52" y="-10.8" font-family="Helvetica,sans-Serif" font-size="14.00">
-<<<<<<< HEAD
-     2021-08-10 20:50:07.996141
-=======
-     2021-08-11 23:54:54.724849
->>>>>>> 475839855 (Add the domain DNS refresh request time field to the DB schema)
+     2021-09-14 16:11:19.580097
    </text> 
    <polygon fill="none" stroke="#888888" points="4668.02,-4 4668.02,-44 4933.02,-44 4933.02,-4 4668.02,-4" /> <!-- allocationtoken_a08ccbef --> 
    <g id="node1" class="node"> 
@@ -9732,6 +9724,23 @@ td.section {
     <td class="minwidth">ds_data_history_revision_id</td> 
     <td class="minwidth">ascending</td> 
    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
+    <tr> 
+     <td colspan="2" class="name">domain_history_to_ds_data_history_idx</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">domain_repo_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">domain_history_revision_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
   </tbody>
  </table> 
  <p>&nbsp;</p> 
@@ -10627,6 +10636,23 @@ td.section {
     <td class="minwidth">id</td> 
     <td class="minwidth">ascending</td> 
    </tr> 
+    <tr> 
+     <td colspan="3"></td> 
+    </tr> 
+    <tr> 
+     <td colspan="2" class="name">domain_history_to_transaction_record_idx</td> 
+     <td class="description right">[non-unique index]</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">domain_repo_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
+    <tr> 
+     <td class="spacer"></td> 
+     <td class="minwidth">history_revision_id</td> 
+     <td class="minwidth">ascending</td> 
+    </tr> 
   </tbody>
  </table> 
  <p>&nbsp;</p> 
@@ -99,3 +99,4 @@ V98__add_last_expiring_cert_notification_sent_date.sql
 V99__drop_kms_secret_table.sql
 V100__database_migration_schedule.sql
 V101__domain_add_dns_refresh_request_time.sql
+V102__add_indexes_to_domain_history_sub_tables.sql
@@ -0,0 +1,19 @@
+-- Copyright 2021 The Nomulus Authors. All Rights Reserved.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+CREATE INDEX IF NOT EXISTS domain_history_to_transaction_record_idx
+  ON "DomainTransactionRecord" (domain_repo_id, history_revision_id);
+
+CREATE INDEX IF NOT EXISTS domain_history_to_ds_data_history_idx
+  ON "DomainDsDataHistory" (domain_repo_id, domain_history_revision_id);
@@ -274,6 +274,7 @@
        billing_contact text,
        deletion_poll_message_id int8,
        deletion_poll_message_history_id int8,
+        dns_refresh_request_time timestamptz,
        domain_name text,
        idn_table_name text,
        last_transfer_time timestamptz,
@@ -347,6 +348,7 @@
        billing_contact text,
        deletion_poll_message_id int8,
        deletion_poll_message_history_id int8,
+        dns_refresh_request_time timestamptz,
        domain_name text,
        idn_table_name text,
        last_transfer_time timestamptz,
@@ -788,6 +790,7 @@ create index IDXc5aw4pk1vkd6ymhvkpanmoadv on "Domain" (domain_name);
 create index IDXr22ciyccwi9rrqmt1ro0s59qf on "Domain" (tech_contact);
 create index IDXrwl38wwkli1j7gkvtywi9jokq on "Domain" (tld);
 create index IDXa7fu0bqynfb79rr80528b4jqt on "Domain" (registrant_contact);
+create index IDXcws5mvmpl8o10wrhde780ors2 on "Domain" (dns_refresh_request_time);
 create index IDXrh4xmrot9bd63o382ow9ltfig on "DomainHistory" (creation_time);
 create index IDXaro1omfuaxjwmotk3vo00trwm on "DomainHistory" (history_registrar_id);
 create index IDXsu1nam10cjes9keobapn5jvxj on "DomainHistory" (history_type);
@@ -1489,6 +1489,20 @@ CREATE UNIQUE INDEX database_migration_state_schedule_singleton ON public."Datab
 CREATE INDEX domain_dns_refresh_request_time_idx ON public."Domain" USING btree (dns_refresh_request_time);


+--
+-- Name: domain_history_to_ds_data_history_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX domain_history_to_ds_data_history_idx ON public."DomainDsDataHistory" USING btree (domain_repo_id, domain_history_revision_id);
+
+
+--
+-- Name: domain_history_to_transaction_record_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX domain_history_to_transaction_record_idx ON public."DomainTransactionRecord" USING btree (domain_repo_id, history_revision_id);
+
+
 --
 -- Name: idx1iy7njgb7wjmj9piml4l2g0qi; Type: INDEX; Schema: public; Owner: -
 --
@@ -972,11 +972,11 @@ as read, and won't be delivered again until the next year of their recurrence.

 An EPP flow for requesting {@link PollMessage}s.

-This flow uses an eventually consistent Datastore query to return the oldest
-poll message for the registrar, as well as the total number of pending messages.
-Note that poll messages whose event time is in the future (i.e. they are
-speculative and could still be changed or rescinded) are ignored. The externally
-visible id for the poll message that the registrar sees is generated by {@link
+This flow uses an eventually consistent query to return the oldest poll message
+for the registrar, as well as the total number of pending messages. Note that
+poll messages whose event time is in the future (i.e. they are speculative and
+could still be changed or rescinded) are ignored. The externally visible id for
+the poll message that the registrar sees is generated by {@link
 PollMessageExternalKeyConverter}.

 ### Errors
Author	SHA1	Message	Date
Weimin Yu	a8ddb5c053	Add indexes to DomainHistory sub tables (#1319 ) * Add indexes to DomainHistory sub tables Add indexes to DomainTransactionRecord and DomainDsDataHistory to speed up query for DomainHistory. Without these indexes, DomainHistory loading is extremely slow: 2 QPS with current production data.	2021-09-14 14:54:21 -04:00
Ben McIlwain	88be34808d	Rename Spec11Pipeline's Subdomain -> DomainNameInfo (#1318 ) * Rename Spec11Pipeline's Subdomain -> DomainNameInfo "Subdomain" never made any sense as a class name because these are all second-level domain names, along with a little bit of metadata such as some registrar info. "DomainNameInfo" is a better fit.	2021-09-14 14:07:26 -04:00
sarahcaseybot	099555c789	Remove "Datastore" from comments when using both DBs (#1310 ) * Change datastore references in comments to be more generic * Update flows doc	2021-09-13 18:02:41 -04:00
Weimin Yu	652d099e0e	Preserve update_time when replicating to SQL (#1316 ) * Preserve update_time when replicating to SQL Prevent InitSqlPipeline from changing the UpdateAutoTimestamp fields in entities.	2021-09-13 16:55:20 -04:00
gbrodman	f31d77c570	Update references to RDAP RFCs (#1313 ) * Update references to RDAP RFCs There were minor changes to the RDAP RFCs used -- we don't need to change anything since we already comply with all of the changes, but we should refer to the newer RFCs in the code.	2021-09-13 15:12:52 -04:00
gbrodman	1ced2b0a5d	Fix URS check in BackfillRegistryLocksCommand (#1315 ) * Fix URS check in BackfillRegistryLocksCommand "reason" can be null	2021-09-13 14:15:25 -04:00
Rachel Guan	0e6b5e949d	Set up sendExpiringCertificateNotificationEmail cron job in prod (#1272 ) * Set up sendExpiringCertificateNotificationEmail cron job in prod	2021-09-13 13:36:11 -04:00
Weimin Yu	fe714329c9	Add a temporary fix to Hibernate detach in BEAM queries (#1314 ) * Add a temporary fix to Hibernate detach in query Make all queries in RegistryQuery (exclusively used by BEAM) use EntityManager.clear() to detach entities. This is a temporary measure that unblocks work in BEAM. We will revert the work once JpaTransactionManager can detach entities properly for all types of queries. Also fixed regression bugs that broke query result streaming: - The code that sets query fetch size was not carried over from QueryComposer. - The new ReadOnlyCheckingTypedQuery class did not override parent's getResultStream() method, which calls getList().	2021-09-10 21:23:07 -04:00
Ben McIlwain	9297b11a57	Add the DNS refresh request time field to the Domain tables (#1279 ) * Add the DNS refresh request time field to the Domain tables This isn't used yet, but it will eventually be the replacement for the dns-pull task queue once we get further in the migration. * Merge branch 'master' into domain-dns-dirty	2021-09-10 14:18:32 -04:00
Michael Muller	031f4ea063	Remove VKeyTranslatorFactory.createVKey(String) (#1312 ) * Remove VKeyTranslatorFactory.createVKey(String) This method serves the same function as VKey.fromWebsafeKey(), and isn't used anywhere. Move the test for it into VKeyTest and use it to instead test fromWebsafeKey() (which didn't previously have a test).	2021-09-09 12:24:02 -04:00
Rachel Guan	0a25182fea	Keep new lines/spaces as the way it shows in config yaml (#1311 ) * Paste the email content as the way it is in doc.	2021-09-08 22:00:32 -04:00