From eeca51667e5e7350aacd8fb118bf8c3e0a08d3d9 Mon Sep 17 00:00:00 2001 From: Weimin Yu Date: Thu, 19 May 2022 23:35:55 -0400 Subject: [PATCH] Optimize RDAP entity event query (#1635) * Optimize RDAP entity event query For each EPP entity, directly load the latest HistoryEntry per event type instead of loading all events through the HistoryEntryDao. Although most entities have a small number of history entries, there are a few entities with many entries, enough to cause OutOfMemory error. --- .../model/reporting/HistoryEntryDao.java | 4 +- .../registry/rdap/RdapJsonFormatter.java | 67 +++++++++++++++---- .../registry/rdap/RdapJsonFormatterTest.java | 15 +++++ 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/google/registry/model/reporting/HistoryEntryDao.java b/core/src/main/java/google/registry/model/reporting/HistoryEntryDao.java index 988717298..d61f3ac78 100644 --- a/core/src/main/java/google/registry/model/reporting/HistoryEntryDao.java +++ b/core/src/main/java/google/registry/model/reporting/HistoryEntryDao.java @@ -211,7 +211,7 @@ public class HistoryEntryDao { jpaTm().criteriaQuery(criteriaQuery).getResultList()); } - private static Class getHistoryClassFromParent( + public static Class getHistoryClassFromParent( Class parent) { if (!RESOURCE_TYPES_TO_HISTORY_TYPES.containsKey(parent)) { throw new IllegalArgumentException( @@ -220,7 +220,7 @@ public class HistoryEntryDao { return RESOURCE_TYPES_TO_HISTORY_TYPES.get(parent); } - private static String getRepoIdFieldNameFromHistoryClass( + public static String getRepoIdFieldNameFromHistoryClass( Class historyClass) { if (!REPO_ID_FIELD_NAMES.containsKey(historyClass)) { throw new IllegalArgumentException( diff --git a/core/src/main/java/google/registry/rdap/RdapJsonFormatter.java b/core/src/main/java/google/registry/rdap/RdapJsonFormatter.java index 026cc984f..92b4c93ea 100644 --- a/core/src/main/java/google/registry/rdap/RdapJsonFormatter.java +++ b/core/src/main/java/google/registry/rdap/RdapJsonFormatter.java @@ -20,11 +20,14 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.common.collect.ImmutableSetMultimap.toImmutableSetMultimap; import static google.registry.model.EppResourceUtils.isLinked; +import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm; import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.persistence.transaction.TransactionManagerUtil.transactIfJpaTm; import static google.registry.rdap.RdapIcannStandardInformation.CONTACT_REDACTED_VALUE; import static google.registry.util.CollectionUtils.union; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -81,6 +84,7 @@ import java.util.Set; import java.util.stream.Stream; import javax.annotation.Nullable; import javax.inject.Inject; +import javax.persistence.Entity; import org.joda.time.DateTime; /** @@ -150,6 +154,19 @@ public class RdapJsonFormatter { INTERNAL } + /** + * JPQL query template for finding the latest history entry per event type for an EPP entity. + * + *

User should replace '%entityName%', '%repoIdField%', and '%repoIdValue%' with valid values. + * A DomainHistory query may look like below: {@code select e from DomainHistory e where + * domainRepoId = '17-Q9JYB4C' and modificationTime in (select max(modificationTime) from + * DomainHistory where domainRepoId = '17-Q9JYB4C' and type is not null group by type)} + */ + private static final String GET_LAST_HISTORY_BY_TYPE_JPQL_TEMPLATE = + "select e from %entityName% e where %repoIdField% = '%repoIdValue%' and modificationTime in " + + " (select max(modificationTime) from %entityName% where " + + " %repoIdField% = '%repoIdValue%' and type is not null group by type)"; + /** Map of EPP status values to the RDAP equivalents. */ private static final ImmutableMap STATUS_TO_RDAP_STATUS_MAP = new ImmutableMap.Builder() @@ -855,17 +872,8 @@ public class RdapJsonFormatter { return rolesBuilder.build(); } - /** - * Creates the list of optional events to list in domain, nameserver, or contact replies. - * - *

Only has entries for optional events that won't be shown in "SUMMARY" versions of these - * objects. These are either stated as optional in the RDAP Response Profile 15feb19, or not - * mentioned at all but thought to be useful anyway. - * - *

Any required event should be added elsewhere, preferably without using HistoryEntries (so - * that we don't need to load HistoryEntries for "summary" responses). - */ - private ImmutableList makeOptionalEvents(EppResource resource) { + @VisibleForTesting + ImmutableMap getLastHistoryEntryByType(EppResource resource) { HashMap lastEntryOfType = Maps.newHashMap(); // Events (such as transfer, but also create) can appear multiple times. We only want the last // time they appeared. @@ -878,8 +886,26 @@ public class RdapJsonFormatter { // 2.3.2.3 An event of *eventAction* type *transfer*, with the last date and time that the // domain was transferred. The event of *eventAction* type *transfer* MUST be omitted if the // domain name has not been transferred since it was created. - Iterable historyEntries = - HistoryEntryDao.loadHistoryObjectsForResource(resource.createVKey()); + Iterable historyEntries; + if (tm().isOfy()) { + historyEntries = HistoryEntryDao.loadHistoryObjectsForResource(resource.createVKey()); + } else { + VKey resourceVkey = resource.createVKey(); + Class historyClass = + HistoryEntryDao.getHistoryClassFromParent(resourceVkey.getKind()); + String entityName = historyClass.getAnnotation(Entity.class).name(); + if (Strings.isNullOrEmpty(entityName)) { + entityName = historyClass.getSimpleName(); + } + String repoIdFieldName = HistoryEntryDao.getRepoIdFieldNameFromHistoryClass(historyClass); + String jpql = + GET_LAST_HISTORY_BY_TYPE_JPQL_TEMPLATE + .replace("%entityName%", entityName) + .replace("%repoIdField%", repoIdFieldName) + .replace("%repoIdValue%", resourceVkey.getSqlKey().toString()); + historyEntries = + jpaTm().transact(() -> jpaTm().getEntityManager().createQuery(jpql).getResultList()); + } for (HistoryEntry historyEntry : historyEntries) { EventAction rdapEventAction = HISTORY_ENTRY_TYPE_TO_RDAP_EVENT_ACTION_MAP.get(historyEntry.getType()); @@ -889,6 +915,21 @@ public class RdapJsonFormatter { } lastEntryOfType.put(rdapEventAction, historyEntry); } + return ImmutableMap.copyOf(lastEntryOfType); + } + + /** + * Creates the list of optional events to list in domain, nameserver, or contact replies. + * + *

Only has entries for optional events that won't be shown in "SUMMARY" versions of these + * objects. These are either stated as optional in the RDAP Response Profile 15feb19, or not + * mentioned at all but thought to be useful anyway. + * + *

Any required event should be added elsewhere, preferably without using HistoryEntries (so + * that we don't need to load HistoryEntries for "summary" responses). + */ + private ImmutableList makeOptionalEvents(EppResource resource) { + ImmutableMap lastEntryOfType = getLastHistoryEntryByType(resource); ImmutableList.Builder eventsBuilder = new ImmutableList.Builder<>(); DateTime creationTime = resource.getCreationTime(); DateTime lastChangeTime = diff --git a/core/src/test/java/google/registry/rdap/RdapJsonFormatterTest.java b/core/src/test/java/google/registry/rdap/RdapJsonFormatterTest.java index 8abca5f88..c53afba9b 100644 --- a/core/src/test/java/google/registry/rdap/RdapJsonFormatterTest.java +++ b/core/src/test/java/google/registry/rdap/RdapJsonFormatterTest.java @@ -15,6 +15,7 @@ package google.registry.rdap; import static com.google.common.truth.Truth.assertThat; +import static google.registry.rdap.RdapDataStructures.EventAction.TRANSFER; import static google.registry.rdap.RdapTestHelper.assertThat; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistResource; @@ -28,7 +29,9 @@ import static google.registry.testing.TestDataHelper.loadFile; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.gson.Gson; import com.google.gson.JsonObject; import google.registry.model.contact.ContactResource; @@ -51,6 +54,7 @@ import google.registry.testing.DualDatabaseTest; import google.registry.testing.FakeClock; import google.registry.testing.InjectExtension; import google.registry.testing.TestOfyAndSql; +import google.registry.testing.TestSqlOnly; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.RegisterExtension; @@ -482,6 +486,17 @@ class RdapJsonFormatterTest { .isEqualTo(loadJson("rdapjson_domain_summary.json")); } + @TestSqlOnly + void testGetLastHistoryEntryByType() { + // Expected data are from "rdapjson_domain_summary.json" + assertThat( + Maps.transformValues( + rdapJsonFormatter.getLastHistoryEntryByType(domainBaseFull), + HistoryEntry::getModificationTime)) + .containsExactlyEntriesIn( + ImmutableMap.of(TRANSFER, DateTime.parse("1999-12-01T00:00:00.000Z"))); + } + @TestOfyAndSql void testDomain_logged_out() { rdapJsonFormatter.rdapAuthorization = RdapAuthorization.PUBLIC_AUTHORIZATION;