mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-04-25 10:40:49 +00:00
Code Issues Releases Wiki Activity

Skip synthetic history entries for resources that don't need them (#1320)

Browse Source 
* Skip synthetic history entries for resources that don't need them

The reason for creating synthetic history entries is so that we can
guarantee that each EppResource's most recent *History object contains
that resource at that point in time. If the most recent *History object
in SQL contains that resource already, there is no need to create a
synthetic *History object for that resource.
This commit is contained in:
gbrodman
2021-09-17 12:10:15 -04:00
committed by GitHub
parent ad6471b3fd
commit 12dac76dc8
11 changed files with 161 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/main/java/google/registry/model/index/ForeignKeyIndex.java
View File

@@ -228,7 +228,7 @@ public abstract class ForeignKeyIndex<E extends EppResource> extends BackupGroup
tm().transact(
() ->
jpaTm()
.query(
.criteriaQuery(
CriteriaQueryBuilder.create(clazz)
.whereFieldIsIn(property, foreignKeys)
.build())

46
core/src/main/java/google/registry/model/reporting/HistoryEntryDao.java
View File

@@ -24,6 +24,7 @@ import static google.registry.util.DateTimeUtils.START_OF_TIME;
import com.google.common.collect.ImmutableCollection;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Streams;
import google.registry.model.EppResource;
import google.registry.model.contact.ContactHistory;
@@ -49,6 +50,25 @@ import org.joda.time.DateTime;
*/
public class HistoryEntryDao {
public static ImmutableMap<Class<? extends EppResource>, Class<? extends HistoryEntry>>
RESOURCE_TYPES_TO_HISTORY_TYPES =
ImmutableMap.of(
ContactResource.class,
ContactHistory.class,
DomainBase.class,
DomainHistory.class,
HostResource.class,
HostHistory.class);
public static ImmutableMap<Class<? extends HistoryEntry>, String> REPO_ID_FIELD_NAMES =
ImmutableMap.of(
ContactHistory.class,
"contactRepoId",
DomainHistory.class,
"domainRepoId",
HostHistory.class,
"hostRepoId");
/** Loads all history objects in the times specified, including all types. */
public static ImmutableList<HistoryEntry> loadAllHistoryObjects(
DateTime afterTime, DateTime beforeTime) {
@@ -164,7 +184,7 @@ public class HistoryEntryDao {
private static <T extends HistoryEntry> Stream<T> loadHistoryObjectFromSqlByRegistrars(
Class<T> historyClass, ImmutableCollection<String> registrarIds) {
return jpaTm()
.query(
.criteriaQuery(
CriteriaQueryBuilder.create(historyClass)
.whereFieldIsIn("clientId", registrarIds)
.build())
@@ -188,34 +208,32 @@ public class HistoryEntryDao {
return ImmutableList.sortedCopyOf(
Comparator.comparing(HistoryEntry::getModificationTime),
jpaTm().query(criteriaQuery).getResultList());
jpaTm().criteriaQuery(criteriaQuery).getResultList());
}
private static Class<? extends HistoryEntry> getHistoryClassFromParent(
Class<? extends EppResource> parent) {
if (parent.equals(ContactResource.class)) {
return ContactHistory.class;
} else if (parent.equals(DomainBase.class)) {
return DomainHistory.class;
} else if (parent.equals(HostResource.class)) {
return HostHistory.class;
if (!RESOURCE_TYPES_TO_HISTORY_TYPES.containsKey(parent)) {
throw new IllegalArgumentException(
String.format("Unknown history type for parent %s", parent.getName()));
}
throw new IllegalArgumentException(
String.format("Unknown history type for parent %s", parent.getName()));
return RESOURCE_TYPES_TO_HISTORY_TYPES.get(parent);
}
private static String getRepoIdFieldNameFromHistoryClass(
Class<? extends HistoryEntry> historyClass) {
return historyClass.equals(ContactHistory.class)
? "contactRepoId"
: historyClass.equals(DomainHistory.class) ? "domainRepoId" : "hostRepoId";
if (!REPO_ID_FIELD_NAMES.containsKey(historyClass)) {
throw new IllegalArgumentException(
String.format("Unknown history type %s", historyClass.getName()));
}
return REPO_ID_FIELD_NAMES.get(historyClass);
}
private static <T extends HistoryEntry> List<T> loadAllHistoryObjectsFromSql(
Class<T> historyClass, DateTime afterTime, DateTime beforeTime) {
CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
return jpaTm()
.query(
.criteriaQuery(
CriteriaQueryBuilder.create(historyClass)
.where("modificationTime", criteriaBuilder::greaterThanOrEqualTo, afterTime)
.where("modificationTime", criteriaBuilder::lessThanOrEqualTo, beforeTime)

2
core/src/main/java/google/registry/persistence/transaction/JpaTransactionManager.java
View File

@@ -36,7 +36,7 @@ public interface JpaTransactionManager extends TransactionManager {
<T> TypedQuery<T> query(String sqlString, Class<T> resultClass);
/** Creates a JPA SQU query for the given criteria query. */
<T> TypedQuery<T> query(CriteriaQuery<T> criteriaQuery);
<T> TypedQuery<T> criteriaQuery(CriteriaQuery<T> criteriaQuery);
/**
* Creates a JPA SQL query for the given query string.

2
core/src/main/java/google/registry/persistence/transaction/JpaTransactionManagerImpl.java
View File

@@ -135,7 +135,7 @@ public class JpaTransactionManagerImpl implements JpaTransactionManager {
}
@Override
public <T> TypedQuery<T> query(CriteriaQuery<T> criteriaQuery) {
public <T> TypedQuery<T> criteriaQuery(CriteriaQuery<T> criteriaQuery) {
return new DetachingTypedQuery<>(getEntityManager().createQuery(criteriaQuery));
}

2
core/src/main/java/google/registry/rdap/RdapDomainSearchAction.java
View File

@@ -591,7 +591,7 @@ public class RdapDomainSearchAction extends RdapSearchActionBase {
cursorString.get());
}
jpaTm()
.query(queryBuilder.build())
.criteriaQuery(queryBuilder.build())
.getResultStream()
.filter(this::isAuthorized)
.forEach(

2
core/src/main/java/google/registry/rdap/RdapSearchActionBase.java
View File

@@ -202,7 +202,7 @@ public abstract class RdapSearchActionBase extends RdapActionBase {
desiredRegistrar.get());
}
List<T> queryResult =
jpaTm().query(builder.build()).setMaxResults(querySizeLimit).getResultList();
jpaTm().criteriaQuery(builder.build()).setMaxResults(querySizeLimit).getResultList();
if (checkForVisibility) {
return filterResourcesByVisibility(queryResult, querySizeLimit);
} else {

91
core/src/main/java/google/registry/tools/javascrap/CreateSyntheticHistoryEntriesAction.java
View File

@@ -15,7 +15,10 @@
package google.registry.tools.javascrap;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.model.reporting.HistoryEntryDao.REPO_ID_FIELD_NAMES;
import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.collect.ImmutableList;
@@ -26,12 +29,16 @@ import google.registry.mapreduce.inputs.EppResourceInputs;
import google.registry.model.EppResource;
import google.registry.model.domain.DomainHistory;
import google.registry.model.reporting.HistoryEntry;
import google.registry.persistence.transaction.CriteriaQueryBuilder;
import google.registry.rde.RdeStagingAction;
import google.registry.request.Action;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.tools.server.GenerateZoneFilesAction;
import java.util.Optional;
import javax.inject.Inject;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
/**
* A mapreduce that creates synthetic history objects in SQL for all {@link EppResource} objects.
@@ -61,6 +68,9 @@ import javax.inject.Inject;
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
public class CreateSyntheticHistoryEntriesAction implements Runnable {
private static final String HISTORY_REASON =
"Backfill EppResource history objects during Cloud SQL migration";
private final MapreduceRunner mrRunner;
private final Response response;
private final String registryAdminRegistrarId;
@@ -97,6 +107,50 @@ public class CreateSyntheticHistoryEntriesAction implements Runnable {
.sendLinkToMapreduceConsole(response);
}
// Lifted from HistoryEntryDao
private static Optional<? extends HistoryEntry> mostRecentHistoryFromSql(EppResource resource) {
return jpaTm()
.transact(
() -> {
// The class we're searching from is based on which parent type (e.g. Domain) we have
Class<? extends HistoryEntry> historyClass =
getHistoryClassFromParent(resource.getClass());
// The field representing repo ID unfortunately varies by history class
String repoIdFieldName = getRepoIdFieldNameFromHistoryClass(historyClass);
CriteriaBuilder criteriaBuilder = jpaTm().getEntityManager().getCriteriaBuilder();
CriteriaQuery<? extends HistoryEntry> criteriaQuery =
CriteriaQueryBuilder.create(historyClass)
.where(repoIdFieldName, criteriaBuilder::equal, resource.getRepoId())
.orderByDesc("modificationTime")
.build();
return jpaTm()
.criteriaQuery(criteriaQuery)
.setMaxResults(1)
.getResultStream()
.findFirst();
});
}
// Lifted from HistoryEntryDao
private static Class<? extends HistoryEntry> getHistoryClassFromParent(
Class<? extends EppResource> parent) {
if (!RESOURCE_TYPES_TO_HISTORY_TYPES.containsKey(parent)) {
throw new IllegalArgumentException(
String.format("Unknown history type for parent %s", parent.getName()));
}
return RESOURCE_TYPES_TO_HISTORY_TYPES.get(parent);
}
// Lifted from HistoryEntryDao
private static String getRepoIdFieldNameFromHistoryClass(
Class<? extends HistoryEntry> historyClass) {
if (!REPO_ID_FIELD_NAMES.containsKey(historyClass)) {
throw new IllegalArgumentException(
String.format("Unknown history type %s", historyClass.getName()));
}
return REPO_ID_FIELD_NAMES.get(historyClass);
}
/** Mapper to re-save all EPP resources. */
public static class CreateSyntheticHistoryEntriesMapper
extends Mapper<Key<EppResource>, Void, Void> {
@@ -109,20 +163,27 @@ public class CreateSyntheticHistoryEntriesAction implements Runnable {
@Override
public final void map(final Key<EppResource> resourceKey) {
tm().transact(
() -> {
EppResource eppResource = auditedOfy().load().key(resourceKey).now();
tm().put(
HistoryEntry.createBuilderForResource(eppResource)
.setRegistrarId(registryAdminRegistrarId)
.setBySuperuser(true)
.setRequestedByRegistrar(false)
.setModificationTime(tm().getTransactionTime())
.setReason(
"Backfill EppResource history objects during Cloud SQL migration")
.setType(HistoryEntry.Type.SYNTHETIC)
.build());
});
EppResource eppResource = auditedOfy().load().key(resourceKey).now();
// Only save new history entries if the most recent history for this object in SQL does not
// have the resource at that point in time already
Optional<? extends HistoryEntry> maybeHistory = mostRecentHistoryFromSql(eppResource);
if (maybeHistory
.map(history -> !history.getResourceAtPointInTime().isPresent())
.orElse(true)) {
ofyTm()
.transact(
() ->
ofyTm()
.put(
HistoryEntry.createBuilderForResource(eppResource)
.setRegistrarId(registryAdminRegistrarId)
.setBySuperuser(true)
.setRequestedByRegistrar(false)
.setModificationTime(ofyTm().getTransactionTime())
.setReason(HISTORY_REASON)
.setType(HistoryEntry.Type.SYNTHETIC)
.build()));
}
}
}
}