mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-09-05 15:46:55 +00:00
Code Issues Releases Wiki Activity

Remove Ofy code from various flow-related classes (#1653)

Browse Source 
This included removing ofy-specific code from various tests. Also, some
of the other tests (e.g. RdapDomainActionTest) had to be configured to
use only SQL -- otherwise, as it currently stands, they were trying to
use ofy.

We also delete the CreateSyntheticHistoryEntriesAction and pipeline
because they're no longer relevant, and impossible to test (the goal of
the actions were to create objects in ofy which doesn't happen any
more).
This commit is contained in:
gbrodman
2022-06-07 11:43:33 -04:00
committed by GitHub
parent 413fd1dc94
commit 9912e35ea2
34 changed files with 200 additions and 2680 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/google/registry/batch/DeleteContactsAndHostsAction.java
-621
View File
@@ -1,621 +0,0 @@
// Copyright 2017 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.batch;
import static com.google.appengine.api.taskqueue.QueueConstants.maxLeaseCount;
import static com.google.appengine.api.taskqueue.QueueFactory.getQueue;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.math.IntMath.divide;
import static com.googlecode.objectify.Key.getKind;
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_CLIENT_TRANSACTION_ID;
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_IS_SUPERUSER;
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTED_TIME;
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_REQUESTING_CLIENT_ID;
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_RESOURCE_KEY;
import static google.registry.batch.AsyncTaskEnqueuer.PARAM_SERVER_TRANSACTION_ID;
import static google.registry.batch.AsyncTaskEnqueuer.QUEUE_ASYNC_DELETE;
import static google.registry.model.EppResourceUtils.isActive;
import static google.registry.model.EppResourceUtils.isDeleted;
import static google.registry.model.ResourceTransferUtils.denyPendingTransfer;
import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete;
import static google.registry.model.ResourceTransferUtils.updateForeignKeyIndexDeletionTime;
import static google.registry.model.eppcommon.StatusValue.PENDING_DELETE;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE;
import static google.registry.model.reporting.HistoryEntry.Type.CONTACT_DELETE_FAILURE;
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE;
import static google.registry.model.reporting.HistoryEntry.Type.HOST_DELETE_FAILURE;
import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static java.math.RoundingMode.CEILING;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.logging.Level.INFO;
import static java.util.logging.Level.SEVERE;
import static org.joda.time.Duration.standardHours;
import com.google.appengine.api.taskqueue.LeaseOptions;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.TaskHandle;
import com.google.appengine.api.taskqueue.TransientFailureException;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.appengine.tools.mapreduce.Reducer;
import com.google.appengine.tools.mapreduce.ReducerInput;
import com.google.auto.value.AutoValue;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
import com.google.common.collect.Multiset;
import com.google.common.flogger.FluentLogger;
import com.googlecode.objectify.Key;
import google.registry.batch.AsyncTaskMetrics.OperationResult;
import google.registry.batch.AsyncTaskMetrics.OperationType;
import google.registry.batch.DeleteContactsAndHostsAction.DeletionResult.Type;
import google.registry.dns.DnsQueue;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.mapreduce.UnlockerOutput;
import google.registry.mapreduce.inputs.EppResourceInputs;
import google.registry.mapreduce.inputs.NullInput;
import google.registry.model.EppResource;
import google.registry.model.ImmutableObject;
import google.registry.model.annotations.DeleteAfterMigration;
import google.registry.model.annotations.ExternalMessagingName;
import google.registry.model.contact.ContactResource;
import google.registry.model.domain.DomainBase;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.eppcommon.Trid;
import google.registry.model.eppoutput.EppResponse.ResponseData;
import google.registry.model.host.HostResource;
import google.registry.model.poll.PendingActionNotificationResponse.ContactPendingActionNotificationResponse;
import google.registry.model.poll.PendingActionNotificationResponse.HostPendingActionNotificationResponse;
import google.registry.model.poll.PollMessage;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.server.Lock;
import google.registry.persistence.VKey;
import google.registry.request.Action;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.Clock;
import google.registry.util.NonFinalForTesting;
import google.registry.util.RequestStatusChecker;
import google.registry.util.Retrier;
import google.registry.util.SystemClock;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.logging.Level;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import org.joda.time.DateTime;
import org.joda.time.Duration;
/**
* A mapreduce that processes batch asynchronous deletions of contact and host resources by mapping
* over all domains and checking for any references to the contacts/hosts in pending deletion.
*/
@Deprecated
@Action(
service = Action.Service.BACKEND,
path = "/_dr/task/deleteContactsAndHosts",
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
@DeleteAfterMigration
public class DeleteContactsAndHostsAction implements Runnable {
static final String KIND_CONTACT = getKind(ContactResource.class);
static final String KIND_HOST = getKind(HostResource.class);
private static final Duration LEASE_LENGTH = standardHours(4);
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
private static final int MAX_REDUCE_SHARDS = 50;
private static final int DELETES_PER_SHARD = 5;
@Inject AsyncTaskMetrics asyncTaskMetrics;
@Inject Clock clock;
@Inject MapreduceRunner mrRunner;
@Inject @Named(QUEUE_ASYNC_DELETE) Queue queue;
@Inject RequestStatusChecker requestStatusChecker;
@Inject Response response;
@Inject Retrier retrier;
@Inject DeleteContactsAndHostsAction() {}
@Override
public void run() {
// Check if the lock can be acquired, and if not, a previous run of this mapreduce is still
// executing, so return early.
Optional<Lock> lock =
Lock.acquire(
DeleteContactsAndHostsAction.class.getSimpleName(),
null,
LEASE_LENGTH,
requestStatusChecker,
false);
if (!lock.isPresent()) {
logRespondAndUnlock(INFO, "Can't acquire lock; aborting.", lock);
return;
}
// Lease the async tasks to process.
LeaseOptions options =
LeaseOptions.Builder.withCountLimit(maxLeaseCount())
.leasePeriod(LEASE_LENGTH.getStandardSeconds(), SECONDS);
List<TaskHandle> tasks = queue.leaseTasks(options);
asyncTaskMetrics.recordContactHostDeletionBatchSize(tasks.size());
// Check if there are no tasks to process, and if so, return early.
if (tasks.isEmpty()) {
logRespondAndUnlock(INFO, "No contact/host deletion tasks in pull queue; finishing.", lock);
return;
}
Multiset<String> kindCounts = HashMultiset.create(2);
ImmutableList.Builder<DeletionRequest> builder = new ImmutableList.Builder<>();
ImmutableList.Builder<Key<? extends EppResource>> resourceKeys = new ImmutableList.Builder<>();
final List<DeletionRequest> requestsToDelete = new ArrayList<>();
for (TaskHandle task : tasks) {
try {
DeletionRequest deletionRequest = DeletionRequest.createFromTask(task, clock.nowUtc());
if (deletionRequest.isDeletionAllowed()) {
builder.add(deletionRequest);
resourceKeys.add(deletionRequest.key());
kindCounts.add(deletionRequest.key().getKind());
} else {
requestsToDelete.add(deletionRequest);
}
} catch (Exception e) {
logger.atSevere().withCause(e).log(
"Could not parse async deletion request, delaying task for a day: %s", task);
// Grab the lease for a whole day, so that it won't continue throwing errors every five
// minutes.
queue.modifyTaskLease(task, 1L, DAYS);
}
}
deleteStaleTasksWithRetry(requestsToDelete);
ImmutableList<DeletionRequest> deletionRequests = builder.build();
if (deletionRequests.isEmpty()) {
logRespondAndUnlock(
INFO, "No async deletions to process because all were already handled.", lock);
} else {
logger.atInfo().log(
"Processing asynchronous deletion of %d contacts and %d hosts: %s",
kindCounts.count(KIND_CONTACT), kindCounts.count(KIND_HOST), resourceKeys.build());
runMapreduce(deletionRequests, lock);
}
}
/**
* Deletes a list of tasks associated with deletion requests from the async delete queue using a
* retrier.
*/
private void deleteStaleTasksWithRetry(final List<DeletionRequest> deletionRequests) {
if (deletionRequests.isEmpty()) {
return;
}
final List<TaskHandle> tasks =
deletionRequests.stream().map(DeletionRequest::task).collect(toImmutableList());
retrier.callWithRetry(() -> queue.deleteTask(tasks), TransientFailureException.class);
deletionRequests.forEach(
deletionRequest ->
asyncTaskMetrics.recordAsyncFlowResult(
deletionRequest.getMetricOperationType(),
OperationResult.STALE,
deletionRequest.requestedTime()));
}
private void runMapreduce(ImmutableList<DeletionRequest> deletionRequests, Optional<Lock> lock) {
try {
int numReducers =
Math.min(MAX_REDUCE_SHARDS, divide(deletionRequests.size(), DELETES_PER_SHARD, CEILING));
mrRunner
.setJobName("Check for EPP resource references and then delete")
.setModuleName("backend")
.setDefaultReduceShards(numReducers)
.runMapreduce(
new DeleteContactsAndHostsMapper(deletionRequests),
new DeleteEppResourceReducer(),
ImmutableList.of(
// Add an extra shard that maps over a null domain. See the mapper code for why.
new NullInput<>(), EppResourceInputs.createEntityInput(DomainBase.class)),
new UnlockerOutput<Void>(lock.get()))
.sendLinkToMapreduceConsole(response);
} catch (Throwable t) {
logRespondAndUnlock(SEVERE, "Error starting mapreduce to delete contacts/hosts.", lock);
}
}
private void logRespondAndUnlock(Level level, String message, Optional<Lock> lock) {
logger.at(level).log(message);
response.setPayload(message);
lock.ifPresent(Lock::release);
}
/**
* A mapper that iterates over all {@link DomainBase} entities.
*
* <p>It emits the target key and {@code true} for domains referencing the target resource. For
* the special input of {@code null} it emits the target key and {@code false}.
*/
public static class DeleteContactsAndHostsMapper
extends Mapper<DomainBase, DeletionRequest, Boolean> {
private static final long serialVersionUID = -253652818502690537L;
private final ImmutableList<DeletionRequest> deletionRequests;
DeleteContactsAndHostsMapper(ImmutableList<DeletionRequest> resourcesToDelete) {
this.deletionRequests = resourcesToDelete;
}
@Override
public void map(DomainBase domain) {
for (DeletionRequest deletionRequest : deletionRequests) {
if (domain == null) {
// The reducer only runs if at least one value is emitted. We add a null input to the
// mapreduce and emit one 'false' for each deletion request so that the reducer always
// runs for each requested deletion (so that it can finish up tasks if nothing else).
emit(deletionRequest, false);
} else if (isActive(domain, deletionRequest.lastUpdateTime())
&& isLinked(domain, deletionRequest.key())) {
emit(deletionRequest, true);
getContext()
.incrementCounter(
String.format("active Domain-%s links found", deletionRequest.key().getKind()));
}
}
if (domain != null) {
getContext().incrementCounter("domains processed");
}
}
/** Determine whether the target resource is a linked resource on the domain. */
private boolean isLinked(DomainBase domain, Key<? extends EppResource> resourceKey) {
if (resourceKey.getKind().equals(KIND_CONTACT)) {
return domain
.getReferencedContacts()
.contains(VKey.from((Key<ContactResource>) resourceKey));
} else if (resourceKey.getKind().equals(KIND_HOST)) {
return domain.getNameservers().contains(VKey.from((Key<HostResource>) resourceKey));
} else {
throw new IllegalStateException("EPP resource key of unknown type: " + resourceKey);
}
}
}
/**
* A reducer that checks if the EPP resource to be deleted is referenced anywhere, and then
* deletes it if not and unmarks it for deletion if so.
*/
public static class DeleteEppResourceReducer
extends Reducer<DeletionRequest, Boolean, Void> {
private static final long serialVersionUID = 6569363449285506326L;
private static final DnsQueue dnsQueue = DnsQueue.create();
@NonFinalForTesting
private static AsyncTaskMetrics asyncTaskMetrics = new AsyncTaskMetrics(new SystemClock());
@Override
public void reduce(final DeletionRequest deletionRequest, ReducerInput<Boolean> values) {
final boolean hasNoActiveReferences = !Iterators.contains(values, true);
logger.atInfo().log("Processing async deletion request for %s.", deletionRequest.key());
DeletionResult result =
tm()
.transactNew(
() -> {
DeletionResult deletionResult =
attemptToDeleteResource(deletionRequest, hasNoActiveReferences);
getQueue(QUEUE_ASYNC_DELETE).deleteTask(deletionRequest.task());
return deletionResult;
});
asyncTaskMetrics.recordAsyncFlowResult(
deletionRequest.getMetricOperationType(),
result.getMetricOperationResult(),
deletionRequest.requestedTime());
String resourceNamePlural = deletionRequest.key().getKind() + "s";
getContext().incrementCounter(result.type().renderCounterText(resourceNamePlural));
logger.atInfo().log(
"Result of async deletion for resource %s: %s",
deletionRequest.key(), result.pollMessageText());
}
private DeletionResult attemptToDeleteResource(
DeletionRequest deletionRequest, boolean hasNoActiveReferences) {
DateTime now = tm().getTransactionTime();
EppResource resource =
auditedOfy().load().key(deletionRequest.key()).now().cloneProjectedAtTime(now);
// Double-check transactionally that the resource is still active and in PENDING_DELETE.
if (!doesResourceStateAllowDeletion(resource, now)) {
return DeletionResult.create(Type.ERRORED, "");
}
// Contacts and external hosts have a direct client id. For subordinate hosts it needs to be
// read off of the superordinate domain.
String resourceRegistrarId = resource.getPersistedCurrentSponsorRegistrarId();
if (resource instanceof HostResource && ((HostResource) resource).isSubordinate()) {
resourceRegistrarId =
tm().loadByKey(((HostResource) resource).getSuperordinateDomain())
.cloneProjectedAtTime(now)
.getCurrentSponsorRegistrarId();
}
boolean requestedByCurrentOwner =
resourceRegistrarId.equals(deletionRequest.requestingClientId());
boolean deleteAllowed =
hasNoActiveReferences && (requestedByCurrentOwner || deletionRequest.isSuperuser());
String resourceTypeName =
resource.getClass().getAnnotation(ExternalMessagingName.class).value();
String pollMessageText =
deleteAllowed
? String.format("Deleted %s %s.", resourceTypeName, resource.getForeignKey())
: String.format(
"Can't delete %s %s because %s.",
resourceTypeName,
resource.getForeignKey(),
requestedByCurrentOwner
? "it is referenced by a domain"
: "it was transferred prior to deletion");
HistoryEntry historyEntry =
HistoryEntry.createBuilderForResource(resource)
.setRegistrarId(deletionRequest.requestingClientId())
.setModificationTime(now)
.setType(getHistoryEntryType(resource, deleteAllowed))
.build();
PollMessage.OneTime pollMessage =
new PollMessage.OneTime.Builder()
.setRegistrarId(deletionRequest.requestingClientId())
.setMsg(pollMessageText)
.setParent(historyEntry)
.setEventTime(now)
.setResponseData(
getPollMessageResponseData(deletionRequest, resource, deleteAllowed, now))
.build();
EppResource resourceToSave;
if (deleteAllowed) {
EppResource.Builder<?, ?> resourceToSaveBuilder;
if (resource instanceof ContactResource) {
ContactResource contact = (ContactResource) resource;
// Handle pending transfers on contact deletion.
if (contact.getStatusValues().contains(StatusValue.PENDING_TRANSFER)) {
contact =
denyPendingTransfer(
contact, SERVER_CANCELLED, now, deletionRequest.requestingClientId());
}
// Wipe out PII on contact deletion.
resourceToSaveBuilder = contact.asBuilder().wipeOut();
} else {
resourceToSaveBuilder = resource.asBuilder();
}
resourceToSave = resourceToSaveBuilder.setDeletionTime(now).setStatusValues(null).build();
performDeleteTasks(resource, resourceToSave, now, historyEntry);
updateForeignKeyIndexDeletionTime(resourceToSave);
} else {
resourceToSave = resource.asBuilder().removeStatusValue(PENDING_DELETE).build();
}
auditedOfy()
.save()
.<ImmutableObject>entities(resourceToSave, historyEntry.asHistoryEntry(), pollMessage);
return DeletionResult.create(
deleteAllowed ? Type.DELETED : Type.NOT_DELETED, pollMessageText);
}
private static ImmutableList<? extends ResponseData> getPollMessageResponseData(
DeletionRequest deletionRequest,
EppResource resource,
boolean deleteAllowed,
DateTime now) {
@Nullable String clientTransactionId = deletionRequest.clientTransactionId();
String serverTransactionId = deletionRequest.serverTransactionId();
Trid trid = Trid.create(clientTransactionId, serverTransactionId);
if (resource instanceof HostResource) {
return ImmutableList.of(
HostPendingActionNotificationResponse.create(
((HostResource) resource).getHostName(), deleteAllowed, trid, now));
} else if (resource instanceof ContactResource) {
return ImmutableList.of(
ContactPendingActionNotificationResponse.create(
((ContactResource) resource).getContactId(), deleteAllowed, trid, now));
} else {
throw new IllegalStateException("EPP resource of unknown type " + Key.create(resource));
}
}
/**
* Determine the proper history entry type for the delete operation, as a function of
* whether or not the delete was successful.
*/
private HistoryEntry.Type getHistoryEntryType(EppResource resource, boolean successfulDelete) {
if (resource instanceof ContactResource) {
return successfulDelete ? CONTACT_DELETE : CONTACT_DELETE_FAILURE;
} else if (resource instanceof HostResource) {
return successfulDelete ? HOST_DELETE : HOST_DELETE_FAILURE;
} else {
throw new IllegalStateException("EPP resource of unknown type: " + Key.create(resource));
}
}
/** Perform any type-specific tasks on the resource to be deleted (and/or its dependencies). */
private void performDeleteTasks(
EppResource existingResource,
EppResource deletedResource,
DateTime deletionTime,
HistoryEntry historyEntryForDelete) {
if (existingResource instanceof ContactResource) {
handlePendingTransferOnDelete(
(ContactResource) existingResource,
(ContactResource) deletedResource,
deletionTime,
historyEntryForDelete);
} else if (existingResource instanceof HostResource) {
HostResource host = (HostResource) existingResource;
if (host.isSubordinate()) {
dnsQueue.addHostRefreshTask(host.getHostName());
tm().put(
tm().loadByKey(host.getSuperordinateDomain())
.asBuilder()
.removeSubordinateHost(host.getHostName())
.build());
}
} else {
throw new IllegalStateException(
"EPP resource of unknown type: " + Key.create(existingResource));
}
}
}
/** A class that encapsulates the values of a request to delete a contact or host. */
@AutoValue
abstract static class DeletionRequest implements Serializable {
private static final long serialVersionUID = -4612618525760839240L;
abstract Key<? extends EppResource> key();
abstract DateTime lastUpdateTime();
/**
* The client id of the registrar that requested this deletion (which might NOT be the same as
* the actual current owner of the resource).
*/
abstract String requestingClientId();
/** First half of TRID for the original request, split for serializability. */
@Nullable
abstract String clientTransactionId();
/** Second half of TRID for the original request, split for serializability. */
abstract String serverTransactionId();
abstract boolean isSuperuser();
abstract DateTime requestedTime();
abstract boolean isDeletionAllowed();
abstract TaskHandle task();
@AutoValue.Builder
abstract static class Builder {
abstract Builder setKey(Key<? extends EppResource> key);
abstract Builder setLastUpdateTime(DateTime lastUpdateTime);
abstract Builder setRequestingClientId(String requestingClientId);
abstract Builder setClientTransactionId(@Nullable String clientTransactionId);
abstract Builder setServerTransactionId(String serverTransactionId);
abstract Builder setIsSuperuser(boolean isSuperuser);
abstract Builder setRequestedTime(DateTime requestedTime);
abstract Builder setIsDeletionAllowed(boolean isDeletionAllowed);
abstract Builder setTask(TaskHandle task);
abstract DeletionRequest build();
}
static DeletionRequest createFromTask(TaskHandle task, DateTime now)
throws Exception {
ImmutableMap<String, String> params = ImmutableMap.copyOf(task.extractParams());
VKey<EppResource> resourceKey =
VKey.create(
checkNotNull(params.get(PARAM_RESOURCE_KEY), "Resource to delete not specified"));
EppResource resource =
checkNotNull(
auditedOfy().load().key(resourceKey.getOfyKey()).now(),
"Resource to delete doesn't exist");
checkState(
resource instanceof ContactResource || resource instanceof HostResource,
"Cannot delete a %s via this action",
resource.getClass().getSimpleName());
return new AutoValue_DeleteContactsAndHostsAction_DeletionRequest.Builder()
.setKey(resourceKey.getOfyKey())
.setLastUpdateTime(resource.getUpdateTimestamp().getTimestamp())
.setRequestingClientId(
checkNotNull(
params.get(PARAM_REQUESTING_CLIENT_ID), "Requesting client id not specified"))
// Note that client transaction ID is optional, in which case this sets it to null.
.setClientTransactionId(params.get(PARAM_CLIENT_TRANSACTION_ID))
.setServerTransactionId(
checkNotNull(
params.get(PARAM_SERVER_TRANSACTION_ID), "Server transaction id not specified"))
.setIsSuperuser(
Boolean.parseBoolean(
checkNotNull(params.get(PARAM_IS_SUPERUSER), "Is superuser not specified")))
.setRequestedTime(
DateTime.parse(
checkNotNull(params.get(PARAM_REQUESTED_TIME), "Requested time not specified")))
.setIsDeletionAllowed(doesResourceStateAllowDeletion(resource, now))
.setTask(task)
.build();
}
OperationType getMetricOperationType() {
if (key().getKind().equals(KIND_CONTACT)) {
return OperationType.CONTACT_DELETE;
} else if (key().getKind().equals(KIND_HOST)) {
return OperationType.HOST_DELETE;
} else {
throw new IllegalStateException(
String.format(
"Cannot determine async operation type for metric for resource %s", key()));
}
}
}
/** A class that encapsulates the values resulting from attempted contact/host deletion. */
@AutoValue
abstract static class DeletionResult {
enum Type {
DELETED("%s deleted", OperationResult.SUCCESS),
NOT_DELETED("%s not deleted", OperationResult.FAILURE),
ERRORED("%s errored out during deletion", OperationResult.ERROR);
private final String counterFormat;
private final OperationResult operationResult;
Type(String counterFormat, OperationResult operationResult) {
this.counterFormat = counterFormat;
this.operationResult = operationResult;
}
String renderCounterText(String resourceName) {
return String.format(counterFormat, resourceName);
}
}
abstract Type type();
abstract String pollMessageText();
static DeletionResult create(Type type, String pollMessageText) {
return new AutoValue_DeleteContactsAndHostsAction_DeletionResult(type, pollMessageText);
}
OperationResult getMetricOperationResult() {
return type().operationResult;
}
}
static boolean doesResourceStateAllowDeletion(EppResource resource, DateTime now) {
Key<EppResource> key = Key.create(resource);
if (isDeleted(resource, now)) {
logger.atWarning().log("Cannot asynchronously delete %s because it is already deleted.", key);
return false;
}
if (!resource.getStatusValues().contains(PENDING_DELETE)) {
logger.atWarning().log(
"Cannot asynchronously delete %s because it is not in PENDING_DELETE.", key);
return false;
}
return true;
}
}
core/src/main/java/google/registry/env/common/backend/WEB-INF/web.xml
Vendored
-15
View File
@@ -334,15 +334,6 @@
<url-pattern>/_dr/task/resaveEntity</url-pattern>
</servlet-mapping>
<!--
Deletes contacts and hosts enqueued for asynchronous deletion if they are
not referenced by any domain.
-->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/deleteContactsAndHosts</url-pattern>
</servlet-mapping>
<!-- Enqueues DNS update tasks following a host rename. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
@@ -422,12 +413,6 @@ have been in the database for a certain period of time. -->
<url-pattern>/_dr/task/wipeOutDatastore</url-pattern>
</servlet-mapping>
<!-- Action to create synthetic history entries during async replication to SQL -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
<url-pattern>/_dr/task/createSyntheticHistoryEntries</url-pattern>
</servlet-mapping>
<!-- Action to sync Datastore to a snapshot of the primary SQL database. -->
<servlet-mapping>
<servlet-name>backend-servlet</servlet-name>
core/src/main/java/google/registry/flows/ResourceFlowUtils.java
+11 -48
View File
@@ -15,7 +15,6 @@
package google.registry.flows;
import static com.google.common.collect.Sets.intersection;
import static google.registry.model.EppResourceUtils.getLinkedDomainKeys;
import static google.registry.model.EppResourceUtils.isLinked;
import static google.registry.model.EppResourceUtils.loadByForeignKey;
import static google.registry.model.index.ForeignKeyIndex.loadAndGetKey;
@@ -54,8 +53,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
import org.joda.time.DateTime;
/** Static utility functions for resource flows. */
@@ -63,12 +60,6 @@ public final class ResourceFlowUtils {
private ResourceFlowUtils() {}
/**
* In {@link #checkLinkedDomains(String, DateTime, Class, Function)}, check this (arbitrary)
* number of query results.
*/
private static final int FAILFAST_CHECK_COUNT = 5;
/** Check that the given registrarId corresponds to the owner of given resource. */
public static void verifyResourceOwnership(String myRegistrarId, EppResource resource)
throws EppException {
@@ -85,46 +76,18 @@ public final class ResourceFlowUtils {
* consistent, so we only check a few domains to fail fast.
*/
public static <R extends EppResource> void checkLinkedDomains(
final String targetId,
final DateTime now,
final Class<R> resourceClass,
final Function<DomainBase, ImmutableSet<?>> getPotentialReferences)
throws EppException {
final String targetId, final DateTime now, final Class<R> resourceClass) throws EppException {
EppException failfastException =
tm().isOfy()
? tm().doTransactionless(
() -> {
final ForeignKeyIndex<R> fki =
ForeignKeyIndex.load(resourceClass, targetId, now);
if (fki == null) {
return new ResourceDoesNotExistException(resourceClass, targetId);
}
// Query for the first few linked domains, and if found, actually load them.
// The query is eventually consistent and so might be very stale, but the
// direct load will not be stale, just non-transactional. If we find at least
// one actual reference then we can reliably fail. If we don't find any,
// we can't trust the query and need to do the full mapreduce.
Iterable<VKey<DomainBase>> keys =
getLinkedDomainKeys(fki.getResourceKey(), now, FAILFAST_CHECK_COUNT);
VKey<R> resourceVKey = fki.getResourceKey();
Predicate<DomainBase> predicate =
domain -> getPotentialReferences.apply(domain).contains(resourceVKey);
return tm().loadByKeys(keys).values().stream().anyMatch(predicate)
? new ResourceToDeleteIsReferencedException()
: null;
})
: tm().transact(
() -> {
final ForeignKeyIndex<R> fki =
ForeignKeyIndex.load(resourceClass, targetId, now);
if (fki == null) {
return new ResourceDoesNotExistException(resourceClass, targetId);
}
return isLinked(fki.getResourceKey(), now)
? new ResourceToDeleteIsReferencedException()
: null;
});
tm().transact(
() -> {
final ForeignKeyIndex<R> fki = ForeignKeyIndex.load(resourceClass, targetId, now);
if (fki == null) {
return new ResourceDoesNotExistException(resourceClass, targetId);
}
return isLinked(fki.getResourceKey(), now)
? new ResourceToDeleteIsReferencedException()
: null;
});
if (failfastException != null) {
throw failfastException;
}
core/src/main/java/google/registry/flows/contact/ContactDeleteFlow.java
+12 -30
View File
@@ -23,7 +23,6 @@ import static google.registry.flows.ResourceFlowUtils.verifyResourceOwnership;
import static google.registry.model.ResourceTransferUtils.denyPendingTransfer;
import static google.registry.model.ResourceTransferUtils.handlePendingTransferOnDelete;
import static google.registry.model.eppoutput.Result.Code.SUCCESS;
import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
import static google.registry.model.transfer.TransferStatus.SERVER_CANCELLED;
import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -39,13 +38,11 @@ import google.registry.flows.TransactionalFlow;
import google.registry.flows.annotations.ReportingSpec;
import google.registry.model.contact.ContactHistory;
import google.registry.model.contact.ContactResource;
import google.registry.model.domain.DomainBase;
import google.registry.model.domain.metadata.MetadataExtension;
import google.registry.model.eppcommon.AuthInfo;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.eppcommon.Trid;
import google.registry.model.eppoutput.EppResponse;
import google.registry.model.eppoutput.Result.Code;
import google.registry.model.reporting.HistoryEntry.Type;
import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
import java.util.Optional;
@@ -97,41 +94,26 @@ public final class ContactDeleteFlow implements TransactionalFlow {
extensionManager.validate();
assertAsyncActionsAreAllowed();
DateTime now = tm().getTransactionTime();
checkLinkedDomains(targetId, now, ContactResource.class, DomainBase::getReferencedContacts);
checkLinkedDomains(targetId, now, ContactResource.class);
ContactResource existingContact = loadAndVerifyExistence(ContactResource.class, targetId, now);
verifyNoDisallowedStatuses(existingContact, DISALLOWED_STATUSES);
verifyOptionalAuthInfo(authInfo, existingContact);
if (!isSuperuser) {
verifyResourceOwnership(registrarId, existingContact);
}
Type historyEntryType;
Code resultCode;
ContactResource newContact;
if (tm().isOfy()) {
asyncTaskEnqueuer.enqueueAsyncDelete(
existingContact, tm().getTransactionTime(), registrarId, trid, isSuperuser);
newContact = existingContact.asBuilder().addStatusValue(StatusValue.PENDING_DELETE).build();
historyEntryType = Type.CONTACT_PENDING_DELETE;
resultCode = SUCCESS_WITH_ACTION_PENDING;
} else {
// Handle pending transfers on contact deletion.
newContact =
existingContact.getStatusValues().contains(StatusValue.PENDING_TRANSFER)
? denyPendingTransfer(existingContact, SERVER_CANCELLED, now, registrarId)
: existingContact;
// Wipe out PII on contact deletion.
newContact =
newContact.asBuilder().wipeOut().setStatusValues(null).setDeletionTime(now).build();
historyEntryType = Type.CONTACT_DELETE;
resultCode = SUCCESS;
}
// Handle pending transfers on contact deletion.
ContactResource newContact =
existingContact.getStatusValues().contains(StatusValue.PENDING_TRANSFER)
? denyPendingTransfer(existingContact, SERVER_CANCELLED, now, registrarId)
: existingContact;
// Wipe out PII on contact deletion.
newContact =
newContact.asBuilder().wipeOut().setStatusValues(null).setDeletionTime(now).build();
ContactHistory contactHistory =
historyBuilder.setType(historyEntryType).setContact(newContact).build();
if (!tm().isOfy()) {
handlePendingTransferOnDelete(existingContact, newContact, now, contactHistory);
}
historyBuilder.setType(Type.CONTACT_DELETE).setContact(newContact).build();
handlePendingTransferOnDelete(existingContact, newContact, now, contactHistory);
tm().insert(contactHistory);
tm().update(newContact);
return responseBuilder.setResultFromCode(resultCode).build();
return responseBuilder.setResultFromCode(SUCCESS).build();
}
}
core/src/main/java/google/registry/flows/domain/DomainFlowUtils.java
+8 -19
View File
@@ -26,7 +26,6 @@ import static com.google.common.collect.Sets.difference;
import static com.google.common.collect.Sets.intersection;
import static com.google.common.collect.Sets.union;
import static google.registry.model.domain.DomainBase.MAX_REGISTRATION_YEARS;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.model.tld.Registries.findTldForName;
import static google.registry.model.tld.Registries.getTlds;
import static google.registry.model.tld.Registry.TldState.GENERAL_AVAILABILITY;
@@ -1159,24 +1158,14 @@ public class DomainFlowUtils {
private static List<? extends HistoryEntry> findRecentHistoryEntries(
DomainBase domainBase, DateTime now, Duration maxSearchPeriod) {
if (tm().isOfy()) {
return auditedOfy()
.load()
.type(HistoryEntry.class)
.ancestor(domainBase)
.filter("modificationTime >=", now.minus(maxSearchPeriod))
.order("modificationTime")
.list();
} else {
return jpaTm()
.query(
"FROM DomainHistory WHERE modificationTime >= :beginning AND domainRepoId = "
+ ":repoId ORDER BY modificationTime ASC",
DomainHistory.class)
.setParameter("beginning", now.minus(maxSearchPeriod))
.setParameter("repoId", domainBase.getRepoId())
.getResultList();
}
return jpaTm()
.query(
"FROM DomainHistory WHERE modificationTime >= :beginning AND domainRepoId = "
+ ":repoId ORDER BY modificationTime ASC",
DomainHistory.class)
.setParameter("beginning", now.minus(maxSearchPeriod))
.setParameter("repoId", domainBase.getRepoId())
.getResultList();
}
/** Resource linked to this domain does not exist. */
core/src/main/java/google/registry/flows/domain/DomainInfoFlow.java
-5
View File
@@ -102,11 +102,6 @@ public final class DomainInfoFlow implements Flow {
verifyOptionalAuthInfo(authInfo, domain);
flowCustomLogic.afterValidation(
AfterValidationParameters.newBuilder().setDomain(domain).build());
// In ofy, refetch all referenced resources.
if (tm().isOfy()) {
tm().loadByKeys(domain.getNameservers());
tm().loadByKeys(domain.getReferencedContacts());
}
// Registrars can only see a few fields on unauthorized domains.
// This is a policy decision that is left up to us by the rfcs.
DomainInfoData.Builder infoBuilder =
core/src/main/java/google/registry/flows/host/HostDeleteFlow.java
+12 -28
View File
@@ -21,7 +21,6 @@ import static google.registry.flows.ResourceFlowUtils.verifyNoDisallowedStatuses
import static google.registry.flows.ResourceFlowUtils.verifyResourceOwnership;
import static google.registry.flows.host.HostFlowUtils.validateHostName;
import static google.registry.model.eppoutput.Result.Code.SUCCESS;
import static google.registry.model.eppoutput.Result.Code.SUCCESS_WITH_ACTION_PENDING;
import static google.registry.persistence.transaction.TransactionManagerFactory.assertAsyncActionsAreAllowed;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
@@ -36,15 +35,12 @@ import google.registry.flows.FlowModule.TargetId;
import google.registry.flows.TransactionalFlow;
import google.registry.flows.annotations.ReportingSpec;
import google.registry.model.EppResource;
import google.registry.model.domain.DomainBase;
import google.registry.model.domain.metadata.MetadataExtension;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.eppcommon.Trid;
import google.registry.model.eppoutput.EppResponse;
import google.registry.model.eppoutput.Result;
import google.registry.model.host.HostHistory;
import google.registry.model.host.HostResource;
import google.registry.model.reporting.HistoryEntry;
import google.registry.model.reporting.HistoryEntry.Type;
import google.registry.model.reporting.IcannReportingTypes.ActivityReportField;
import javax.inject.Inject;
@@ -100,7 +96,7 @@ public final class HostDeleteFlow implements TransactionalFlow {
assertAsyncActionsAreAllowed();
DateTime now = tm().getTransactionTime();
validateHostName(targetId);
checkLinkedDomains(targetId, now, HostResource.class, DomainBase::getNameservers);
checkLinkedDomains(targetId, now, HostResource.class);
HostResource existingHost = loadAndVerifyExistence(HostResource.class, targetId, now);
verifyNoDisallowedStatuses(existingHost, DISALLOWED_STATUSES);
if (!isSuperuser) {
@@ -112,31 +108,19 @@ public final class HostDeleteFlow implements TransactionalFlow {
: existingHost;
verifyResourceOwnership(registrarId, owningResource);
}
HistoryEntry.Type historyEntryType;
Result.Code resultCode;
HostResource newHost;
if (tm().isOfy()) {
asyncTaskEnqueuer.enqueueAsyncDelete(
existingHost, tm().getTransactionTime(), registrarId, trid, isSuperuser);
newHost = existingHost.asBuilder().addStatusValue(StatusValue.PENDING_DELETE).build();
historyEntryType = Type.HOST_PENDING_DELETE;
resultCode = SUCCESS_WITH_ACTION_PENDING;
} else {
newHost = existingHost.asBuilder().setStatusValues(null).setDeletionTime(now).build();
if (existingHost.isSubordinate()) {
dnsQueue.addHostRefreshTask(existingHost.getHostName());
tm().update(
tm().loadByKey(existingHost.getSuperordinateDomain())
.asBuilder()
.removeSubordinateHost(existingHost.getHostName())
.build());
}
historyEntryType = Type.HOST_DELETE;
resultCode = SUCCESS;
HostResource newHost =
existingHost.asBuilder().setStatusValues(null).setDeletionTime(now).build();
if (existingHost.isSubordinate()) {
dnsQueue.addHostRefreshTask(existingHost.getHostName());
tm().update(
tm().loadByKey(existingHost.getSuperordinateDomain())
.asBuilder()
.removeSubordinateHost(existingHost.getHostName())
.build());
}
historyBuilder.setType(historyEntryType).setHost(newHost);
historyBuilder.setType(Type.HOST_DELETE).setHost(newHost);
tm().insert(historyBuilder.build());
tm().update(newHost);
return responseBuilder.setResultFromCode(resultCode).build();
return responseBuilder.setResultFromCode(SUCCESS).build();
}
}
core/src/main/java/google/registry/flows/poll/PollAckFlow.java
+1 -7
View File
@@ -102,19 +102,13 @@ public final class PollAckFlow implements TransactionalFlow {
// This keeps track of whether we should include the current acked message in the updated
// message count that's returned to the user. The only case where we do so is if an autorenew
// poll message is acked, but its next event is already ready to be delivered.
boolean includeAckedMessageInCount = ackPollMessage(pollMessage);
ackPollMessage(pollMessage);
// We need to return the new queue length. If this was the last message in the queue being
// acked, then we return a special status code indicating that. Note that the query will
// include the message being acked.
int messageCount = tm().doTransactionless(() -> getPollMessageCount(registrarId, now));
// Within the same transaction, Datastore will not reflect the updated count (potentially
// reduced by one thanks to the acked poll message). SQL will, however, so we shouldn't reduce
// the count in the SQL case.
if (!includeAckedMessageInCount && tm().isOfy()) {
messageCount--;
}
if (messageCount <= 0) {
return responseBuilder.setResultFromCode(SUCCESS_WITH_NO_MESSAGES).build();
}
core/src/main/java/google/registry/flows/poll/PollFlowUtils.java
+1 -4
View File
@@ -47,13 +47,12 @@ public final class PollFlowUtils {
* <p>The only case where we do so is if an autorenew poll message is acked, but its next event is
* already ready to be delivered.
*/
public static boolean ackPollMessage(PollMessage pollMessage) {
public static void ackPollMessage(PollMessage pollMessage) {
checkArgument(
isBeforeOrAt(pollMessage.getEventTime(), tm().getTransactionTime()),
"Cannot ACK poll message with ID %s because its event time is in the future: %s",
pollMessage.getId(),
pollMessage.getEventTime());
boolean includeAckedMessageInCount = false;
if (pollMessage instanceof PollMessage.OneTime) {
// One-time poll messages are deleted once acked.
tm().delete(pollMessage.createVKey());
@@ -68,14 +67,12 @@ public final class PollFlowUtils {
// autorenew poll message has no more events to deliver and should be deleted.
if (nextEventTime.isBefore(autorenewPollMessage.getAutorenewEndTime())) {
tm().put(autorenewPollMessage.asBuilder().setEventTime(nextEventTime).build());
includeAckedMessageInCount = isBeforeOrAt(nextEventTime, tm().getTransactionTime());
} else {
tm().delete(autorenewPollMessage.createVKey());
}
} else {
throw new IllegalArgumentException("Unknown poll message type: " + pollMessage.getClass());
}
return includeAckedMessageInCount;
}
/**
core/src/main/java/google/registry/model/reporting/HistoryEntryDao.java
+21 -57
View File
@@ -16,9 +16,7 @@ package google.registry.model.reporting;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
@@ -72,29 +70,16 @@ public class HistoryEntryDao {
/** Loads all history objects in the times specified, including all types. */
public static ImmutableList<HistoryEntry> loadAllHistoryObjects(
DateTime afterTime, DateTime beforeTime) {
if (tm().isOfy()) {
return Streams.stream(
auditedOfy()
.load()
.type(HistoryEntry.class)
.order("modificationTime")
.filter("modificationTime >=", afterTime)
.filter("modificationTime <=", beforeTime))
.map(HistoryEntry::toChildHistoryEntity)
.collect(toImmutableList());
} else {
return jpaTm()
.transact(
() ->
new ImmutableList.Builder<HistoryEntry>()
.addAll(
loadAllHistoryObjectsFromSql(ContactHistory.class, afterTime, beforeTime))
.addAll(
loadAllHistoryObjectsFromSql(DomainHistory.class, afterTime, beforeTime))
.addAll(
loadAllHistoryObjectsFromSql(HostHistory.class, afterTime, beforeTime))
.build());
}
return jpaTm()
.transact(
() ->
new ImmutableList.Builder<HistoryEntry>()
.addAll(
loadAllHistoryObjectsFromSql(ContactHistory.class, afterTime, beforeTime))
.addAll(
loadAllHistoryObjectsFromSql(DomainHistory.class, afterTime, beforeTime))
.addAll(loadAllHistoryObjectsFromSql(HostHistory.class, afterTime, beforeTime))
.build());
}
/** Loads all history objects corresponding to the given {@link EppResource}. */
@@ -115,21 +100,8 @@ public class HistoryEntryDao {
/** Loads all history objects in the time period specified for the given {@link EppResource}. */
public static ImmutableList<HistoryEntry> loadHistoryObjectsForResource(
VKey<? extends EppResource> parentKey, DateTime afterTime, DateTime beforeTime) {
if (tm().isOfy()) {
return Streams.stream(
auditedOfy()
.load()
.type(HistoryEntry.class)
.ancestor(parentKey.getOfyKey())
.order("modificationTime")
.filter("modificationTime >=", afterTime)
.filter("modificationTime <=", beforeTime))
.map(HistoryEntry::toChildHistoryEntity)
.collect(toImmutableList());
} else {
return jpaTm()
.transact(() -> loadHistoryObjectsForResourceFromSql(parentKey, afterTime, beforeTime));
}
return jpaTm()
.transact(() -> loadHistoryObjectsForResourceFromSql(parentKey, afterTime, beforeTime));
}
/**
@@ -162,23 +134,15 @@ public class HistoryEntryDao {
/** Loads all history objects from all time from the given registrars. */
public static Iterable<HistoryEntry> loadHistoryObjectsByRegistrars(
ImmutableCollection<String> registrarIds) {
if (tm().isOfy()) {
return auditedOfy()
.load()
.type(HistoryEntry.class)
.filter("clientId in", registrarIds)
.order("modificationTime");
} else {
return jpaTm()
.transact(
() ->
Streams.concat(
loadHistoryObjectFromSqlByRegistrars(ContactHistory.class, registrarIds),
loadHistoryObjectFromSqlByRegistrars(DomainHistory.class, registrarIds),
loadHistoryObjectFromSqlByRegistrars(HostHistory.class, registrarIds))
.sorted(Comparator.comparing(HistoryEntry::getModificationTime))
.collect(toImmutableList()));
}
return jpaTm()
.transact(
() ->
Streams.concat(
loadHistoryObjectFromSqlByRegistrars(ContactHistory.class, registrarIds),
loadHistoryObjectFromSqlByRegistrars(DomainHistory.class, registrarIds),
loadHistoryObjectFromSqlByRegistrars(HostHistory.class, registrarIds))
.sorted(Comparator.comparing(HistoryEntry::getModificationTime))
.collect(toImmutableList()));
}
private static <T extends HistoryEntry> Stream<T> loadHistoryObjectFromSqlByRegistrars(
core/src/main/java/google/registry/module/backend/BackendRequestComponent.java
-6
View File
@@ -23,7 +23,6 @@ import google.registry.backup.ExportCommitLogDiffAction;
import google.registry.backup.ReplayCommitLogsToSqlAction;
import google.registry.backup.SyncDatastoreToSqlSnapshotAction;
import google.registry.batch.BatchModule;
import google.registry.batch.DeleteContactsAndHostsAction;
import google.registry.batch.DeleteExpiredDomainsAction;
import google.registry.batch.DeleteLoadTestDataAction;
import google.registry.batch.DeleteProberDataAction;
@@ -93,7 +92,6 @@ import google.registry.tmch.TmchCrlAction;
import google.registry.tmch.TmchDnlAction;
import google.registry.tmch.TmchModule;
import google.registry.tmch.TmchSmdrlAction;
import google.registry.tools.javascrap.CreateSyntheticHistoryEntriesAction;
/** Dagger component with per-request lifetime for "backend" App Engine module. */
@RequestScope
@@ -137,10 +135,6 @@ interface BackendRequestComponent {
CopyDetailReportsAction copyDetailReportAction();
CreateSyntheticHistoryEntriesAction createSyntheticHistoryEntriesAction();
DeleteContactsAndHostsAction deleteContactsAndHostsAction();
DeleteExpiredDomainsAction deleteExpiredDomainsAction();
DeleteLoadTestDataAction deleteLoadTestDataAction();
core/src/main/java/google/registry/tools/javascrap/CreateSyntheticHistoryEntriesAction.java
-198
View File
@@ -1,198 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools.javascrap;
import static google.registry.model.ofy.ObjectifyService.auditedOfy;
import static google.registry.model.reporting.HistoryEntryDao.REPO_ID_FIELD_NAMES;
import static google.registry.model.reporting.HistoryEntryDao.RESOURCE_TYPES_TO_HISTORY_TYPES;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.ofyTm;
import com.google.appengine.tools.mapreduce.Mapper;
import com.google.common.base.CaseFormat;
import com.google.common.collect.ImmutableList;
import com.googlecode.objectify.Key;
import google.registry.config.RegistryConfig.Config;
import google.registry.mapreduce.MapreduceRunner;
import google.registry.mapreduce.inputs.EppResourceInputs;
import google.registry.model.EppResource;
import google.registry.model.domain.DomainHistory;
import google.registry.model.reporting.HistoryEntry;
import google.registry.rde.RdeStagingAction;
import google.registry.request.Action;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.tools.server.GenerateZoneFilesAction;
import javax.inject.Inject;
/**
* A mapreduce that creates synthetic history objects in SQL for all {@link EppResource} objects.
*
* <p>Certain operations, e.g. {@link RdeStagingAction} or {@link GenerateZoneFilesAction}, require
* that we are able to answer the question of "What did this EPP resource look like at a point in
* time?" In the Datastore world, we are able to answer this question using the commit logs, however
* this is no longer possible in the SQL world. Instead, we will use the history objects, e.g.
* {@link DomainHistory} to see what a particular resource looked like at that point in time, since
* history objects store a snapshot of that resource.
*
* <p>This command creates a synthetic history object at the current point in time for every single
* EPP resource to guarantee that later on, when examining in-the-past resources, we have some
* history object for which the EppResource field is filled. This synthetic history object contains
* basically nothing and its only purpose is to create a populated history object in SQL through
* asynchronous replication.
*
* <p>NB: This class operates entirely in Datastore, which may be counterintuitive at first glance.
* However, since this is meant to be run during the Datastore-primary, SQL-secondary stage of the
* migration, we want to make sure that we are using the most up-to-date version of the data. The
* resource field of the history objects will be populated during asynchronous migration, e.g. in
* {@link DomainHistory#beforeSqlSaveOnReplay}.
*/
@Action(
service = Action.Service.BACKEND,
path = "/_dr/task/createSyntheticHistoryEntries",
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
public class CreateSyntheticHistoryEntriesAction implements Runnable {
private static final String HISTORY_REASON =
"Backfill EppResource history objects during Cloud SQL migration";
private final MapreduceRunner mrRunner;
private final Response response;
private final String registryAdminRegistrarId;
@Inject
CreateSyntheticHistoryEntriesAction(
MapreduceRunner mrRunner,
Response response,
@Config("registryAdminClientId") String registryAdminRegistrarId) {
this.mrRunner = mrRunner;
this.response = response;
this.registryAdminRegistrarId = registryAdminRegistrarId;
}
/**
* The default number of shards to run the map-only mapreduce on.
*
* <p>This is much lower than the default of 100 because we can afford it being slow and we want
* to avoid overloading SQL.
*/
private static final int NUM_SHARDS = 10;
@Override
public void run() {
mrRunner
.setJobName("Create a synthetic HistoryEntry for each EPP resource")
.setModuleName("backend")
.setDefaultMapShards(NUM_SHARDS)
.runMapOnly(
new CreateSyntheticHistoryEntriesMapper(registryAdminRegistrarId),
ImmutableList.of(EppResourceInputs.createKeyInput(EppResource.class)))
.sendLinkToMapreduceConsole(response);
}
// Returns true iff any of the *History objects in SQL contain a representation of this resource
// at the point in time that the *History object was created.
private static boolean hasHistoryContainingResource(EppResource resource) {
return jpaTm()
.transact(
() -> {
// Use READ COMMITTED isolation level so that any long-living queries don't cause
// collection of predicate locks to spiral out of control (as would happen with a
// SERIALIZABLE isolation level)
//
// NB: setting the isolation level inside the transaction only works for Postgres and
// will be reverted to the default once the transaction is committed.
jpaTm()
.getEntityManager()
.createNativeQuery("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
.executeUpdate();
// The class we're searching from is based on which parent type (e.g. Domain) we have
Class<? extends HistoryEntry> historyClass =
getHistoryClassFromParent(resource.getClass());
// The field representing repo ID unfortunately varies by history class
String repoIdFieldName =
CaseFormat.LOWER_CAMEL.to(
CaseFormat.LOWER_UNDERSCORE,
getRepoIdFieldNameFromHistoryClass(historyClass));
// The "history" fields in the *History objects are all prefixed with "history_". If
// any of the non-"history_" fields are non-null, that means that that row contains
// a representation of that EppResource at that point in time. We use creation_time as
// a marker since it's the simplest field and all EppResources will have it.
return (boolean)
jpaTm()
.getEntityManager()
.createNativeQuery(
String.format(
"SELECT EXISTS (SELECT 1 FROM \"%s\" WHERE %s = :repoId AND"
+ " creation_time IS NOT NULL)",
historyClass.getSimpleName(), repoIdFieldName))
.setParameter("repoId", resource.getRepoId())
.getSingleResult();
});
}
// Lifted from HistoryEntryDao
private static Class<? extends HistoryEntry> getHistoryClassFromParent(
Class<? extends EppResource> parent) {
if (!RESOURCE_TYPES_TO_HISTORY_TYPES.containsKey(parent)) {
throw new IllegalArgumentException(
String.format("Unknown history type for parent %s", parent.getName()));
}
return RESOURCE_TYPES_TO_HISTORY_TYPES.get(parent);
}
// Lifted from HistoryEntryDao
private static String getRepoIdFieldNameFromHistoryClass(
Class<? extends HistoryEntry> historyClass) {
if (!REPO_ID_FIELD_NAMES.containsKey(historyClass)) {
throw new IllegalArgumentException(
String.format("Unknown history type %s", historyClass.getName()));
}
return REPO_ID_FIELD_NAMES.get(historyClass);
}
/** Mapper to re-save all EPP resources. */
public static class CreateSyntheticHistoryEntriesMapper
extends Mapper<Key<EppResource>, Void, Void> {
private final String registryAdminRegistrarId;
public CreateSyntheticHistoryEntriesMapper(String registryAdminRegistrarId) {
this.registryAdminRegistrarId = registryAdminRegistrarId;
}
@Override
public final void map(final Key<EppResource> resourceKey) {
EppResource eppResource = auditedOfy().load().key(resourceKey).now();
// Only save new history entries if the most recent history for this object in SQL does not
// have the resource at that point in time already
if (!hasHistoryContainingResource(eppResource)) {
ofyTm()
.transact(
() ->
ofyTm()
.put(
HistoryEntry.createBuilderForResource(eppResource)
.setRegistrarId(registryAdminRegistrarId)
.setBySuperuser(true)
.setRequestedByRegistrar(false)
.setModificationTime(ofyTm().getTransactionTime())
.setReason(HISTORY_REASON)
.setType(HistoryEntry.Type.SYNTHETIC)
.build()));
}
}
}
}
core/src/main/java/google/registry/tools/javascrap/CreateSyntheticHistoryEntriesPipeline.java
-135
View File
@@ -1,135 +0,0 @@
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.tools.javascrap;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import com.google.common.collect.ImmutableList;
import dagger.Component;
import google.registry.beam.common.RegistryJpaIO;
import google.registry.beam.common.RegistryPipelineOptions;
import google.registry.config.RegistryConfig.Config;
import google.registry.config.RegistryConfig.ConfigModule;
import google.registry.model.EppResource;
import google.registry.model.UpdateAutoTimestamp;
import google.registry.model.UpdateAutoTimestamp.DisableAutoUpdateResource;
import google.registry.model.contact.ContactResource;
import google.registry.model.domain.DomainBase;
import google.registry.model.host.HostResource;
import google.registry.model.reporting.HistoryEntry;
import google.registry.persistence.PersistenceModule.TransactionIsolationLevel;
import google.registry.persistence.VKey;
import java.io.Serializable;
import javax.inject.Singleton;
import javax.persistence.Entity;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.MapElements;
import org.apache.beam.sdk.values.TypeDescriptor;
/**
* Pipeline that creates a synthetic history entry for every {@link EppResource} in SQL at the
* current time.
*
* <p>The history entries in Datastore does not have the EPP resource embedded in them. Therefore
* after {@link google.registry.beam.initsql.InitSqlPipeline} runs, these fields will all be empty.
* This pipeline loads all EPP resources and for each of them creates a synthetic history entry that
* contains the resource and saves them back to SQL, so that they can be used in the RDE pipeline.
*
* <p>Note that this pipeline should only be run in a test environment right after the init SQL
* pipeline finishes, and no EPP update is being made to the system, otherwise there is no garuantee
* that the latest history entry for a given EPP resource does not already have the resource
* embedded within it.
*
* <p>To run the pipeline:
*
* <p><code>
* $ ./nom_build :core:cSHE --args="--region=us-central1
* --runner=DataflowRunner
* --registryEnvironment=CRASH
* --project={project-id}
* --workerMachineType=n2-standard-4"
* </code>
*
* @see google.registry.tools.javascrap.CreateSyntheticHistoryEntriesAction
*/
public class CreateSyntheticHistoryEntriesPipeline implements Serializable {
private static final ImmutableList<Class<? extends EppResource>> EPP_RESOURCE_CLASSES =
ImmutableList.of(DomainBase.class, ContactResource.class, HostResource.class);
private static final String HISTORY_REASON =
"Backfill EppResource history objects after initial backup to SQL";
static void setup(Pipeline pipeline, String registryAdminRegistrarId) {
for (Class<? extends EppResource> clazz : EPP_RESOURCE_CLASSES) {
pipeline
.apply(
String.format("Read all %s", clazz.getSimpleName()),
RegistryJpaIO.read(
"SELECT id FROM %entity%"
.replace("%entity%", clazz.getAnnotation(Entity.class).name()),
String.class,
repoId -> VKey.createSql(clazz, repoId)))
.apply(
String.format("Save a synthetic HistoryEntry for each %s", clazz),
MapElements.into(TypeDescriptor.of(Void.class))
.via(
(VKey<? extends EppResource> key) -> {
jpaTm()
.transact(
() -> {
EppResource eppResource = jpaTm().loadByKey(key);
try (DisableAutoUpdateResource disable =
UpdateAutoTimestamp.disableAutoUpdate()) {
jpaTm()
.put(
HistoryEntry.createBuilderForResource(eppResource)
.setRegistrarId(registryAdminRegistrarId)
.setBySuperuser(true)
.setRequestedByRegistrar(false)
.setModificationTime(jpaTm().getTransactionTime())
.setReason(HISTORY_REASON)
.setType(HistoryEntry.Type.SYNTHETIC)
.build());
}
});
return null;
}));
}
}
public static void main(String[] args) {
RegistryPipelineOptions options =
PipelineOptionsFactory.fromArgs(args).withValidation().as(RegistryPipelineOptions.class);
RegistryPipelineOptions.validateRegistryPipelineOptions(options);
options.setIsolationOverride(TransactionIsolationLevel.TRANSACTION_READ_COMMITTED);
String registryAdminRegistrarId =
DaggerCreateSyntheticHistoryEntriesPipeline_ConfigComponent.create()
.getRegistryAdminRegistrarId();
Pipeline pipeline = Pipeline.create(options);
setup(pipeline, registryAdminRegistrarId);
pipeline.run();
}
@Singleton
@Component(modules = ConfigModule.class)
interface ConfigComponent {
@Config("registryAdminClientId")
String getRegistryAdminRegistrarId();
}
}