Make RefreshDnsForAllDomains SQL-aware (#1197)

Also marks a few mapreduce actions as @Deprecated as they are no longer
needed in SQL.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/google/nomulus/1197)
<!-- Reviewable:end -->
This commit is contained in:
Lai Jiang
2021-06-10 21:09:19 -04:00
committed by GitHub
parent c7096a1b71
commit a7210a26b4
6 changed files with 103 additions and 27 deletions
@@ -66,6 +66,8 @@ import org.joda.time.Duration;
service = Action.Service.BACKEND,
path = "/_dr/task/deleteOldCommitLogs",
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
// No longer needed in SQL. Subject to future removal.
@Deprecated
public final class DeleteOldCommitLogsAction implements Runnable {
private static final int NUM_MAP_SHARDS = 20;
@@ -54,6 +54,8 @@ import javax.inject.Inject;
service = Action.Service.BACKEND,
path = "/_dr/task/resaveAllEppResources",
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
// No longer needed in SQL. Subject to future removal.
@Deprecated
public class ResaveAllEppResourcesAction implements Runnable {
@Inject MapreduceRunner mrRunner;
@@ -49,6 +49,8 @@ import javax.inject.Inject;
path = "/_dr/task/killAllCommitLogs",
method = POST,
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
// No longer needed in SQL. Subject to future removal.
@Deprecated
public class KillAllCommitLogsAction implements Runnable {
@Inject MapreduceRunner mrRunner;
@@ -18,6 +18,8 @@ import static com.google.common.base.Preconditions.checkArgument;
import static google.registry.mapreduce.inputs.EppResourceInputs.createEntityInput;
import static google.registry.model.EppResourceUtils.isActive;
import static google.registry.model.registry.Registries.assertTldsExist;
import static google.registry.persistence.transaction.TransactionManagerFactory.jpaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.request.RequestParameters.PARAM_TLDS;
import com.google.appengine.tools.mapreduce.Mapper;
@@ -32,11 +34,12 @@ import google.registry.request.Action;
import google.registry.request.Parameter;
import google.registry.request.Response;
import google.registry.request.auth.Auth;
import google.registry.util.Clock;
import google.registry.util.NonFinalForTesting;
import java.util.Random;
import javax.inject.Inject;
import org.apache.http.HttpStatus;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Duration;
/**
@@ -74,6 +77,8 @@ public class RefreshDnsForAllDomainsAction implements Runnable {
@Parameter("smearMinutes")
int smearMinutes;
@Inject DnsQueue dnsQueue;
@Inject Clock clock;
@Inject Random random;
@Inject
@@ -83,14 +88,41 @@ public class RefreshDnsForAllDomainsAction implements Runnable {
public void run() {
assertTldsExist(tlds);
checkArgument(smearMinutes > 0, "Must specify a positive number of smear minutes");
mrRunner
.setJobName("Refresh DNS for all domains")
.setModuleName("tools")
.setDefaultMapShards(10)
.runMapOnly(
new RefreshDnsForAllDomainsActionMapper(tlds, smearMinutes, random),
ImmutableList.of(createEntityInput(DomainBase.class)))
.sendLinkToMapreduceConsole(response);
if (tm().isOfy()) {
mrRunner
.setJobName("Refresh DNS for all domains")
.setModuleName("tools")
.setDefaultMapShards(10)
.runMapOnly(
new RefreshDnsForAllDomainsActionMapper(tlds, smearMinutes, random, clock.nowUtc()),
ImmutableList.of(createEntityInput(DomainBase.class)))
.sendLinkToMapreduceConsole(response);
} else {
tm().transact(
() ->
jpaTm()
.query(
"SELECT fullyQualifiedDomainName FROM Domain "
+ "WHERE tld IN (:tlds) "
+ "AND deletionTime > :now",
String.class)
.setParameter("tlds", tlds)
.setParameter("now", clock.nowUtc())
.getResultStream()
.forEach(
domainName -> {
try {
// Smear the task execution time over the next N minutes.
dnsQueue.addDomainRefreshTask(
domainName,
Duration.standardMinutes(random.nextInt(smearMinutes)));
} catch (Throwable t) {
logger.atSevere().withCause(t).log(
"Error while enqueuing DNS refresh for domain %s", domainName);
response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR);
}
}));
}
}
/** Mapper to refresh DNS for all active domain resources. */
@@ -104,19 +136,21 @@ public class RefreshDnsForAllDomainsAction implements Runnable {
private final ImmutableSet<String> tlds;
private final int smearMinutes;
private final Random random;
private final DateTime now;
RefreshDnsForAllDomainsActionMapper(
ImmutableSet<String> tlds, int smearMinutes, Random random) {
ImmutableSet<String> tlds, int smearMinutes, Random random, DateTime now) {
this.tlds = tlds;
this.smearMinutes = smearMinutes;
this.random = random;
this.now = now;
}
@Override
public void map(final DomainBase domain) {
String domainName = domain.getDomainName();
if (tlds.contains(domain.getTld())) {
if (isActive(domain, DateTime.now(DateTimeZone.UTC))) {
if (isActive(domain, now)) {
try {
// Smear the task execution time over the next N minutes.
dnsQueue.addDomainRefreshTask(
@@ -124,7 +158,7 @@ public class RefreshDnsForAllDomainsAction implements Runnable {
getContext().incrementCounter("active domains refreshed");
} catch (Throwable t) {
logger.atSevere().withCause(t).log(
"Error while refreshing DNS for domain %s", domainName);
"Error while enqueuing DNS refresh for domain %s", domainName);
getContext().incrementCounter("active domains errored");
}
} else {
@@ -42,6 +42,8 @@ import javax.inject.Inject;
service = Action.Service.TOOLS,
path = "/_dr/task/resaveAllHistoryEntries",
auth = Auth.AUTH_INTERNAL_OR_ADMIN)
// No longer needed in SQL. Subject to future removal.
@Deprecated
public class ResaveAllHistoryEntriesAction implements Runnable {
@Inject MapreduceRunner mrRunner;