mirrors/nomulus
1
0
Fork 0
mirror of https://github.com/google/nomulus synced 2026-08-15 11:46:08 +00:00
Code Issues Releases Wiki Activity

Sometimes include deletion times in domain-list exports (#2602)

Browse Source 
We only include the deletion time if the domain is in the 5-day
PENDING_DELETE period after the 30 day REDEMPTION period. For all other
domains, we just have an empty string as that field.

This is behind a feature flag so that we can control when it is enabled
This commit is contained in:
gbrodman
2024-11-06 17:59:30 +00:00
committed by GitHub
parent cc20f7d76d
commit ae61cd443d
5 changed files with 250 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/google/registry/export/ExportDomainListsAction.java
+77 -39
View File
@@ -17,6 +17,7 @@ package google.registry.export;
import static com.google.common.base.Verify.verifyNotNull;
import static google.registry.model.tld.Tlds.getTldsOfType;
import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ;
import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.request.Action.Method.POST;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -28,6 +29,9 @@ import com.google.common.flogger.FluentLogger;
import com.google.common.net.MediaType;
import google.registry.config.RegistryConfig.Config;
import google.registry.gcs.GcsUtils;
import google.registry.model.common.FeatureFlag;
import google.registry.model.domain.rgp.GracePeriodStatus;
import google.registry.model.eppcommon.StatusValue;
import google.registry.model.tld.Tld;
import google.registry.model.tld.Tld.TldType;
import google.registry.request.Action;
@@ -38,8 +42,13 @@ import google.registry.util.Clock;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.time.Instant;
import java.util.List;
import javax.inject.Inject;
import org.hibernate.query.NativeQuery;
import org.hibernate.query.TupleTransformer;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
/**
* An action that exports the list of active domains on all real TLDs to Google Drive and GCS.
@@ -55,7 +64,21 @@ import javax.inject.Inject;
public class ExportDomainListsAction implements Runnable {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
public static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
private static final String SELECT_DOMAINS_STATEMENT =
"SELECT domainName FROM Domain WHERE tld = :tld AND deletionTime > :now ORDER by domainName";
private static final String SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT =
"""
SELECT d.domain_name, d.deletion_time, d.statuses, gp.type FROM "Domain" d
LEFT JOIN (SELECT type, domain_repo_id FROM "GracePeriod"
WHERE type = 'REDEMPTION'
AND expiration_time > CAST(:now AS timestamptz)) AS gp
ON d.repo_id = gp.domain_repo_id
WHERE d.tld = :tld
AND d.deletion_time > CAST(:now AS timestamptz)
ORDER BY d.domain_name""";
// This may be a CSV, but it is uses a .txt file extension for back-compatibility
static final String REGISTERED_DOMAINS_FILENAME = "registered_domains.txt";
@Inject Clock clock;
@Inject DriveConnection driveConnection;
@@ -68,47 +91,50 @@ public class ExportDomainListsAction implements Runnable {
public void run() {
ImmutableSet<String> realTlds = getTldsOfType(TldType.REAL);
logger.atInfo().log("Exporting domain lists for TLDs %s.", realTlds);
boolean includeDeletionTimes =
tm().transact(
() ->
FeatureFlag.isActiveNowOrElse(
FeatureFlag.FeatureName.INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS, false));
realTlds.forEach(
tld -> {
List<String> domains =
tm().transact(
List<String> domainsList =
replicaTm()
.transact(
TRANSACTION_REPEATABLE_READ,
() ->
// Note that if we had "creationTime <= :now" in the condition (not
// necessary as there is no pending creation, the order of deletionTime
// and creationTime in the query would have been significant and it
// should come after deletionTime. When Hibernate substitutes "now" it
// will first validate that the **first** field that is to be compared
// with it (deletionTime) is assignable from the substituted Java object
// (click.nowUtc()). Since creationTime is a CreateAutoTimestamp, if it
// comes first, we will need to substitute "now" with
// CreateAutoTimestamp.create(clock.nowUtc()). This might look a bit
// strange as the Java object type is clearly incompatible between the
// two fields deletionTime (DateTime) and creationTime, yet they are
// compared with the same "now". It is actually OK because in the end
// Hibernate converts everything to SQL types (and Java field names to
// SQL column names) to run the query. Both CreateAutoTimestamp and
// DateTime are persisted as timestamp_z in SQL. It is only the
// validation that compares the Java types, and only with the first
// field that compares with the substituted value.
tm().query(
"SELECT domainName FROM Domain "
+ "WHERE tld = :tld "
+ "AND deletionTime > :now "
+ "ORDER by domainName ASC",
String.class)
() -> {
if (includeDeletionTimes) {
// We want to include deletion times, but only for domains in the 5-day
// PENDING_DELETE period after the REDEMPTION grace period. In order to
// accomplish this without loading the entire list of domains, we use a
// native query to join against the GracePeriod table to find
// PENDING_DELETE domains that don't have a REDEMPTION grace period.
return replicaTm()
.getEntityManager()
.createNativeQuery(SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT)
.unwrap(NativeQuery.class)
.setTupleTransformer(new DomainResultTransformer())
.setParameter("tld", tld)
.setParameter("now", clock.nowUtc())
.getResultList());
String domainsList = Joiner.on("\n").join(domains);
.setParameter("now", replicaTm().getTransactionTime().toString())
.getResultList();
} else {
return replicaTm()
.query(SELECT_DOMAINS_STATEMENT, String.class)
.setParameter("tld", tld)
.setParameter("now", replicaTm().getTransactionTime())
.getResultList();
}
});
logger.atInfo().log(
"Exporting %d domains for TLD %s to GCS and Drive.", domains.size(), tld);
exportToGcs(tld, domainsList, gcsBucket, gcsUtils);
exportToDrive(tld, domainsList, driveConnection);
"Exporting %d domains for TLD %s to GCS and Drive.", domainsList.size(), tld);
String domainsListOutput = Joiner.on('\n').join(domainsList);
exportToGcs(tld, domainsListOutput, gcsBucket, gcsUtils);
exportToDrive(tld, domainsListOutput, driveConnection);
});
}
protected static boolean exportToDrive(
protected static void exportToDrive(
String tldStr, String domains, DriveConnection driveConnection) {
verifyNotNull(driveConnection, "Expecting non-null driveConnection");
try {
@@ -131,12 +157,10 @@ public class ExportDomainListsAction implements Runnable {
} catch (Throwable e) {
logger.atSevere().withCause(e).log(
"Error exporting registered domains for TLD %s to Drive, skipping...", tldStr);
return false;
}
return true;
}
protected static boolean exportToGcs(
protected static void exportToGcs(
String tld, String domains, String gcsBucket, GcsUtils gcsUtils) {
BlobId blobId = BlobId.of(gcsBucket, tld + ".txt");
try (OutputStream gcsOutput = gcsUtils.openOutputStream(blobId);
@@ -145,8 +169,22 @@ public class ExportDomainListsAction implements Runnable {
} catch (Throwable e) {
logger.atSevere().withCause(e).log(
"Error exporting registered domains for TLD %s to GCS, skipping...", tld);
return false;
}
return true;
}
/** Transforms the multiple columns selected from SQL into the output line. */
private static class DomainResultTransformer implements TupleTransformer<String> {
@Override
public String transformTuple(Object[] domainResult, String[] strings) {
String domainName = (String) domainResult[0];
Instant deletionInstant = (Instant) domainResult[1];
DateTime deletionTime = new DateTime(deletionInstant.toEpochMilli(), DateTimeZone.UTC);
String[] domainStatuses = (String[]) domainResult[2];
String gracePeriodType = (String) domainResult[3];
boolean inPendingDelete =
ImmutableSet.copyOf(domainStatuses).contains(StatusValue.PENDING_DELETE.toString())
&& !GracePeriodStatus.REDEMPTION.toString().equals(gracePeriodType);
return String.format("%s,%s", domainName, inPendingDelete ? deletionTime : "");
}
}
}
core/src/main/java/google/registry/model/common/FeatureFlag.java
+10
View File
@@ -66,6 +66,7 @@ public class FeatureFlag extends ImmutableObject implements Buildable {
TEST_FEATURE,
MINIMUM_DATASET_CONTACTS_OPTIONAL,
MINIMUM_DATASET_CONTACTS_PROHIBITED,
INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS
}
/** The name of the flag/feature. */
@@ -154,6 +155,15 @@ public class FeatureFlag extends ImmutableObject implements Buildable {
return status.getValueAtTime(time);
}
/** Returns if the flag is active, or the default value if the flag does not exist. */
public static boolean isActiveNowOrElse(FeatureName featureName, boolean defaultValue) {
tm().assertInTransaction();
return CACHE
.get(featureName)
.map(flag -> flag.getStatus(tm().getTransactionTime()).equals(ACTIVE))
.orElse(defaultValue);
}
/** Returns if the FeatureFlag with the given FeatureName is active now. */
public static boolean isActiveNow(FeatureName featureName) {
tm().assertInTransaction();