From a5c84aa4e0cbadc552202d2ae9d49b1cf514fcb1 Mon Sep 17 00:00:00 2001 From: Ben McIlwain Date: Mon, 31 Aug 2026 19:58:42 +0000 Subject: [PATCH] Export domain drop list for open TLDs to Drive (#3223) Redo the exported drop list mechanism by replacing the legacy per-TLD mode in ExportDomainListsAction with a dedicated, once-daily ExportDropListAction. Key changes: 1. Reverted ExportDomainListsAction to unconditionally export single-column active registered domains and deprecated the INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS feature flag. 2. Implemented ExportDropListAction at /_dr/task/exportDropList to query the read replica for upcoming deletions on all open TLDs (where invoicing is enabled) and output an alphabetically sorted CSV file (domain_name,tld,deletion_time) to a designated Google Drive folder. 3. Added domainDropListDriveFolderId configuration setting and provider. 4. Registered the action in RequestComponent, routing.txt, and Cloud Scheduler tasks for production and sandbox. 5. Added comprehensive test coverage in ExportDropListActionTest and cleaned up legacy test cases in ExportDomainListsActionTest. BUG=b/553658111 --- .../registry/config/RegistryConfig.java | 13 ++ .../config/RegistryConfigSettings.java | 1 + .../registry/config/files/default-config.yaml | 3 + .../cloud-scheduler-tasks-production.xml | 9 + .../tasks/cloud-scheduler-tasks-sandbox.xml | 9 + .../export/ExportDomainListsAction.java | 62 +----- .../registry/export/ExportDropListAction.java | 153 ++++++++++++++ .../registry/model/common/FeatureFlag.java | 4 + .../registry/module/RequestComponent.java | 3 + .../export/ExportDomainListsActionTest.java | 116 +--------- .../export/ExportDropListActionTest.java | 199 ++++++++++++++++++ .../google/registry/module/routing.txt | 1 + 12 files changed, 401 insertions(+), 172 deletions(-) create mode 100644 core/src/main/java/google/registry/export/ExportDropListAction.java create mode 100644 core/src/test/java/google/registry/export/ExportDropListActionTest.java diff --git a/core/src/main/java/google/registry/config/RegistryConfig.java b/core/src/main/java/google/registry/config/RegistryConfig.java index b4f963825..0affb980f 100644 --- a/core/src/main/java/google/registry/config/RegistryConfig.java +++ b/core/src/main/java/google/registry/config/RegistryConfig.java @@ -36,6 +36,7 @@ import dagger.Module; import dagger.Provides; import google.registry.bsa.UploadBsaUnavailableDomainsAction; import google.registry.dns.ReadDnsRefreshRequestsAction; +import google.registry.export.ExportDropListAction; import google.registry.model.common.DnsRefreshRequest; import google.registry.mosapi.MosApiClient; import google.registry.persistence.transaction.JpaTransactionManager; @@ -1159,6 +1160,18 @@ public final class RegistryConfig { .collect(toImmutableMap(entry -> CurrencyUnit.of(entry.getKey()), Entry::getValue)); } + /** + * Returns the Google Drive folder ID for exporting the domain drop list. + * + * @see ExportDropListAction + */ + @Provides + @Config("domainDropListDriveFolderId") + public static Optional provideDomainDropListDriveFolderId( + RegistryConfigSettings config) { + return Optional.ofNullable(config.registryPolicy.domainDropListDriveFolderId); + } + @Singleton @Provides static RegistryConfigSettings provideRegistryConfigSettings() { diff --git a/core/src/main/java/google/registry/config/RegistryConfigSettings.java b/core/src/main/java/google/registry/config/RegistryConfigSettings.java index 8c7ba2475..68e2e1675 100644 --- a/core/src/main/java/google/registry/config/RegistryConfigSettings.java +++ b/core/src/main/java/google/registry/config/RegistryConfigSettings.java @@ -97,6 +97,7 @@ public class RegistryConfigSettings { public String tmchMarksDbUrl; public String registryAdminClientId; public DomainExpiryAccessPeriod domainExpiryAccessPeriod; + public String domainDropListDriveFolderId; public String premiumTermsExportDisclaimer; public String reservedTermsExportDisclaimer; public String rdapTos; diff --git a/core/src/main/java/google/registry/config/files/default-config.yaml b/core/src/main/java/google/registry/config/files/default-config.yaml index 370a68eee..9b903f022 100644 --- a/core/src/main/java/google/registry/config/files/default-config.yaml +++ b/core/src/main/java/google/registry/config/files/default-config.yaml @@ -110,6 +110,9 @@ registryPolicy: USD: 10 JPY: 1000 + # Google Drive folder ID for exporting the domain drop list. + domainDropListDriveFolderId: + # Disclaimer at the top of the exported premium terms list. premiumTermsExportDisclaimer: | This list contains domains for the TLD offered at a premium price. This diff --git a/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-production.xml b/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-production.xml index 020e4c3c6..72a27387a 100644 --- a/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-production.xml +++ b/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-production.xml @@ -121,6 +121,15 @@ 0 */12 * * * + + + exportDropList + + This job exports the upcoming domain drop list for all open TLDs to Google Drive once daily. + + 0 6 * * * + + expandBillingRecurrences diff --git a/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-sandbox.xml b/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-sandbox.xml index 79bc60224..23eca33ff 100644 --- a/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-sandbox.xml +++ b/core/src/main/java/google/registry/config/files/tasks/cloud-scheduler-tasks-sandbox.xml @@ -81,6 +81,15 @@ 0 */12 * * * + + + exportDropList + + This job exports the upcoming domain drop list for all open TLDs to Google Drive once daily. + + 0 6 * * * + + expandBillingRecurrences diff --git a/core/src/main/java/google/registry/export/ExportDomainListsAction.java b/core/src/main/java/google/registry/export/ExportDomainListsAction.java index 46250a5e0..cef0af6e8 100644 --- a/core/src/main/java/google/registry/export/ExportDomainListsAction.java +++ b/core/src/main/java/google/registry/export/ExportDomainListsAction.java @@ -15,11 +15,9 @@ package google.registry.export; import static com.google.common.base.Verify.verifyNotNull; -import static google.registry.model.common.FeatureFlag.FeatureName.INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS; import static google.registry.model.tld.Tlds.getTldsOfType; import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ; import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm; -import static google.registry.persistence.transaction.TransactionManagerFactory.tm; import static google.registry.request.Action.Method.POST; import static java.nio.charset.StandardCharsets.UTF_8; @@ -30,23 +28,16 @@ import com.google.common.flogger.FluentLogger; import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.gcs.GcsUtils; -import google.registry.model.common.FeatureFlag; -import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.eppcommon.StatusValue; import google.registry.model.tld.Tld; import google.registry.model.tld.Tld.TldType; import google.registry.request.Action; import google.registry.request.auth.Auth; import google.registry.storage.drive.DriveConnection; -import google.registry.util.Clock; import jakarta.inject.Inject; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; -import java.time.Instant; import java.util.List; -import org.hibernate.query.NativeQuery; -import org.hibernate.query.TupleTransformer; /** * An action that exports the list of active domains on all real TLDs to Google Drive and GCS. @@ -64,22 +55,9 @@ public class ExportDomainListsAction implements Runnable { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final String SELECT_DOMAINS_STATEMENT = "SELECT domainName FROM Domain WHERE tld = :tld AND deletionTime > :now ORDER by domainName"; - private static final String SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT = - """ - SELECT d.domain_name, d.deletion_time, d.statuses, gp.type FROM "Domain" d - LEFT JOIN (SELECT type, domain_repo_id FROM "GracePeriod" - WHERE type = 'REDEMPTION' - AND expiration_time > CAST(:now AS timestamptz)) AS gp - ON d.repo_id = gp.domain_repo_id - WHERE d.tld = :tld - AND d.deletion_time > CAST(:now AS timestamptz) - ORDER BY d.domain_name - """; - // This may be a CSV, but it is uses a .txt file extension for back-compatibility static final String REGISTERED_DOMAINS_FILENAME_FORMAT = "registered_domains_%s.txt"; - @Inject Clock clock; @Inject DriveConnection driveConnection; @Inject GcsUtils gcsUtils; @@ -91,37 +69,18 @@ public class ExportDomainListsAction implements Runnable { ImmutableSet realTlds = getTldsOfType(TldType.REAL); logger.atInfo().log("Exporting domain lists for TLDs %s.", realTlds); - boolean includeDeletionTimes = - tm().transact(() -> FeatureFlag.isActiveNow(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS)); realTlds.forEach( tld -> { List domainsList = replicaTm() .transact( TRANSACTION_REPEATABLE_READ, - () -> { - if (includeDeletionTimes) { - // We want to include deletion times, but only for domains in the 5-day - // PENDING_DELETE period after the REDEMPTION grace period. In order to - // accomplish this without loading the entire list of domains, we use a - // native query to join against the GracePeriod table to find - // PENDING_DELETE domains that don't have a REDEMPTION grace period. - return replicaTm() - .getEntityManager() - .createNativeQuery(SELECT_DOMAINS_AND_DELETION_TIMES_STATEMENT) - .unwrap(NativeQuery.class) - .setTupleTransformer(new DomainResultTransformer()) - .setParameter("tld", tld) - .setParameter("now", replicaTm().getTxTime().toString()) - .getResultList(); - } else { - return replicaTm() + () -> + replicaTm() .query(SELECT_DOMAINS_STATEMENT, String.class) .setParameter("tld", tld) .setParameter("now", replicaTm().getTxTime()) - .getResultList(); - } - }); + .getResultList()); logger.atInfo().log( "Exporting %d domains for TLD %s to GCS and Drive.", domainsList.size(), tld); String domainsListOutput = Joiner.on('\n').join(domainsList); @@ -167,19 +126,4 @@ public class ExportDomainListsAction implements Runnable { "Error exporting registered domains for TLD %s to GCS, skipping...", tld); } } - - /** Transforms the multiple columns selected from SQL into the output line. */ - private static class DomainResultTransformer implements TupleTransformer { - @Override - public String transformTuple(Object[] domainResult, String[] strings) { - String domainName = (String) domainResult[0]; - Instant deletionTime = (Instant) domainResult[1]; - String[] domainStatuses = (String[]) domainResult[2]; - String gracePeriodType = (String) domainResult[3]; - boolean inPendingDelete = - ImmutableSet.copyOf(domainStatuses).contains(StatusValue.PENDING_DELETE.toString()) - && !GracePeriodStatus.REDEMPTION.toString().equals(gracePeriodType); - return String.format("%s,%s", domainName, inPendingDelete ? deletionTime : ""); - } - } } diff --git a/core/src/main/java/google/registry/export/ExportDropListAction.java b/core/src/main/java/google/registry/export/ExportDropListAction.java new file mode 100644 index 000000000..d33ffb535 --- /dev/null +++ b/core/src/main/java/google/registry/export/ExportDropListAction.java @@ -0,0 +1,153 @@ +// Copyright 2026 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.base.Verify.verifyNotNull; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static google.registry.model.tld.Tlds.getTldEntitiesOfType; +import static google.registry.persistence.PersistenceModule.TransactionIsolationLevel.TRANSACTION_REPEATABLE_READ; +import static google.registry.persistence.transaction.TransactionManagerFactory.replicaTm; +import static google.registry.request.Action.Method.POST; +import static google.registry.util.DateTimeUtils.END_INSTANT; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import com.google.common.net.MediaType; +import google.registry.config.RegistryConfig.Config; +import google.registry.model.tld.Tld; +import google.registry.model.tld.Tld.ExpiryAccessPeriodMode; +import google.registry.model.tld.Tld.TldType; +import google.registry.request.Action; +import google.registry.request.auth.Auth; +import google.registry.storage.drive.DriveConnection; +import google.registry.util.Clock; +import jakarta.inject.Inject; +import java.io.IOException; +import java.io.StringWriter; +import java.io.UncheckedIOException; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; + +/** An action that exports the upcoming domain drop list across all open TLDs to Google Drive. */ +@Action( + service = Action.Service.BACKEND, + path = "/_dr/task/exportDropList", + method = POST, + auth = Auth.AUTH_ADMIN) +public class ExportDropListAction implements Runnable { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + static final String DROPLIST_FILENAME = "domain_drop_list.csv"; + static final ImmutableList CSV_HEADER = + ImmutableList.of("domain_name", "tld", "deletion_time"); + private static final CSVFormat CSV_FORMAT = + CSVFormat.DEFAULT.builder().setRecordSeparator('\n').get(); + + private static final String SELECT_UPCOMING_DELETIONS_STATEMENT = + """ + SELECT domainName, tld, deletionTime FROM Domain + WHERE tld IN :xapTlds + AND deletionTime > :now + AND deletionTime < :endOfTime + ORDER BY domainName + """; + + @Inject Clock clock; + @Inject DriveConnection driveConnection; + + @Inject + @Config("domainDropListDriveFolderId") + Optional driveFolderId; + + @Inject + ExportDropListAction() {} + + @Override + public void run() { + if (driveFolderId.isEmpty() || driveFolderId.get().isEmpty()) { + logger.atInfo().log("Skipping domain drop list export because Drive folder isn't specified."); + return; + } + + Instant now = clock.now(); + ImmutableSet xapTlds = + getTldEntitiesOfType(TldType.REAL).stream() + .filter(Tld::isInvoicingEnabled) + .filter(tld -> tld.getExpiryAccessPeriodModeAt(now) == ExpiryAccessPeriodMode.ENABLED) + .map(Tld::getTldStr) + .collect(toImmutableSet()); + logger.atInfo().log("Exporting domain drop list for open TLDs with XAP enabled: %s", xapTlds); + + if (xapTlds.isEmpty()) { + logger.atInfo().log("No open TLDs found with XAP enabled."); + exportToDrive(createCsv(ImmutableList.of())); + return; + } + + // Any database transaction failures will throw an unchecked PersistenceException, causing + // RequestHandler to set HTTP 500 so Cloud Tasks can retry the task on retryable-cron-tasks. + List queryResults = + replicaTm() + .transact( + TRANSACTION_REPEATABLE_READ, + () -> + replicaTm() + .query(SELECT_UPCOMING_DELETIONS_STATEMENT, Object[].class) + .setParameter("xapTlds", xapTlds) + .setParameter("now", now) + .setParameter("endOfTime", END_INSTANT) + .getResultList()); + + exportToDrive(createCsv(queryResults)); + } + + private static String createCsv(List queryResults) { + StringWriter stringWriter = new StringWriter(); + try (CSVPrinter printer = new CSVPrinter(stringWriter, CSV_FORMAT)) { + printer.printRecord(CSV_HEADER); + for (Object[] row : queryResults) { + printer.printRecord(row); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to create drop list CSV", e); + } + return stringWriter.toString(); + } + + private void exportToDrive(String csvContent) { + verifyNotNull(driveConnection, "Expecting non-null driveConnection"); + try { + String resultMsg = + driveConnection.createOrUpdateFile( + DROPLIST_FILENAME, + MediaType.CSV_UTF_8, + driveFolderId.get(), + csvContent.getBytes(UTF_8)); + logger.atInfo().log("Exporting domain drop list succeeded, response was: %s", resultMsg); + } catch (IOException e) { + // Rethrow as an unchecked exception so RequestHandler sets HTTP 500, causing Cloud Tasks to + // retry this action on the retryable-cron-tasks queue upon transient Drive failures. + throw new RuntimeException( + String.format("Error exporting domain drop list to Drive folder %s", driveFolderId.get()), + e); + } + } +} diff --git a/core/src/main/java/google/registry/model/common/FeatureFlag.java b/core/src/main/java/google/registry/model/common/FeatureFlag.java index f3f30f2f7..eb6e1ec79 100644 --- a/core/src/main/java/google/registry/model/common/FeatureFlag.java +++ b/core/src/main/java/google/registry/model/common/FeatureFlag.java @@ -80,7 +80,11 @@ public class FeatureFlag extends ImmutableObject implements Buildable { /** * If we're including the upcoming domain drop date in the exported list of registered domains. + * + * @deprecated Legacy drop list mode in {@code ExportDomainListsAction} is removed; upcoming + * drops are exported via {@code ExportDropListAction}. */ + @Deprecated INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS(FeatureStatus.INACTIVE), /** If we're prohibiting the inclusion of the contact object URI on login. */ diff --git a/core/src/main/java/google/registry/module/RequestComponent.java b/core/src/main/java/google/registry/module/RequestComponent.java index 7708ded42..4f7fa7a78 100644 --- a/core/src/main/java/google/registry/module/RequestComponent.java +++ b/core/src/main/java/google/registry/module/RequestComponent.java @@ -42,6 +42,7 @@ import google.registry.dns.RefreshDnsOnHostRenameAction; import google.registry.dns.writer.DnsWritersModule; import google.registry.dns.writer.dnsupdate.DnsUpdateConfigModule; import google.registry.export.ExportDomainListsAction; +import google.registry.export.ExportDropListAction; import google.registry.export.ExportPremiumTermsAction; import google.registry.export.ExportReservedTermsAction; import google.registry.export.SyncGroupMembersAction; @@ -225,6 +226,8 @@ public interface RequestComponent { ExportDomainListsAction exportDomainListsAction(); + ExportDropListAction exportDropListAction(); + ExportPremiumTermsAction exportPremiumTermsAction(); ExportReservedTermsAction exportReservedTermsAction(); diff --git a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java index 193aeae72..1131d6db9 100644 --- a/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java +++ b/core/src/test/java/google/registry/export/ExportDomainListsActionTest.java @@ -15,15 +15,10 @@ package google.registry.export; import static com.google.common.truth.Truth.assertThat; -import static google.registry.model.common.FeatureFlag.FeatureName.INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS; -import static google.registry.model.common.FeatureFlag.FeatureStatus.ACTIVE; -import static google.registry.model.common.FeatureFlag.FeatureStatus.INACTIVE; import static google.registry.testing.DatabaseHelper.createTld; import static google.registry.testing.DatabaseHelper.persistActiveDomain; import static google.registry.testing.DatabaseHelper.persistDeletedDomain; -import static google.registry.testing.DatabaseHelper.persistFeatureFlag; import static google.registry.testing.DatabaseHelper.persistResource; -import static google.registry.util.DateTimeUtils.plusDays; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.eq; @@ -37,10 +32,6 @@ import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; import com.google.common.collect.ImmutableList; import com.google.common.net.MediaType; import google.registry.gcs.GcsUtils; -import google.registry.model.domain.Domain; -import google.registry.model.domain.GracePeriod; -import google.registry.model.domain.rgp.GracePeriodStatus; -import google.registry.model.eppcommon.StatusValue; import google.registry.model.tld.Tld; import google.registry.model.tld.Tld.TldType; import google.registry.persistence.transaction.JpaTestExtensions; @@ -76,9 +67,7 @@ class ExportDomainListsActionTest { action = new ExportDomainListsAction(); action.gcsBucket = "outputbucket"; action.gcsUtils = gcsUtils; - action.clock = clock; action.driveConnection = driveConnection; - persistFeatureFlag(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS, INACTIVE); } private void verifyExportedToDrive(String folderId, String filename, String domains) @@ -93,7 +82,7 @@ class ExportDomainListsActionTest { } @Test - void test_outputsOnlyActiveDomains_txt() throws Exception { + void test_outputsOnlyActiveDomains() throws Exception { persistActiveDomain("onetwo.tld"); persistActiveDomain("rudnitzky.tld"); persistDeletedDomain("mortuary.tld", Instant.parse("2001-03-14T10:11:12Z")); @@ -107,22 +96,7 @@ class ExportDomainListsActionTest { } @Test - void test_outputsOnlyActiveDomains_csv() throws Exception { - persistFeatureFlag(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS, ACTIVE); - persistActiveDomain("onetwo.tld"); - persistActiveDomain("rudnitzky.tld"); - persistDeletedDomain("mortuary.tld", Instant.parse("2001-03-14T10:11:12Z")); - action.run(); - BlobId existingFile = BlobId.of("outputbucket", "tld.txt"); - String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8); - // Check that it only contains the active domains, not the dead one. - assertThat(tlds).isEqualTo("onetwo.tld,\nrudnitzky.tld,"); - verifyExportedToDrive("brouhaha", "registered_domains_tld.txt", "onetwo.tld,\nrudnitzky.tld,"); - verifyNoMoreInteractions(driveConnection); - } - - @Test - void test_outputsOnlyDomainsOnRealTlds_txt() throws Exception { + void test_outputsOnlyDomainsOnRealTlds() throws Exception { persistActiveDomain("onetwo.tld"); persistActiveDomain("rudnitzky.tld"); persistActiveDomain("wontgo.testtld"); @@ -141,60 +115,7 @@ class ExportDomainListsActionTest { } @Test - void test_outputsOnlyDomainsOnRealTlds_csv() throws Exception { - persistFeatureFlag(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS, ACTIVE); - persistActiveDomain("onetwo.tld"); - persistActiveDomain("rudnitzky.tld"); - persistActiveDomain("wontgo.testtld"); - action.run(); - BlobId existingFile = BlobId.of("outputbucket", "tld.txt"); - String tlds = new String(gcsUtils.readBytesFrom(existingFile), UTF_8).trim(); - // Check that it only contains the domains on the real TLD, and not the test one. - assertThat(tlds).isEqualTo("onetwo.tld,\nrudnitzky.tld,"); - // Make sure that the test TLD file wasn't written out. - BlobId nonexistentFile = BlobId.of("outputbucket", "testtld.txt"); - assertThrows(StorageException.class, () -> gcsUtils.readBytesFrom(nonexistentFile)); - ImmutableList ls = gcsUtils.listFolderObjects("outputbucket", ""); - assertThat(ls).containsExactly("tld.txt"); - verifyExportedToDrive("brouhaha", "registered_domains_tld.txt", "onetwo.tld,\nrudnitzky.tld,"); - verifyNoMoreInteractions(driveConnection); - } - - @Test - void test_outputIncludesDeletionTimes_forPendingDeletes_notRdemption() throws Exception { - persistFeatureFlag(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS, ACTIVE); - // Domains pending delete (meaning the 5 day period, not counting the 30 day redemption period) - // should include their pending deletion date - persistActiveDomain("active.tld"); - Domain redemption = persistActiveDomain("redemption.tld"); - persistResource( - redemption - .asBuilder() - .addStatusValue(StatusValue.PENDING_DELETE) - .addGracePeriod( - GracePeriod.createWithoutBillingEvent( - GracePeriodStatus.REDEMPTION, - redemption.getRepoId(), - plusDays(clock.now(), 20), - redemption.getCurrentSponsorRegistrarId())) - .build()); - persistResource( - persistActiveDomain("pendingdelete.tld") - .asBuilder() - .addStatusValue(StatusValue.PENDING_DELETE) - .setDeletionTime(plusDays(clock.now(), 3)) - .build()); - - action.run(); - - verifyExportedToDrive( - "brouhaha", - "registered_domains_tld.txt", - "active.tld,\npendingdelete.tld,2020-02-05T02:02:02Z\nredemption.tld,"); - } - - @Test - void test_outputsDomainsFromDifferentTldsToMultipleFiles_txt() throws Exception { + void test_outputsDomainsFromDifferentTldsToMultipleFiles() throws Exception { createTld("tldtwo"); persistResource(Tld.get("tldtwo").asBuilder().setDriveFolderId("hooray").build()); @@ -222,35 +143,4 @@ class ExportDomainListsActionTest { // tldthree does not have a drive id, so no export to drive is performed. verifyNoMoreInteractions(driveConnection); } - - @Test - void test_outputsDomainsFromDifferentTldsToMultipleFiles_csv() throws Exception { - persistFeatureFlag(INCLUDE_PENDING_DELETE_DATE_FOR_DOMAINS, ACTIVE); - createTld("tldtwo"); - persistResource(Tld.get("tldtwo").asBuilder().setDriveFolderId("hooray").build()); - - createTld("tldthree"); - // You'd think this test was written around Christmas, but it wasn't. - persistActiveDomain("dasher.tld"); - persistActiveDomain("prancer.tld"); - persistActiveDomain("rudolph.tldtwo"); - persistActiveDomain("santa.tldtwo"); - persistActiveDomain("buddy.tldtwo"); - persistActiveDomain("cupid.tldthree"); - action.run(); - BlobId firstTldFile = BlobId.of("outputbucket", "tld.txt"); - String tlds = new String(gcsUtils.readBytesFrom(firstTldFile), UTF_8).trim(); - assertThat(tlds).isEqualTo("dasher.tld,\nprancer.tld,"); - BlobId secondTldFile = BlobId.of("outputbucket", "tldtwo.txt"); - String moreTlds = new String(gcsUtils.readBytesFrom(secondTldFile), UTF_8).trim(); - assertThat(moreTlds).isEqualTo("buddy.tldtwo,\nrudolph.tldtwo,\nsanta.tldtwo,"); - BlobId thirdTldFile = BlobId.of("outputbucket", "tldthree.txt"); - String evenMoreTlds = new String(gcsUtils.readBytesFrom(thirdTldFile), UTF_8).trim(); - assertThat(evenMoreTlds).isEqualTo("cupid.tldthree,"); - verifyExportedToDrive("brouhaha", "registered_domains_tld.txt", "dasher.tld,\nprancer.tld,"); - verifyExportedToDrive( - "hooray", "registered_domains_tldtwo.txt", "buddy.tldtwo,\nrudolph.tldtwo,\nsanta.tldtwo,"); - // tldthree does not have a drive id, so no export to drive is performed. - verifyNoMoreInteractions(driveConnection); - } } diff --git a/core/src/test/java/google/registry/export/ExportDropListActionTest.java b/core/src/test/java/google/registry/export/ExportDropListActionTest.java new file mode 100644 index 000000000..8612e5cfe --- /dev/null +++ b/core/src/test/java/google/registry/export/ExportDropListActionTest.java @@ -0,0 +1,199 @@ +// Copyright 2026 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.export; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.DatabaseHelper.createTld; +import static google.registry.testing.DatabaseHelper.persistActiveDomain; +import static google.registry.testing.DatabaseHelper.persistDeletedDomain; +import static google.registry.testing.DatabaseHelper.persistResource; +import static google.registry.util.DateTimeUtils.START_INSTANT; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.net.MediaType; +import google.registry.model.tld.Tld; +import google.registry.model.tld.Tld.ExpiryAccessPeriodMode; +import google.registry.model.tld.Tld.TldType; +import google.registry.persistence.transaction.JpaTestExtensions; +import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; +import google.registry.storage.drive.DriveConnection; +import google.registry.testing.FakeClock; +import java.io.IOException; +import java.time.Instant; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; + +/** Unit tests for {@link ExportDropListAction}. */ +class ExportDropListActionTest { + + private final DriveConnection driveConnection = mock(DriveConnection.class); + private final ArgumentCaptor bytesExportedToDrive = ArgumentCaptor.forClass(byte[].class); + private ExportDropListAction action; + private final FakeClock clock = new FakeClock(Instant.parse("2020-02-02T02:02:02Z")); + + @RegisterExtension + final JpaIntegrationTestExtension jpa = + new JpaTestExtensions.Builder().withClock(clock).buildIntegrationTestExtension(); + + @BeforeEach + void beforeEach() { + createTld("open1"); + persistResource( + Tld.get("open1") + .asBuilder() + .setInvoicingEnabled(true) + .setExpiryAccessPeriodTransitions( + ImmutableSortedMap.of(START_INSTANT, ExpiryAccessPeriodMode.ENABLED)) + .build()); + + createTld("open2"); + persistResource( + Tld.get("open2") + .asBuilder() + .setInvoicingEnabled(true) + .setExpiryAccessPeriodTransitions( + ImmutableSortedMap.of(START_INSTANT, ExpiryAccessPeriodMode.ENABLED)) + .build()); + + createTld("closed"); + persistResource(Tld.get("closed").asBuilder().setInvoicingEnabled(false).build()); + + createTld("testtld"); + persistResource( + Tld.get("testtld").asBuilder().setTldType(TldType.TEST).setInvoicingEnabled(true).build()); + + action = new ExportDropListAction(); + action.clock = clock; + action.driveConnection = driveConnection; + action.driveFolderId = Optional.of("drop_list_folder_id"); + } + + private void verifyExportedToDrive(String expectedCsv) throws Exception { + verify(driveConnection) + .createOrUpdateFile( + eq("domain_drop_list.csv"), + eq(MediaType.CSV_UTF_8), + eq("drop_list_folder_id"), + bytesExportedToDrive.capture()); + assertThat(new String(bytesExportedToDrive.getValue(), UTF_8)).isEqualTo(expectedCsv); + verifyNoMoreInteractions(driveConnection); + } + + @Test + void test_exportsDropListAcrossOpenTlds_sortedByDomainName() throws Exception { + // Active domain with no drop date (END_INSTANT) on open TLD -> excluded + persistActiveDomain("active.open1"); + + // Pending delete domains on open TLDs -> included + persistDeletedDomain("zebra.open1", Instant.parse("2020-02-07T02:02:02Z")); + persistDeletedDomain("alpha.open2", Instant.parse("2020-02-04T02:02:02Z")); + + // Pending delete domain on non-invoicing (closed) TLD -> excluded + persistDeletedDomain("closed.closed", Instant.parse("2020-02-05T02:02:02Z")); + + // Pending delete domain on open TLD with XAP disabled -> excluded + createTld("noxap"); + persistResource( + Tld.get("noxap") + .asBuilder() + .setInvoicingEnabled(true) + .setExpiryAccessPeriodTransitions( + ImmutableSortedMap.of(START_INSTANT, ExpiryAccessPeriodMode.DISABLED)) + .build()); + persistDeletedDomain("noxap.noxap", Instant.parse("2020-02-05T02:02:02Z")); + + // Pending delete domain on test TLD -> excluded + persistDeletedDomain("test.testtld", Instant.parse("2020-02-06T02:02:02Z")); + + // Already deleted domain on open TLD -> excluded + persistDeletedDomain("deleted.open1", Instant.parse("2020-02-01T02:02:02Z")); + + action.run(); + + verifyExportedToDrive( + """ + domain_name,tld,deletion_time + alpha.open2,open2,2020-02-04T02:02:02Z + zebra.open1,open1,2020-02-07T02:02:02Z + """); + } + + @Test + void test_skipsDriveExport_whenDriveFolderIdIsEmpty() { + action.driveFolderId = Optional.empty(); + action.run(); + verifyNoInteractions(driveConnection); + } + + @Test + void test_skipsDriveExport_whenDriveFolderIdIsEmptyString() { + action.driveFolderId = Optional.of(""); + action.run(); + verifyNoInteractions(driveConnection); + } + + @Test + void test_emptyDropList_outputsHeaderOnly() throws Exception { + persistActiveDomain("active.open1"); + action.run(); + verifyExportedToDrive("domain_name,tld,deletion_time\n"); + } + + @Test + void test_noOpenTldsWithXap_outputsHeaderOnly() throws Exception { + persistResource( + Tld.get("open1") + .asBuilder() + .setExpiryAccessPeriodTransitions( + ImmutableSortedMap.of(START_INSTANT, ExpiryAccessPeriodMode.DISABLED)) + .build()); + persistResource( + Tld.get("open2") + .asBuilder() + .setExpiryAccessPeriodTransitions( + ImmutableSortedMap.of(START_INSTANT, ExpiryAccessPeriodMode.DISABLED)) + .build()); + + persistDeletedDomain("zebra.open1", Instant.parse("2020-02-07T02:02:02Z")); + + action.run(); + verifyExportedToDrive("domain_name,tld,deletion_time\n"); + } + + @Test + void test_rethrowsRuntimeException_whenDriveFails() throws Exception { + persistDeletedDomain("alpha.open2", Instant.parse("2020-02-04T02:02:02Z")); + when(driveConnection.createOrUpdateFile(any(), any(), any(), any())) + .thenThrow(new IOException("Drive timeout")); + + RuntimeException thrown = assertThrows(RuntimeException.class, () -> action.run()); + assertThat(thrown) + .hasMessageThat() + .isEqualTo("Error exporting domain drop list to Drive folder drop_list_folder_id"); + assertThat(thrown).hasCauseThat().hasMessageThat().isEqualTo("Drive timeout"); + } +} diff --git a/core/src/test/resources/google/registry/module/routing.txt b/core/src/test/resources/google/registry/module/routing.txt index 6079a5bdc..a9634071d 100644 --- a/core/src/test/resources/google/registry/module/routing.txt +++ b/core/src/test/resources/google/registry/module/routing.txt @@ -27,6 +27,7 @@ BACKEND /_dr/task/dnsRefresh RefreshDnsAction BACKEND /_dr/task/executeCannedScript CannedScriptExecutionAction POST,GET y APP ADMIN BACKEND /_dr/task/expandBillingRecurrences ExpandBillingRecurrencesAction GET n APP ADMIN BACKEND /_dr/task/exportDomainLists ExportDomainListsAction POST n APP ADMIN +BACKEND /_dr/task/exportDropList ExportDropListAction POST n APP ADMIN BACKEND /_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n APP ADMIN BACKEND /_dr/task/exportReservedTerms ExportReservedTermsAction POST n APP ADMIN BACKEND /_dr/task/generateInvoices GenerateInvoicesAction POST n APP ADMIN