mirror of
https://github.com/google/nomulus
synced 2026-09-06 16:17:13 +00:00
Add mapreduce action to create synthetic history entries (#1125)
* Add mapreduce action to create synthetic history entries RDE and zone file generation require being able to tell what objects looked like in the past (though not beyond 30 days, or whatever the Datastore retention period is set to). In Datastore, to answer this we look at commit logs, and in SQL we will look at the History objects stored for each EPP resource. This action can be run once while in Datastore-primary-SQL-secondary to make sure that every EPP resource has at least one history entry for which the resource-at-this-time field is filled out in the SQL world.
This commit is contained in:
gbrodman
+115
@@ -0,0 +1,115 @@
|
||||
// Copyright 2021 The Nomulus Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package google.registry.tools.javascrap;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.loadByKey;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
|
||||
import static google.registry.testing.DatabaseHelper.persistActiveHost;
|
||||
import static google.registry.testing.DatabaseHelper.persistDomainAsDeleted;
|
||||
import static google.registry.testing.DatabaseHelper.persistDomainWithDependentResources;
|
||||
import static google.registry.util.DateTimeUtils.END_OF_TIME;
|
||||
import static google.registry.util.DateTimeUtils.START_OF_TIME;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.googlecode.objectify.Key;
|
||||
import google.registry.model.EppResource;
|
||||
import google.registry.model.contact.ContactResource;
|
||||
import google.registry.model.domain.DomainBase;
|
||||
import google.registry.model.host.HostResource;
|
||||
import google.registry.model.reporting.HistoryEntry;
|
||||
import google.registry.model.reporting.HistoryEntryDao;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.mapreduce.MapreduceTestCase;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/** Tests for {@link CreateSyntheticHistoryEntriesAction}. */
|
||||
public class CreateSyntheticHistoryEntriesActionTest
|
||||
extends MapreduceTestCase<CreateSyntheticHistoryEntriesAction> {
|
||||
|
||||
private DomainBase domain;
|
||||
private ContactResource contact;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
action =
|
||||
new CreateSyntheticHistoryEntriesAction(
|
||||
makeDefaultRunner(), new FakeResponse(), "adminRegistrarId");
|
||||
|
||||
createTld("tld");
|
||||
domain = persistActiveDomain("example.tld");
|
||||
contact = loadByKey(domain.getAdminContact());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCreation_forAllTypes() throws Exception {
|
||||
DomainBase domain2 = persistActiveDomain("exampletwo.tld");
|
||||
ContactResource contact2 = loadByKey(domain2.getAdminContact());
|
||||
HostResource host = persistActiveHost("ns1.foobar.tld");
|
||||
HostResource host2 = persistActiveHost("ns1.baz.tld");
|
||||
|
||||
assertThat(HistoryEntryDao.loadAllHistoryObjects(START_OF_TIME, END_OF_TIME)).isEmpty();
|
||||
runMapreduce();
|
||||
|
||||
for (EppResource resource : ImmutableList.of(contact, contact2, domain, domain2, host, host2)) {
|
||||
HistoryEntry historyEntry =
|
||||
Iterables.getOnlyElement(
|
||||
HistoryEntryDao.loadHistoryObjectsForResource(resource.createVKey()));
|
||||
assertThat(historyEntry.getParent()).isEqualTo(Key.create(resource));
|
||||
assertThat(historyEntry.getType()).isEqualTo(HistoryEntry.Type.SYNTHETIC);
|
||||
}
|
||||
assertThat(HistoryEntryDao.loadAllHistoryObjects(START_OF_TIME, END_OF_TIME)).hasSize(6);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCreation_withPreviousHistoryEntry() throws Exception {
|
||||
DateTime now = DateTime.parse("1999-04-03T22:00:00.0Z");
|
||||
DomainBase withHistoryEntry =
|
||||
persistDomainWithDependentResources("foobar", "tld", contact, now, now, now.plusYears(1));
|
||||
assertThat(
|
||||
Iterables.getOnlyElement(
|
||||
HistoryEntryDao.loadHistoryObjectsForResource(withHistoryEntry.createVKey()))
|
||||
.getType())
|
||||
.isEqualTo(HistoryEntry.Type.DOMAIN_CREATE);
|
||||
|
||||
runMapreduce();
|
||||
|
||||
Iterable<? extends HistoryEntry> historyEntries =
|
||||
HistoryEntryDao.loadHistoryObjectsForResource(withHistoryEntry.createVKey());
|
||||
assertThat(historyEntries).hasSize(2);
|
||||
assertThat(Iterables.getLast(historyEntries).getType()).isEqualTo(HistoryEntry.Type.SYNTHETIC);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCreation_forDeletedResource() throws Exception {
|
||||
persistDomainAsDeleted(domain, domain.getCreationTime().plusMonths(6));
|
||||
runMapreduce();
|
||||
|
||||
assertThat(
|
||||
Iterables.getOnlyElement(
|
||||
HistoryEntryDao.loadHistoryObjectsForResource(domain.createVKey()))
|
||||
.getType())
|
||||
.isEqualTo(HistoryEntry.Type.SYNTHETIC);
|
||||
}
|
||||
|
||||
private void runMapreduce() throws Exception {
|
||||
action.run();
|
||||
executeTasksUntilEmpty("mapreduce");
|
||||
}
|
||||
}
|
||||
@@ -1,47 +1,48 @@
|
||||
PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY
|
||||
/_dr/cron/commitLogCheckpoint CommitLogCheckpointAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/cron/commitLogFanout CommitLogFanoutAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/cron/fanout TldFanoutAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/cron/readDnsQueue ReadDnsQueueAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/dnsRefresh RefreshDnsAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL,API APP ADMIN
|
||||
/_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteOldCommitLogs DeleteOldCommitLogsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteProberData DeleteProberDataAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/expandRecurringBillingEvents ExpandRecurringBillingEventsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportCommitLogDiff ExportCommitLogDiffAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/generateSpec11 GenerateSpec11ReportAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/icannReportingUpload IcannReportingUploadAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/nordnUpload NordnUploadAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED
|
||||
/_dr/task/publishDnsUpdates PublishDnsUpdatesAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/publishInvoices PublishInvoicesAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/publishSpec11 PublishSpec11ReportAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/rdeReport RdeReportAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/rdeStaging RdeStagingAction GET,POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/rdeUpload RdeUploadAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/refreshDnsOnHostRename RefreshDnsOnHostRenameAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/relockDomain RelockDomainAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/resaveAllEppResources ResaveAllEppResourcesAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/resaveEntity ResaveEntityAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/syncGroupMembers SyncGroupMembersAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/syncRegistrarsSheet SyncRegistrarsSheetAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/tmchCrl TmchCrlAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/tmchDnl TmchDnlAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/tmchSmdrl TmchSmdrlAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/updateRegistrarRdapBaseUrls UpdateRegistrarRdapBaseUrlsAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/task/updateSnapshotView UpdateSnapshotViewAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/uploadDatastoreBackup UploadDatastoreBackupAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/wipeOutCloudSql WipeOutCloudSqlAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/wipeOutDatastore WipeoutDatastoreAction GET n INTERNAL,API APP ADMIN
|
||||
PATH CLASS METHODS OK AUTH_METHODS MIN USER_POLICY
|
||||
/_dr/cron/commitLogCheckpoint CommitLogCheckpointAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/cron/commitLogFanout CommitLogFanoutAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/cron/fanout TldFanoutAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/cron/readDnsQueue ReadDnsQueueAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/dnsRefresh RefreshDnsAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/task/backupDatastore BackupDatastoreAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/brdaCopy BrdaCopyAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/checkDatastoreBackup CheckBackupAction POST,GET y INTERNAL,API APP ADMIN
|
||||
/_dr/task/copyDetailReports CopyDetailReportsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/createSyntheticHistoryEntries CreateSyntheticHistoryEntriesAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteContactsAndHosts DeleteContactsAndHostsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteOldCommitLogs DeleteOldCommitLogsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/deleteProberData DeleteProberDataAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/expandRecurringBillingEvents ExpandRecurringBillingEventsAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportCommitLogDiff ExportCommitLogDiffAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportDomainLists ExportDomainListsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/exportReservedTerms ExportReservedTermsAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/generateInvoices GenerateInvoicesAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/generateSpec11 GenerateSpec11ReportAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/icannReportingStaging IcannReportingStagingAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/icannReportingUpload IcannReportingUploadAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/nordnUpload NordnUploadAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/nordnVerify NordnVerifyAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/pollBigqueryJob BigqueryPollJobAction GET,POST y INTERNAL APP IGNORED
|
||||
/_dr/task/publishDnsUpdates PublishDnsUpdatesAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/publishInvoices PublishInvoicesAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/publishSpec11 PublishSpec11ReportAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/rdeReport RdeReportAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/rdeStaging RdeStagingAction GET,POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/rdeUpload RdeUploadAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/refreshDnsOnHostRename RefreshDnsOnHostRenameAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/relockDomain RelockDomainAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/resaveAllEppResources ResaveAllEppResourcesAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/resaveEntity ResaveEntityAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/syncGroupMembers SyncGroupMembersAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/syncRegistrarsSheet SyncRegistrarsSheetAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/tmchCrl TmchCrlAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/tmchDnl TmchDnlAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/tmchSmdrl TmchSmdrlAction POST y INTERNAL,API APP ADMIN
|
||||
/_dr/task/updateRegistrarRdapBaseUrls UpdateRegistrarRdapBaseUrlsAction GET y INTERNAL,API APP ADMIN
|
||||
/_dr/task/updateSnapshotView UpdateSnapshotViewAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/uploadDatastoreBackup UploadDatastoreBackupAction POST n INTERNAL,API APP ADMIN
|
||||
/_dr/task/wipeOutCloudSql WipeOutCloudSqlAction GET n INTERNAL,API APP ADMIN
|
||||
/_dr/task/wipeOutDatastore WipeoutDatastoreAction GET n INTERNAL,API APP ADMIN
|
||||
|
||||
Reference in New Issue
Block a user