1
0
mirror of https://github.com/google/nomulus synced 2026-04-21 16:50:44 +00:00

Fix BsaRefreshAction bugs (#2294)

* Fix BsaRefreshAction bugs

Added functional tests for BsaRefreshAction, which checks for changes in
domain registration and reservation, and apply them to the Unblockable
domain list.

Fixed a few bugs exposed by the tests.

Also refactored a few other tests.
This commit is contained in:
Weimin Yu
2024-01-22 12:23:29 -05:00
committed by GitHub
parent c414e38a98
commit f61579b350
14 changed files with 509 additions and 74 deletions

View File

@@ -118,15 +118,8 @@ class BsaDownloadFunctionalTest {
gcsClient.readBlockList(downloadJob, BlockListType.BLOCK_PLUS)) {
assertThat(blockListFile).containsExactly(BSA_CSV_HEADER, "abc,2", "def,3");
}
ImmutableList<String> persistedLabels =
ImmutableList.copyOf(
tm().transact(
() ->
tm().getEntityManager()
.createNativeQuery("SELECT label from \"BsaLabel\"")
.getResultList()));
// TODO(weiminyu): check intermediate files
assertThat(persistedLabels).containsExactly("abc", "def");
assertThat(getPersistedLabels()).containsExactly("abc", "def");
}
@Test

View File

@@ -0,0 +1,319 @@
// Copyright 2023 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package google.registry.bsa;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedDomainToList;
import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedListsToTld;
import static google.registry.bsa.ReservedDomainsTestingUtils.createReservedList;
import static google.registry.bsa.ReservedDomainsTestingUtils.removeReservedDomainFromList;
import static google.registry.bsa.persistence.BsaTestingUtils.createDownloadScheduler;
import static google.registry.bsa.persistence.BsaTestingUtils.persistBsaLabel;
import static google.registry.bsa.persistence.BsaTestingUtils.queryUnblockableDomains;
import static google.registry.model.tld.Tlds.getTldEntitiesOfType;
import static google.registry.model.tld.label.ReservationType.RESERVED_FOR_SPECIFIC_USE;
import static google.registry.testing.DatabaseHelper.createTlds;
import static google.registry.testing.DatabaseHelper.deleteTestDomain;
import static google.registry.testing.DatabaseHelper.persistActiveDomain;
import static google.registry.testing.DatabaseHelper.persistResource;
import static google.registry.util.DateTimeUtils.START_OF_TIME;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import google.registry.bsa.api.BsaReportSender;
import google.registry.bsa.api.UnblockableDomain;
import google.registry.bsa.api.UnblockableDomain.Reason;
import google.registry.bsa.api.UnblockableDomainChange;
import google.registry.bsa.persistence.BsaTestingUtils;
import google.registry.gcs.GcsUtils;
import google.registry.model.domain.Domain;
import google.registry.model.tld.Tld.TldType;
import google.registry.persistence.transaction.JpaTestExtensions;
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
import google.registry.request.Response;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeLockHandler;
import google.registry.testing.FakeResponse;
import java.util.Optional;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Functional tests for refreshing the unblockable domains with recent registration and reservation
* changes.
*/
@ExtendWith(MockitoExtension.class)
class BsaRefreshFunctionalTest {
static final DateTime TEST_START_TIME = DateTime.parse("2024-01-01T00:00:00Z");
static final String RESERVED_LIST_NAME = "reserved";
private final FakeClock fakeClock = new FakeClock(TEST_START_TIME);
@RegisterExtension
JpaIntegrationWithCoverageExtension jpa =
new JpaTestExtensions.Builder().withClock(fakeClock).buildIntegrationWithCoverageExtension();
@Mock BsaReportSender bsaReportSender;
private GcsClient gcsClient;
private Response response;
private BsaRefreshAction action;
@BeforeEach
void setup() throws Exception {
gcsClient =
new GcsClient(new GcsUtils(LocalStorageHelper.getOptions()), "my-bucket", "SHA-256");
response = new FakeResponse();
action =
new BsaRefreshAction(
BsaTestingUtils.createRefreshScheduler(),
gcsClient,
bsaReportSender,
/* transactionBatchSize= */ 5,
/* domainCreateTxnCommitTimeLag= */ Duration.millis(1),
new BsaLock(
new FakeLockHandler(/* lockSucceeds= */ true), Duration.standardSeconds(30)),
fakeClock,
response);
initDb();
}
private String getRefreshJobName(DateTime jobStartTime) {
return jobStartTime.toString() + "-refresh";
}
private void initDb() {
createTlds("app", "dev");
getTldEntitiesOfType(TldType.REAL)
.forEach(
tld ->
persistResource(
tld.asBuilder().setBsaEnrollStartTime(Optional.of(START_OF_TIME)).build()));
createReservedList(RESERVED_LIST_NAME, "dummy", RESERVED_FOR_SPECIFIC_USE);
addReservedListsToTld("app", ImmutableList.of(RESERVED_LIST_NAME));
persistBsaLabel("blocked1");
persistBsaLabel("blocked2");
// Creates a download record so that refresher will not quit immediately.
createDownloadScheduler(fakeClock).schedule().get().updateJobStage(DownloadStage.DONE);
fakeClock.advanceOneMilli();
}
@Test
void newReservedDomain_addedAsUnblockable() throws Exception {
addReservedDomainToList(
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
String jobName = getRefreshJobName(fakeClock.nowUtc());
action.run();
UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.app", Reason.RESERVED);
assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
assertThat(gcsClient.readRefreshChanges(jobName))
.containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
verify(bsaReportSender, never()).removeUnblockableDomainsUpdates(anyString());
verify(bsaReportSender, times(1))
.addUnblockableDomainsUpdates("{\n \"reserved\": [\n \"blocked1.app\"\n ]\n}");
}
@Test
void newRegisteredDomain_addedAsUnblockable() throws Exception {
persistActiveDomain("blocked1.dev", fakeClock.nowUtc());
persistActiveDomain("dummy.dev", fakeClock.nowUtc());
String jobName = getRefreshJobName(fakeClock.nowUtc());
action.run();
UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.dev", Reason.REGISTERED);
assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
assertThat(gcsClient.readRefreshChanges(jobName))
.containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
verify(bsaReportSender, never()).removeUnblockableDomainsUpdates(anyString());
verify(bsaReportSender, times(1))
.addUnblockableDomainsUpdates("{\n \"registered\": [\n \"blocked1.dev\"\n ]\n}");
}
@Test
void registeredUnblockable_unregistered() {
Domain domain = persistActiveDomain("blocked1.dev", fakeClock.nowUtc());
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.dev", Reason.REGISTERED));
fakeClock.advanceOneMilli();
deleteTestDomain(domain, fakeClock.nowUtc());
fakeClock.advanceOneMilli();
String jobName = getRefreshJobName(fakeClock.nowUtc());
Mockito.reset(bsaReportSender);
action.run();
assertThat(queryUnblockableDomains()).isEmpty();
assertThat(gcsClient.readRefreshChanges(jobName))
.containsExactly(
UnblockableDomainChange.ofDeleted(
UnblockableDomain.of("blocked1.dev", Reason.REGISTERED)));
verify(bsaReportSender, never()).addUnblockableDomainsUpdates(anyString());
verify(bsaReportSender, times(1)).removeUnblockableDomainsUpdates("[\n \"blocked1.dev\"\n]");
}
@Test
void reservedUnblockable_noLongerReserved() {
addReservedDomainToList(
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
fakeClock.advanceOneMilli();
removeReservedDomainFromList(RESERVED_LIST_NAME, ImmutableSet.of("blocked1"));
String jobName = getRefreshJobName(fakeClock.nowUtc());
Mockito.reset(bsaReportSender);
action.run();
assertThat(queryUnblockableDomains()).isEmpty();
assertThat(gcsClient.readRefreshChanges(jobName))
.containsExactly(
UnblockableDomainChange.ofDeleted(
UnblockableDomain.of("blocked1.app", Reason.RESERVED)));
verify(bsaReportSender, never()).addUnblockableDomainsUpdates(anyString());
verify(bsaReportSender, times(1)).removeUnblockableDomainsUpdates("[\n \"blocked1.app\"\n]");
}
@Test
void registeredAndReservedUnblockable_noLongerRegistered_stillUnblockable() throws Exception {
addReservedDomainToList(
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
Domain domain = persistActiveDomain("blocked1.app", fakeClock.nowUtc());
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
fakeClock.advanceOneMilli();
deleteTestDomain(domain, fakeClock.nowUtc());
fakeClock.advanceOneMilli();
String jobName = getRefreshJobName(fakeClock.nowUtc());
Mockito.reset(bsaReportSender);
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
assertThat(gcsClient.readRefreshChanges(jobName))
.containsExactly(
UnblockableDomainChange.ofChanged(
UnblockableDomain.of("blocked1.app", Reason.REGISTERED), Reason.RESERVED));
InOrder inOrder = Mockito.inOrder(bsaReportSender);
inOrder.verify(bsaReportSender).removeUnblockableDomainsUpdates("[\n \"blocked1.app\"\n]");
inOrder
.verify(bsaReportSender)
.addUnblockableDomainsUpdates("{\n \"reserved\": [\n \"blocked1.app\"\n ]\n}");
}
@Test
void reservedUblockable_becomesRegistered_changeToRegisterd() throws Exception {
addReservedDomainToList(
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.RESERVED));
fakeClock.advanceOneMilli();
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
fakeClock.advanceOneMilli();
Mockito.reset(bsaReportSender);
String jobName = getRefreshJobName(fakeClock.nowUtc());
action.run();
UnblockableDomain changed = UnblockableDomain.of("blocked1.app", Reason.REGISTERED);
assertThat(queryUnblockableDomains()).containsExactly(changed);
assertThat(gcsClient.readRefreshChanges(jobName))
.containsExactly(
UnblockableDomainChange.ofChanged(
UnblockableDomain.of("blocked1.app", Reason.RESERVED), Reason.REGISTERED));
InOrder inOrder = Mockito.inOrder(bsaReportSender);
inOrder.verify(bsaReportSender).removeUnblockableDomainsUpdates("[\n \"blocked1.app\"\n]");
inOrder
.verify(bsaReportSender)
.addUnblockableDomainsUpdates("{\n \"registered\": [\n \"blocked1.app\"\n ]\n}");
}
@Test
void newRegisteredAndReservedDomain_addedAsRegisteredUnblockable() throws Exception {
addReservedDomainToList(
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
String jobName = getRefreshJobName(fakeClock.nowUtc());
action.run();
UnblockableDomain newUnblockable = UnblockableDomain.of("blocked1.app", Reason.REGISTERED);
assertThat(queryUnblockableDomains()).containsExactly(newUnblockable);
assertThat(gcsClient.readRefreshChanges(jobName))
.containsExactly(UnblockableDomainChange.ofNew(newUnblockable));
}
@Test
void registeredAndReservedUnblockable_noLongerReserved_noChange() throws Exception {
addReservedDomainToList(
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
fakeClock.advanceOneMilli();
removeReservedDomainFromList(RESERVED_LIST_NAME, ImmutableSet.of("blocked1"));
fakeClock.advanceOneMilli();
Mockito.reset(bsaReportSender);
String jobName = getRefreshJobName(fakeClock.nowUtc());
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
assertThat(gcsClient.readRefreshChanges(jobName)).isEmpty();
verifyNoInteractions(bsaReportSender);
}
@Test
void registeredUblockable_becomesReserved_noChange() throws Exception {
persistActiveDomain("blocked1.app", fakeClock.nowUtc());
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
fakeClock.advanceOneMilli();
addReservedDomainToList(
RESERVED_LIST_NAME, ImmutableMap.of("blocked1", RESERVED_FOR_SPECIFIC_USE));
fakeClock.advanceOneMilli();
Mockito.reset(bsaReportSender);
String jobName = getRefreshJobName(fakeClock.nowUtc());
action.run();
assertThat(queryUnblockableDomains())
.containsExactly(UnblockableDomain.of("blocked1.app", Reason.REGISTERED));
assertThat(gcsClient.readRefreshChanges(jobName)).isEmpty();
verifyNoInteractions(bsaReportSender);
}
}

View File

@@ -62,4 +62,43 @@ public final class ReservedDomainsTestingUtils {
.build();
persistResource(tld.asBuilder().setReservedListsByName(reservedLists).build());
}
public static void addReservedDomainToList(
String listName, ImmutableMap<String, ReservationType> reservedLabels) {
ImmutableMap<String, ReservedListEntry> existingEntries =
ReservedList.get(listName).get().getReservedListEntries();
ImmutableMap<String, ReservedListEntry> newEntries =
ImmutableMap.copyOf(
Maps.transformEntries(
reservedLabels, (key, value) -> ReservedListEntry.create(key, value, "")));
ReservedListDao.save(
new ReservedList.Builder()
.setName(listName)
.setCreationTimestamp(START_OF_TIME)
.setShouldPublish(true)
.setReservedListMap(
new ImmutableMap.Builder<String, ReservedListEntry>()
.putAll(existingEntries)
.putAll(newEntries)
.buildKeepingLast())
.build());
}
public static void removeReservedDomainFromList(
String listName, ImmutableSet<String> removedLabels) {
ImmutableMap<String, ReservedListEntry> existingEntries =
ReservedList.get(listName).get().getReservedListEntries();
ImmutableMap<String, ReservedListEntry> newEntries =
ImmutableMap.copyOf(
Maps.filterEntries(existingEntries, entry -> !removedLabels.contains(entry.getKey())));
ReservedListDao.save(
new ReservedList.Builder()
.setName(listName)
.setCreationTimestamp(START_OF_TIME)
.setShouldPublish(true)
.setReservedListMap(newEntries)
.build());
}
}

View File

@@ -14,8 +14,11 @@
package google.registry.bsa.persistence;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import com.google.common.collect.ImmutableList;
import google.registry.bsa.api.UnblockableDomain;
import google.registry.util.Clock;
import org.joda.time.DateTime;
import org.joda.time.Duration;
@@ -35,7 +38,21 @@ public final class BsaTestingUtils {
tm().transact(() -> tm().put(new BsaLabel(domainLabel, BSA_LABEL_CREATION_TIME)));
}
public static void persistUnblockableDomain(UnblockableDomain unblockableDomain) {
tm().transact(() -> tm().put(BsaUnblockableDomain.of(unblockableDomain)));
}
public static DownloadScheduler createDownloadScheduler(Clock clock) {
return new DownloadScheduler(DEFAULT_DOWNLOAD_INTERVAL, DEFAULT_NOP_INTERVAL, clock);
}
public static RefreshScheduler createRefreshScheduler() {
return new RefreshScheduler();
}
public static ImmutableList<UnblockableDomain> queryUnblockableDomains() {
return tm().transact(() -> tm().loadAllOf(BsaUnblockableDomain.class)).stream()
.map(BsaUnblockableDomain::toUnblockableDomain)
.collect(toImmutableList());
}
}

View File

@@ -16,6 +16,8 @@ package google.registry.bsa.persistence;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth8.assertThat;
import static google.registry.bsa.ReservedDomainsTestingUtils.addReservedListsToTld;
import static google.registry.bsa.ReservedDomainsTestingUtils.createReservedList;
import static google.registry.bsa.persistence.LabelDiffUpdates.applyLabelDiff;
import static google.registry.persistence.transaction.TransactionManagerFactory.tm;
import static google.registry.testing.DatabaseHelper.createTld;
@@ -26,7 +28,6 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import google.registry.bsa.IdnChecker;
@@ -36,9 +37,6 @@ import google.registry.bsa.api.UnblockableDomain;
import google.registry.bsa.persistence.BsaUnblockableDomain.Reason;
import google.registry.model.tld.Tld;
import google.registry.model.tld.label.ReservationType;
import google.registry.model.tld.label.ReservedList;
import google.registry.model.tld.label.ReservedList.ReservedListEntry;
import google.registry.model.tld.label.ReservedListDao;
import google.registry.persistence.transaction.JpaTestExtensions;
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationWithCoverageExtension;
import google.registry.testing.FakeClock;
@@ -139,18 +137,8 @@ class LabelDiffUpdatesTest {
@Test
void applyLabelDiffs_newLabel() {
persistActiveDomain("label.app");
ReservedListDao.save(
new ReservedList.Builder()
.setReservedListMap(
ImmutableMap.of(
"label",
ReservedListEntry.create(
"label", ReservationType.RESERVED_FOR_SPECIFIC_USE, null)))
.setName("page_reserved")
.setCreationTimestamp(fakeClock.nowUtc())
.build());
ReservedList reservedList = ReservedList.get("page_reserved").get();
tm().transact(() -> tm().put(page.asBuilder().setReservedLists(reservedList).build()));
createReservedList("page_reserved", "label", ReservationType.RESERVED_FOR_SPECIFIC_USE);
addReservedListsToTld("page", ImmutableList.of("page_reserved"));
when(idnChecker.getForbiddingTlds(any()))
.thenReturn(Sets.difference(ImmutableSet.of(dev), ImmutableSet.of()).immutableCopy());

View File

@@ -208,6 +208,32 @@ class QueriesTest {
.containsExactly("d1.tld", "d2.tld");
}
@Test
void queryNewlyCreatedDomains_onlyDomainsAfterMinCreationTimeReturned() {
DateTime testStartTime = fakeClock.nowUtc();
createTlds("tld");
persistNewRegistrar("TheRegistrar");
// time 0:
persistResource(
newDomain("d1.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build());
// time 0, deletion time 1
persistDomainAsDeleted(
newDomain("will-delete.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build(),
fakeClock.nowUtc().plusMillis(1));
fakeClock.advanceOneMilli();
// time 1
persistResource(
newDomain("d2.tld").asBuilder().setCreationTimeForTest(fakeClock.nowUtc()).build());
fakeClock.advanceOneMilli();
// Now is time 2, ask for domains created since time 1
assertThat(
bsaQuery(
() ->
queryNewlyCreatedDomains(
ImmutableList.of("tld"), testStartTime.plusMillis(1), fakeClock.nowUtc())))
.containsExactly("d2.tld");
}
@Test
void queryNewlyCreatedDomains_onlyDomainsInRequestedTldsReturned() {
DateTime testStartTime = fakeClock.nowUtc();