diff --git a/core/src/main/java/google/registry/tools/CommandWithRemoteApi.java b/core/src/main/java/google/registry/tools/CommandWithRemoteApi.java index 0d7dd855a..eab2c48ee 100644 --- a/core/src/main/java/google/registry/tools/CommandWithRemoteApi.java +++ b/core/src/main/java/google/registry/tools/CommandWithRemoteApi.java @@ -20,4 +20,4 @@ package google.registry.tools; *

Just implementing this is sufficient to use the remote api; {@link RegistryTool} will install * it as needed. */ -interface CommandWithRemoteApi extends Command {} +public interface CommandWithRemoteApi extends Command {} diff --git a/core/src/main/java/google/registry/tools/RegistryTool.java b/core/src/main/java/google/registry/tools/RegistryTool.java index 6e15298d9..4ffde7201 100644 --- a/core/src/main/java/google/registry/tools/RegistryTool.java +++ b/core/src/main/java/google/registry/tools/RegistryTool.java @@ -15,6 +15,7 @@ package google.registry.tools; import com.google.common.collect.ImmutableMap; +import google.registry.tools.javascrap.BackfillRegistryLocksCommand; import google.registry.tools.javascrap.PopulateNullRegistrarFieldsCommand; import google.registry.tools.javascrap.RemoveIpAddressCommand; @@ -30,6 +31,7 @@ public final class RegistryTool { public static final ImmutableMap> COMMAND_MAP = new ImmutableMap.Builder>() .put("ack_poll_messages", AckPollMessagesCommand.class) + .put("backfill_registry_locks", BackfillRegistryLocksCommand.class) .put("canonicalize_labels", CanonicalizeLabelsCommand.class) .put("check_domain", CheckDomainCommand.class) .put("check_domain_claims", CheckDomainClaimsCommand.class) diff --git a/core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java b/core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java new file mode 100644 index 000000000..7457af7ad --- /dev/null +++ b/core/src/main/java/google/registry/tools/javascrap/BackfillRegistryLocksCommand.java @@ -0,0 +1,152 @@ +// Copyright 2020 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.javascrap; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import com.googlecode.objectify.Key; +import google.registry.config.RegistryConfig.Config; +import google.registry.model.domain.DomainBase; +import google.registry.model.registry.RegistryLockDao; +import google.registry.model.reporting.HistoryEntry; +import google.registry.schema.domain.RegistryLock; +import google.registry.tools.CommandWithRemoteApi; +import google.registry.tools.ConfirmingCommand; +import google.registry.util.Clock; +import google.registry.util.StringGenerator; +import java.util.Comparator; +import java.util.List; +import javax.inject.Inject; +import javax.inject.Named; +import org.joda.time.DateTime; + +/** + * Scrap tool to backfill {@link RegistryLock}s for domains previously locked. + * + *

This will save new objects for all existing domains that are locked but don't have any + * corresponding lock objects already in the database. + */ +@Parameters( + separators = " =", + commandDescription = + "Backfills RegistryLock objects for specified domain resource IDs that are locked but don't" + + " already have a corresponding RegistryLock object.") +public class BackfillRegistryLocksCommand extends ConfirmingCommand + implements CommandWithRemoteApi { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private static final int VERIFICATION_CODE_LENGTH = 32; + + @Parameter( + names = {"--domain_roids"}, + description = "Comma-separated list of domain roids to check") + protected List roids; + + // Inject here so that we can create the command automatically for tests + @Inject Clock clock; + + @Inject + @Config("registryAdminClientId") + String registryAdminClientId; + + @Inject + @Named("base58StringGenerator") + StringGenerator stringGenerator; + + private DateTime now; + private ImmutableList lockedDomains; + + @Override + protected String prompt() { + checkArgument( + roids != null && !roids.isEmpty(), "Must provide non-empty domain_roids argument"); + now = clock.nowUtc(); + lockedDomains = getLockedDomainsWithoutLocks(); + ImmutableList lockedDomainNames = + lockedDomains.stream() + .map(DomainBase::getFullyQualifiedDomainName) + .collect(toImmutableList()); + return String.format( + "Locked domains for which there does not exist a RegistryLock object: %s", + lockedDomainNames); + } + + @Override + protected String execute() { + ImmutableSet.Builder failedDomainsBuilder = new ImmutableSet.Builder<>(); + for (DomainBase domainBase : lockedDomains) { + try { + RegistryLockDao.save( + new RegistryLock.Builder() + .isSuperuser(true) + .setRegistrarId(registryAdminClientId) + .setRepoId(domainBase.getRepoId()) + .setDomainName(domainBase.getFullyQualifiedDomainName()) + .setLockCompletionTimestamp(getLockCompletionTimestamp(domainBase, now)) + .setVerificationCode(stringGenerator.createString(VERIFICATION_CODE_LENGTH)) + .build()); + } catch (Throwable t) { + logger.atSevere().withCause(t).log( + "Error when creating lock object for domain %s.", + domainBase.getFullyQualifiedDomainName()); + failedDomainsBuilder.add(domainBase); + } + } + ImmutableSet failedDomains = failedDomainsBuilder.build(); + if (failedDomains.isEmpty()) { + return String.format( + "Successfully created lock objects for %d domains.", lockedDomains.size()); + } else { + return String.format( + "Successfully created lock objects for %d domains. We failed to create locks " + + "for the following domains: %s", + lockedDomains.size() - failedDomains.size(), lockedDomains); + } + } + + private DateTime getLockCompletionTimestamp(DomainBase domainBase, DateTime now) { + // Best-effort, if a domain was URS-locked we should use that time + // If we can't find that, return now. + return ofy().load().type(HistoryEntry.class).ancestor(domainBase).list().stream() + // sort by modification time descending so we get the most recent one if it was locked twice + .sorted(Comparator.comparing(HistoryEntry::getModificationTime).reversed()) + .filter(entry -> entry.getReason().equals("Uniform Rapid Suspension")) + .findFirst() + .map(HistoryEntry::getModificationTime) + .orElse(now); + } + + private ImmutableList getLockedDomainsWithoutLocks() { + return ImmutableList.copyOf( + ofy().load() + .keys( + roids.stream() + .map(roid -> Key.create(DomainBase.class, roid)) + .collect(toImmutableList())) + .values().stream() + .filter(d -> d.getDeletionTime().isAfter(now)) + .filter(d -> d.getStatusValues().containsAll(REGISTRY_LOCK_STATUSES)) + .filter(d -> !RegistryLockDao.getMostRecentByRepoId(d.getRepoId()).isPresent()) + .collect(toImmutableList())); + } +} diff --git a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java index 6bd2db5c9..0c3a8a26a 100644 --- a/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java +++ b/core/src/test/java/google/registry/schema/integration/SqlIntegrationTestSuite.java @@ -32,6 +32,7 @@ import google.registry.tools.LockDomainCommandTest; import google.registry.tools.UnlockDomainCommandTest; import google.registry.tools.UpdateRegistrarCommandTest; import google.registry.tools.UpdateReservedListCommandTest; +import google.registry.tools.javascrap.BackfillRegistryLocksCommandTest; import google.registry.tools.server.CreatePremiumListActionTest; import google.registry.tools.server.UpdatePremiumListActionTest; import google.registry.ui.server.registrar.RegistryLockGetActionTest; @@ -56,6 +57,7 @@ import org.junit.runners.Suite.SuiteClasses; */ @RunWith(Suite.class) @SuiteClasses({ + BackfillRegistryLocksCommandTest.class, ClaimsListDaoTest.class, CreatePremiumListActionTest.class, CreateRegistrarCommandTest.class, diff --git a/core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java b/core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java new file mode 100644 index 000000000..49aebade2 --- /dev/null +++ b/core/src/test/java/google/registry/tools/javascrap/BackfillRegistryLocksCommandTest.java @@ -0,0 +1,176 @@ +// Copyright 2020 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.tools.javascrap; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.testing.DatastoreHelper.createTld; +import static google.registry.testing.DatastoreHelper.persistActiveDomain; +import static google.registry.testing.DatastoreHelper.persistDeletedDomain; +import static google.registry.testing.DatastoreHelper.persistNewRegistrar; +import static google.registry.testing.DatastoreHelper.persistResource; +import static google.registry.tools.LockOrUnlockDomainCommand.REGISTRY_LOCK_STATUSES; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.truth.Truth8; +import google.registry.model.domain.DomainBase; +import google.registry.model.registrar.Registrar; +import google.registry.model.registry.RegistryLockDao; +import google.registry.model.reporting.HistoryEntry; +import google.registry.persistence.transaction.JpaTestRules; +import google.registry.persistence.transaction.JpaTestRules.JpaIntegrationWithCoverageRule; +import google.registry.schema.domain.RegistryLock; +import google.registry.testing.DeterministicStringGenerator; +import google.registry.testing.FakeClock; +import google.registry.tools.CommandTestCase; +import google.registry.util.StringGenerator.Alphabets; +import java.util.Optional; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link BackfillRegistryLocksCommand}. */ +@RunWith(JUnit4.class) +public class BackfillRegistryLocksCommandTest + extends CommandTestCase { + + private final FakeClock fakeClock = new FakeClock(); + + @Rule + public final JpaIntegrationWithCoverageRule jpaRule = + new JpaTestRules.Builder().buildIntegrationWithCoverageRule(); + + @Before + public void before() { + persistNewRegistrar("adminreg", "Admin Registrar", Registrar.Type.REAL, 693L); + createTld("tld"); + command.registryAdminClientId = "adminreg"; + command.clock = fakeClock; + command.stringGenerator = new DeterministicStringGenerator(Alphabets.BASE_58); + } + + @Test + public void testSimpleBackfill() throws Exception { + DomainBase domain = persistLockedDomain("example.tld"); + Truth8.assertThat(RegistryLockDao.getMostRecentByRepoId(domain.getRepoId())).isEmpty(); + + runCommandForced("--domain_roids", domain.getRepoId()); + + Optional lockOptional = RegistryLockDao.getMostRecentByRepoId(domain.getRepoId()); + Truth8.assertThat(lockOptional).isPresent(); + Truth8.assertThat(lockOptional.get().getLockCompletionTimestamp()).isPresent(); + } + + @Test + public void testBackfill_onlyLockedDomains() throws Exception { + DomainBase neverLockedDomain = persistActiveDomain("neverlocked.tld"); + DomainBase previouslyLockedDomain = persistLockedDomain("unlocked.tld"); + persistResource(previouslyLockedDomain.asBuilder().setStatusValues(ImmutableSet.of()).build()); + DomainBase lockedDomain = persistLockedDomain("locked.tld"); + + runCommandForced( + "--domain_roids", + String.format( + "%s,%s,%s", + neverLockedDomain.getRepoId(), + previouslyLockedDomain.getRepoId(), + lockedDomain.getRepoId())); + + ImmutableList locks = RegistryLockDao.getLockedDomainsByRegistrarId("adminreg"); + assertThat(locks).hasSize(1); + assertThat(Iterables.getOnlyElement(locks).getDomainName()).isEqualTo("locked.tld"); + } + + @Test + public void testBackfill_skipsDeletedDomains() throws Exception { + DomainBase domain = persistDeletedDomain("example.tld", fakeClock.nowUtc()); + persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build()); + fakeClock.advanceBy(Duration.standardSeconds(1)); + runCommandForced("--domain_roids", domain.getRepoId()); + Truth8.assertThat(RegistryLockDao.getMostRecentByRepoId(domain.getRepoId())).isEmpty(); + } + + @Test + public void testBackfill_skipsDomains_ifLockAlreadyExists() throws Exception { + DomainBase domain = persistLockedDomain("example.tld"); + + RegistryLock previousLock = + RegistryLockDao.save( + new RegistryLock.Builder() + .isSuperuser(true) + .setRegistrarId("adminreg") + .setRepoId(domain.getRepoId()) + .setDomainName(domain.getFullyQualifiedDomainName()) + .setLockCompletionTimestamp(fakeClock.nowUtc()) + .setVerificationCode(command.stringGenerator.createString(32)) + .build()); + + fakeClock.advanceBy(Duration.standardDays(1)); + runCommandForced("--domain_roids", domain.getRepoId()); + + assertThat( + RegistryLockDao.getMostRecentByRepoId(domain.getRepoId()) + .get() + .getLockCompletionTimestamp()) + .isEqualTo(previousLock.getLockCompletionTimestamp()); + } + + @Test + public void testBackfill_usesUrsTime_ifExists() throws Exception { + DateTime ursTime = fakeClock.nowUtc(); + DomainBase ursDomain = persistLockedDomain("urs.tld"); + HistoryEntry historyEntry = + new HistoryEntry.Builder() + .setBySuperuser(true) + .setClientId("adminreg") + .setModificationTime(ursTime) + .setParent(ursDomain) + .setReason("Uniform Rapid Suspension") + .setType(HistoryEntry.Type.DOMAIN_UPDATE) + .setRequestedByRegistrar(false) + .build(); + persistResource(historyEntry); + DomainBase nonUrsDomain = persistLockedDomain("nonurs.tld"); + + fakeClock.advanceBy(Duration.standardDays(10)); + runCommandForced( + "--domain_roids", String.format("%s,%s", ursDomain.getRepoId(), nonUrsDomain.getRepoId())); + + RegistryLock ursLock = + RegistryLockDao.getMostRecentVerifiedLockByRepoId(ursDomain.getRepoId()).get(); + assertThat(ursLock.getLockCompletionTimestamp().get()).isEqualTo(ursTime); + RegistryLock nonUrsLock = + RegistryLockDao.getMostRecentVerifiedLockByRepoId(nonUrsDomain.getRepoId()).get(); + assertThat(nonUrsLock.getLockCompletionTimestamp().get()).isEqualTo(fakeClock.nowUtc()); + } + + @Test + public void testFailure_mustProvideDomainRoids() { + assertThat(assertThrows(IllegalArgumentException.class, () -> runCommandForced())) + .hasMessageThat() + .isEqualTo("Must provide non-empty domain_roids argument"); + } + + private static DomainBase persistLockedDomain(String domainName) { + DomainBase domain = persistActiveDomain(domainName); + return persistResource(domain.asBuilder().setStatusValues(REGISTRY_LOCK_STATUSES).build()); + } +}