mirror of
https://github.com/google/nomulus
synced 2026-01-05 13:07:04 +00:00
Add a BulkDomainTransferCommand (#2898)
This is a decently simple wrapper around the previously-created BulkDomainTransferAction that batches a provided list of domains up and sends them along to be transferred.
This commit is contained in:
@@ -54,14 +54,16 @@ import org.joda.time.Duration;
|
|||||||
* Acquisition) process in order to transfer a (possibly large) list of domains from one registrar
|
* Acquisition) process in order to transfer a (possibly large) list of domains from one registrar
|
||||||
* to another, though it may be used in other situations as well.
|
* to another, though it may be used in other situations as well.
|
||||||
*
|
*
|
||||||
|
* <p>The body of the HTTP post request should be a JSON list of the domains to be transferred.
|
||||||
|
* Because the list of domains to process can be quite large, this action should be called by a tool
|
||||||
|
* that batches the list of domains into reasonable sizes if necessary. The recommended usage path
|
||||||
|
* is to call this through the {@link google.registry.tools.BulkDomainTransferCommand}, which
|
||||||
|
* handles batching and input handling.
|
||||||
|
*
|
||||||
* <p>This runs as a single-threaded idempotent action that runs a superuser domain transfer on each
|
* <p>This runs as a single-threaded idempotent action that runs a superuser domain transfer on each
|
||||||
* domain to process. We go through the standard EPP process to make sure that we have an accurate
|
* domain to process. We go through the standard EPP process to make sure that we have an accurate
|
||||||
* historical representation of events (rather than force-modifying the domains in place).
|
* historical representation of events (rather than force-modifying the domains in place).
|
||||||
*
|
*
|
||||||
* <p>The body of the HTTP post request should be a JSON list of the domains to be transferred.
|
|
||||||
* Because the list of domains to process can be quite large, this action should be called by a tool
|
|
||||||
* that batches the list of domains into reasonable sizes if necessary.
|
|
||||||
*
|
|
||||||
* <p>Consider passing in an "maxQps" parameter based on the number of domains being transferred,
|
* <p>Consider passing in an "maxQps" parameter based on the number of domains being transferred,
|
||||||
* otherwise the default is {@link BatchModule#DEFAULT_MAX_QPS}.
|
* otherwise the default is {@link BatchModule#DEFAULT_MAX_QPS}.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ package google.registry.module;
|
|||||||
import dagger.Module;
|
import dagger.Module;
|
||||||
import dagger.Subcomponent;
|
import dagger.Subcomponent;
|
||||||
import google.registry.batch.BatchModule;
|
import google.registry.batch.BatchModule;
|
||||||
|
import google.registry.batch.BulkDomainTransferAction;
|
||||||
import google.registry.batch.CannedScriptExecutionAction;
|
import google.registry.batch.CannedScriptExecutionAction;
|
||||||
import google.registry.batch.DeleteExpiredDomainsAction;
|
import google.registry.batch.DeleteExpiredDomainsAction;
|
||||||
import google.registry.batch.DeleteLoadTestDataAction;
|
import google.registry.batch.DeleteLoadTestDataAction;
|
||||||
@@ -171,6 +172,8 @@ interface RequestComponent {
|
|||||||
|
|
||||||
BsaValidateAction bsaValidateAction();
|
BsaValidateAction bsaValidateAction();
|
||||||
|
|
||||||
|
BulkDomainTransferAction bulkDomainTransferAction();
|
||||||
|
|
||||||
CannedScriptExecutionAction cannedScriptExecutionAction();
|
CannedScriptExecutionAction cannedScriptExecutionAction();
|
||||||
|
|
||||||
CheckApiAction checkApiAction();
|
CheckApiAction checkApiAction();
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
// Copyright 2025 The Nomulus Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package google.registry.tools;
|
||||||
|
|
||||||
|
import static com.google.common.base.Preconditions.checkArgument;
|
||||||
|
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||||
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
|
|
||||||
|
import com.beust.jcommander.Parameter;
|
||||||
|
import com.beust.jcommander.Parameters;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.collect.ImmutableList;
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
|
import com.google.common.collect.Iterables;
|
||||||
|
import com.google.common.io.Files;
|
||||||
|
import com.google.common.net.MediaType;
|
||||||
|
import com.google.gson.Gson;
|
||||||
|
import google.registry.batch.BulkDomainTransferAction;
|
||||||
|
import google.registry.model.registrar.Registrar;
|
||||||
|
import google.registry.util.DomainNameUtils;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A command to bulk-transfer any number of domains from one registrar to another.
|
||||||
|
*
|
||||||
|
* <p>This should be used as part of the BTAPPA (Bulk Transfer After a Partial Portfolio
|
||||||
|
* Acquisition) process in order to transfer a (possibly large) list of domains from one registrar
|
||||||
|
* to another, though it may be used in other situations as well.
|
||||||
|
*
|
||||||
|
* <p>For a true bulk transfer of domains, one should pass in a file with a list of domains (one per
|
||||||
|
* line) but if we need to do an ad-hoc transfer of one domain we can do that as well.
|
||||||
|
*
|
||||||
|
* <p>For BTAPPA purposes, we expect "requestedByRegistrar" to be true; this may not be the case for
|
||||||
|
* other purposes e.g. legal compliance transfers.
|
||||||
|
*/
|
||||||
|
@Parameters(
|
||||||
|
separators = " =",
|
||||||
|
commandDescription = "Transfer domain(s) in bulk with immediate effect.")
|
||||||
|
public class BulkDomainTransferCommand extends ConfirmingCommand implements CommandWithConnection {
|
||||||
|
|
||||||
|
// we don't need any configuration on the Gson because all we need is a list of strings
|
||||||
|
private static final Gson GSON = new Gson();
|
||||||
|
private static final int DOMAIN_TRANSFER_BATCH_SIZE = 1000;
|
||||||
|
|
||||||
|
@Parameter(
|
||||||
|
names = {"--domains"},
|
||||||
|
description =
|
||||||
|
"Comma-separated list of domains to transfer, otherwise use --domain_names_file to"
|
||||||
|
+ " specify a possibly-large list of domains")
|
||||||
|
private List<String> domains;
|
||||||
|
|
||||||
|
@Parameter(
|
||||||
|
names = {"-d", "--domain_names_file"},
|
||||||
|
description = "A file with a list of newline-delimited domain names to create tokens for")
|
||||||
|
private String domainNamesFile;
|
||||||
|
|
||||||
|
@Parameter(
|
||||||
|
names = {"-g", "--gaining_registrar_id"},
|
||||||
|
description = "The ID of the registrar to which domains should be transferred",
|
||||||
|
required = true)
|
||||||
|
private String gainingRegistrarId;
|
||||||
|
|
||||||
|
@Parameter(
|
||||||
|
names = {"-l", "--losing_registrar_id"},
|
||||||
|
description = "The ID of the registrar from which domains should be transferred",
|
||||||
|
required = true)
|
||||||
|
private String losingRegistrarId;
|
||||||
|
|
||||||
|
@Parameter(
|
||||||
|
names = {"--reason"},
|
||||||
|
description = "Reason to transfer the domains",
|
||||||
|
required = true)
|
||||||
|
private String reason;
|
||||||
|
|
||||||
|
@Parameter(
|
||||||
|
names = {"--registrar_request"},
|
||||||
|
description = "Whether the change was requested by a registrar.")
|
||||||
|
private boolean requestedByRegistrar = false;
|
||||||
|
|
||||||
|
@Parameter(
|
||||||
|
names = {"--max_qps"},
|
||||||
|
description =
|
||||||
|
"Maximum queries to run per second, otherwise the default (maxQps) will be used")
|
||||||
|
private int maxQps;
|
||||||
|
|
||||||
|
private ServiceConnection connection;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setConnection(ServiceConnection connection) {
|
||||||
|
this.connection = connection;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String prompt() throws Exception {
|
||||||
|
checkArgument(
|
||||||
|
domainNamesFile != null ^ (domains != null && !domains.isEmpty()),
|
||||||
|
"Must specify exactly one input method, either --domains or --domain_names_file");
|
||||||
|
return String.format("Attempt to transfer %d domains?", getDomainList().size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected String execute() throws Exception {
|
||||||
|
checkArgument(
|
||||||
|
Registrar.loadByRegistrarIdCached(gainingRegistrarId).isPresent(),
|
||||||
|
"Gaining registrar %s doesn't exist",
|
||||||
|
gainingRegistrarId);
|
||||||
|
checkArgument(
|
||||||
|
Registrar.loadByRegistrarIdCached(losingRegistrarId).isPresent(),
|
||||||
|
"Losing registrar %s doesn't exist",
|
||||||
|
losingRegistrarId);
|
||||||
|
|
||||||
|
ImmutableMap.Builder<String, Object> paramsBuilder = new ImmutableMap.Builder<>();
|
||||||
|
paramsBuilder.put("gainingRegistrarId", gainingRegistrarId);
|
||||||
|
paramsBuilder.put("losingRegistrarId", losingRegistrarId);
|
||||||
|
paramsBuilder.put("requestedByRegistrar", requestedByRegistrar);
|
||||||
|
paramsBuilder.put("reason", reason);
|
||||||
|
if (maxQps > 0) {
|
||||||
|
paramsBuilder.put("maxQps", maxQps);
|
||||||
|
}
|
||||||
|
ImmutableMap<String, Object> params = paramsBuilder.build();
|
||||||
|
|
||||||
|
for (List<String> batch : Iterables.partition(getDomainList(), DOMAIN_TRANSFER_BATCH_SIZE)) {
|
||||||
|
System.out.printf("Sending batch of %d domains\n", batch.size());
|
||||||
|
byte[] domainsList = GSON.toJson(batch).getBytes(UTF_8);
|
||||||
|
System.out.println(
|
||||||
|
connection.sendPostRequest(
|
||||||
|
BulkDomainTransferAction.PATH, params, MediaType.PLAIN_TEXT_UTF_8, domainsList));
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private ImmutableList<String> getDomainList() throws IOException {
|
||||||
|
return domainNamesFile == null ? ImmutableList.copyOf(domains) : loadDomainsFromFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
private ImmutableList<String> loadDomainsFromFile() throws IOException {
|
||||||
|
return Splitter.on('\n')
|
||||||
|
.omitEmptyStrings()
|
||||||
|
.trimResults()
|
||||||
|
.splitToStream(Files.asCharSource(new File(domainNamesFile), UTF_8).read())
|
||||||
|
.map(DomainNameUtils::canonicalizeHostname)
|
||||||
|
.collect(toImmutableList());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -30,6 +30,7 @@ public final class RegistryTool {
|
|||||||
public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
|
public static final ImmutableMap<String, Class<? extends Command>> COMMAND_MAP =
|
||||||
new ImmutableMap.Builder<String, Class<? extends Command>>()
|
new ImmutableMap.Builder<String, Class<? extends Command>>()
|
||||||
.put("ack_poll_messages", AckPollMessagesCommand.class)
|
.put("ack_poll_messages", AckPollMessagesCommand.class)
|
||||||
|
.put("bulk_domain_transfer", BulkDomainTransferCommand.class)
|
||||||
.put("canonicalize_labels", CanonicalizeLabelsCommand.class)
|
.put("canonicalize_labels", CanonicalizeLabelsCommand.class)
|
||||||
.put("check_domain", CheckDomainCommand.class)
|
.put("check_domain", CheckDomainCommand.class)
|
||||||
.put("check_domain_claims", CheckDomainClaimsCommand.class)
|
.put("check_domain_claims", CheckDomainClaimsCommand.class)
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
// Copyright 2025 The Nomulus Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package google.registry.tools;
|
||||||
|
|
||||||
|
import static com.google.common.truth.Truth.assertThat;
|
||||||
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
|
import static org.junit.Assert.assertThrows;
|
||||||
|
import static org.mockito.ArgumentMatchers.eq;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
|
import com.google.common.io.CharSink;
|
||||||
|
import com.google.common.io.Files;
|
||||||
|
import com.google.common.net.MediaType;
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.stream.IntStream;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.mockito.ArgumentCaptor;
|
||||||
|
|
||||||
|
/** Tests fir {@link BulkDomainTransferCommand}. */
|
||||||
|
public class BulkDomainTransferCommandTest extends CommandTestCase<BulkDomainTransferCommand> {
|
||||||
|
|
||||||
|
private ServiceConnection connection;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void beforeEach() {
|
||||||
|
connection = mock(ServiceConnection.class);
|
||||||
|
command.setConnection(connection);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSuccess_validParametersSent() throws Exception {
|
||||||
|
runCommandForced(
|
||||||
|
"--gaining_registrar_id", "NewRegistrar",
|
||||||
|
"--losing_registrar_id", "TheRegistrar",
|
||||||
|
"--reason", "someReason",
|
||||||
|
"--domains", "foo.tld,bar.tld");
|
||||||
|
assertInStdout("Sending batch of 2 domains");
|
||||||
|
verify(connection)
|
||||||
|
.sendPostRequest(
|
||||||
|
"/_dr/task/bulkDomainTransfer",
|
||||||
|
ImmutableMap.of(
|
||||||
|
"gainingRegistrarId",
|
||||||
|
"NewRegistrar",
|
||||||
|
"losingRegistrarId",
|
||||||
|
"TheRegistrar",
|
||||||
|
"requestedByRegistrar",
|
||||||
|
false,
|
||||||
|
"reason",
|
||||||
|
"someReason"),
|
||||||
|
MediaType.PLAIN_TEXT_UTF_8,
|
||||||
|
"[\"foo.tld\",\"bar.tld\"]".getBytes(UTF_8));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testSuccess_fileInBatches() throws Exception {
|
||||||
|
File domainNamesFile = tmpDir.resolve("domain_names.txt").toFile();
|
||||||
|
CharSink sink = Files.asCharSink(domainNamesFile, UTF_8);
|
||||||
|
sink.writeLines(IntStream.range(0, 1003).mapToObj(i -> String.format("foo%d.tld", i)));
|
||||||
|
runCommandForced(
|
||||||
|
"--gaining_registrar_id", "NewRegistrar",
|
||||||
|
"--losing_registrar_id", "TheRegistrar",
|
||||||
|
"--reason", "someReason",
|
||||||
|
"--domain_names_file", domainNamesFile.getPath());
|
||||||
|
assertInStdout("Sending batch of 1000 domains");
|
||||||
|
assertInStdout("Sending batch of 3 domains");
|
||||||
|
ArgumentCaptor<byte[]> listCaptor = ArgumentCaptor.forClass(byte[].class);
|
||||||
|
verify(connection, times(2))
|
||||||
|
.sendPostRequest(
|
||||||
|
eq("/_dr/task/bulkDomainTransfer"),
|
||||||
|
eq(
|
||||||
|
ImmutableMap.of(
|
||||||
|
"gainingRegistrarId",
|
||||||
|
"NewRegistrar",
|
||||||
|
"losingRegistrarId",
|
||||||
|
"TheRegistrar",
|
||||||
|
"requestedByRegistrar",
|
||||||
|
false,
|
||||||
|
"reason",
|
||||||
|
"someReason")),
|
||||||
|
eq(MediaType.PLAIN_TEXT_UTF_8),
|
||||||
|
listCaptor.capture());
|
||||||
|
assertThat(listCaptor.getValue())
|
||||||
|
.isEqualTo("[\"foo1000.tld\",\"foo1001.tld\",\"foo1002.tld\"]".getBytes(UTF_8));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFailure_badGaining() {
|
||||||
|
assertThat(
|
||||||
|
assertThrows(
|
||||||
|
IllegalArgumentException.class,
|
||||||
|
() ->
|
||||||
|
runCommandForced(
|
||||||
|
"--gaining_registrar_id", "Bad",
|
||||||
|
"--losing_registrar_id", "TheRegistrar",
|
||||||
|
"--reason", "someReason",
|
||||||
|
"--domains", "foo.tld,baz.tld")))
|
||||||
|
.hasMessageThat()
|
||||||
|
.isEqualTo("Gaining registrar Bad doesn't exist");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFailure_badLosing() {
|
||||||
|
assertThat(
|
||||||
|
assertThrows(
|
||||||
|
IllegalArgumentException.class,
|
||||||
|
() ->
|
||||||
|
runCommandForced(
|
||||||
|
"--gaining_registrar_id", "NewRegistrar",
|
||||||
|
"--losing_registrar_id", "Bad",
|
||||||
|
"--reason", "someReason",
|
||||||
|
"--domains", "foo.tld,baz.tld")))
|
||||||
|
.hasMessageThat()
|
||||||
|
.isEqualTo("Losing registrar Bad doesn't exist");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFailure_noDomainsSpecified() {
|
||||||
|
assertThat(
|
||||||
|
assertThrows(
|
||||||
|
IllegalArgumentException.class,
|
||||||
|
() ->
|
||||||
|
runCommandForced(
|
||||||
|
"--gaining_registrar_id", "NewRegistrar",
|
||||||
|
"--losing_registrar_id", "TheRegistrar",
|
||||||
|
"--reason", "someReason")))
|
||||||
|
.hasMessageThat()
|
||||||
|
.isEqualTo(
|
||||||
|
"Must specify exactly one input method, either --domains or --domain_names_file");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFailure_bothDomainMethodsSpecified() {
|
||||||
|
assertThat(
|
||||||
|
assertThrows(
|
||||||
|
IllegalArgumentException.class,
|
||||||
|
() ->
|
||||||
|
runCommandForced(
|
||||||
|
"--gaining_registrar_id",
|
||||||
|
"NewRegistrar",
|
||||||
|
"--losing_registrar_id",
|
||||||
|
"TheRegistrar",
|
||||||
|
"--reason",
|
||||||
|
"someReason",
|
||||||
|
"--domains",
|
||||||
|
"foo.tld,baz.tld",
|
||||||
|
"--domain_names_file",
|
||||||
|
"foo.txt")))
|
||||||
|
.hasMessageThat()
|
||||||
|
.isEqualTo(
|
||||||
|
"Must specify exactly one input method, either --domains or --domain_names_file");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ BACKEND /_dr/task/brdaCopy BrdaCopyAction
|
|||||||
BACKEND /_dr/task/bsaDownload BsaDownloadAction GET,POST n APP ADMIN
|
BACKEND /_dr/task/bsaDownload BsaDownloadAction GET,POST n APP ADMIN
|
||||||
BACKEND /_dr/task/bsaRefresh BsaRefreshAction GET,POST n APP ADMIN
|
BACKEND /_dr/task/bsaRefresh BsaRefreshAction GET,POST n APP ADMIN
|
||||||
BACKEND /_dr/task/bsaValidate BsaValidateAction GET,POST n APP ADMIN
|
BACKEND /_dr/task/bsaValidate BsaValidateAction GET,POST n APP ADMIN
|
||||||
|
BACKEND /_dr/task/bulkDomainTransfer BulkDomainTransferAction POST n APP ADMIN
|
||||||
BACKEND /_dr/task/copyDetailReports CopyDetailReportsAction POST n APP ADMIN
|
BACKEND /_dr/task/copyDetailReports CopyDetailReportsAction POST n APP ADMIN
|
||||||
BACKEND /_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n APP ADMIN
|
BACKEND /_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n APP ADMIN
|
||||||
BACKEND /_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n APP ADMIN
|
BACKEND /_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n APP ADMIN
|
||||||
|
|||||||
Reference in New Issue
Block a user