fragment) {
final XjcRdeHost xjcHost = fragment.getInstance().getValue();
try {
- logger.infofmt("Converting xml for host %s", xjcHost.getName());
// Record number of attempted map operations
getContext().incrementCounter("host imports attempted");
logger.infofmt("Saving host %s", xjcHost.getName());
@@ -134,19 +133,13 @@ public class RdeHostImportAction implements Runnable {
logger.infofmt("Host %s was imported successfully", xjcHost.getName());
} catch (ResourceExistsException e) {
// Record the number of hosts already in the registry
- getContext().incrementCounter("hosts skipped");
+ getContext().incrementCounter("existing hosts skipped");
logger.infofmt("Host %s already exists", xjcHost.getName());
} catch (Exception e) {
// Record the number of hosts with unexpected errors
getContext().incrementCounter("host import errors");
- throw new HostImportException(xjcHost.getName(), xjcHost.toString(), e);
+ logger.severefmt(e, "Error processing host %s; xml=%s", xjcHost.getName(), xjcHost);
}
}
}
-
- private static class HostImportException extends RuntimeException {
- HostImportException(String hostName, String xml, Throwable cause) {
- super(String.format("Error processing host %s; xml=%s", hostName, xml), cause);
- }
- }
}
diff --git a/java/google/registry/rde/imports/RdeHostLinkAction.java b/java/google/registry/rde/imports/RdeHostLinkAction.java
new file mode 100644
index 000000000..43cdc5c40
--- /dev/null
+++ b/java/google/registry/rde/imports/RdeHostLinkAction.java
@@ -0,0 +1,168 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rde.imports;
+
+import static com.google.common.base.Preconditions.checkState;
+import static google.registry.mapreduce.MapreduceRunner.PARAM_MAP_SHARDS;
+import static google.registry.model.EppResourceUtils.loadByForeignKey;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.model.registry.Registries.findTldForName;
+import static google.registry.util.PipelineUtils.createJobPath;
+
+import com.google.appengine.tools.mapreduce.Mapper;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.net.InternetDomainName;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.VoidWork;
+import google.registry.config.RegistryConfig.Config;
+import google.registry.mapreduce.MapreduceRunner;
+import google.registry.model.domain.DomainResource;
+import google.registry.model.host.HostResource;
+import google.registry.request.Action;
+import google.registry.request.Parameter;
+import google.registry.request.Response;
+import google.registry.util.FormattingLogger;
+import google.registry.xjc.JaxbFragment;
+import google.registry.xjc.rdehost.XjcRdeHost;
+import google.registry.xjc.rdehost.XjcRdeHostElement;
+import javax.inject.Inject;
+import org.joda.time.DateTime;
+
+/**
+ * A mapreduce that links hosts from an escrow file to their superordinate domains.
+ *
+ * This mapreduce is run as the last step of the process of importing escrow files. For each host
+ * in the escrow file, the corresponding {@link HostResource} record in the datastore is linked to
+ * its superordinate {@link DomainResource} only if it is an in-zone host. This is necessary because
+ * all hosts must exist before domains can be imported, due to references in host objects, and
+ * domains must exist before hosts can be linked to their superordinate domains.
+ *
+ *
Specify the escrow file to import with the "path" parameter.
+ */
+@Action(path = "/_dr/task/linkRdeHosts")
+public class RdeHostLinkAction implements Runnable {
+
+ private static final FormattingLogger logger = FormattingLogger.getLoggerForCallerClass();
+
+ private final MapreduceRunner mrRunner;
+ private final Response response;
+ private final String importBucketName;
+ private final String importFileName;
+ private final Optional mapShards;
+
+ @Inject
+ public RdeHostLinkAction(
+ MapreduceRunner mrRunner,
+ Response response,
+ @Config("rdeImportBucket") String importBucketName,
+ @Parameter("path") String importFileName,
+ @Parameter(PARAM_MAP_SHARDS) Optional mapShards) {
+ this.mrRunner = mrRunner;
+ this.response = response;
+ this.importBucketName = importBucketName;
+ this.importFileName = importFileName;
+ this.mapShards = mapShards;
+ }
+
+ @Override
+ public void run() {
+ response.sendJavaScriptRedirect(createJobPath(mrRunner
+ .setJobName("Link hosts from escrow file")
+ .setModuleName("backend")
+ .runMapOnly(
+ new RdeHostPostImportMapper(),
+ ImmutableList.of(new RdeHostInput(mapShards, importBucketName, importFileName)))));
+ }
+
+ /** Mapper to link hosts from an escrow file to their superordinate domains. */
+ public static class RdeHostPostImportMapper
+ extends Mapper, Void, Void> {
+
+ private static final long serialVersionUID = -2898753709127134419L;
+
+ @Override
+ public void map(JaxbFragment fragment) {
+ // Record number of attempted map operations
+ getContext().incrementCounter("post-import hosts read");
+ final XjcRdeHost xjcHost = fragment.getInstance().getValue();
+ logger.infofmt("Attempting to link superordinate domain for host %s", xjcHost.getName());
+ try {
+ InternetDomainName hostName = InternetDomainName.from(xjcHost.getName());
+ Optional superordinateDomain =
+ lookupSuperordinateDomain(hostName, DateTime.now());
+ // if suporordinateDomain is null, this is an out of zone host and can't be linked
+ if (!superordinateDomain.isPresent()) {
+ getContext().incrementCounter("post-import hosts out of zone");
+ logger.infofmt("Host %s is out of zone", xjcHost.getName());
+ return;
+ }
+ // at this point, the host is definitely in zone and should be linked
+ getContext().incrementCounter("post-import hosts in zone");
+ final Key superordinateDomainKey = Key.create(superordinateDomain.get());
+ ofy().transact(new VoidWork() {
+ @Override
+ public void vrun() {
+ HostResource host =
+ ofy().load().now(Key.create(HostResource.class, xjcHost.getRoid()));
+ ofy().save()
+ .entity(host.asBuilder().setSuperordinateDomain(superordinateDomainKey).build());
+ }
+ });
+ logger.infofmt(
+ "Successfully linked host %s to superordinate domain %s",
+ xjcHost.getName(),
+ superordinateDomain.get().getFullyQualifiedDomainName());
+ // Record number of hosts successfully linked
+ getContext().incrementCounter("post-import hosts linked");
+ } catch (Exception e) {
+ // Record the number of hosts with unexpected errors
+ getContext().incrementCounter("post-import host errors");
+ throw new HostLinkException(xjcHost.getName(), xjcHost.toString(), e);
+ }
+ }
+
+ /**
+ * Return the {@link DomainResource} this host is subordinate to, or absent for out of zone
+ * hosts.
+ *
+ * @throws IllegalStateException for hosts without superordinate domains
+ */
+ private static Optional lookupSuperordinateDomain(
+ InternetDomainName hostName, DateTime now) {
+ Optional tld = findTldForName(hostName);
+ // out of zone hosts cannot be linked
+ if (!tld.isPresent()) {
+ return Optional.absent();
+ }
+ // This is a subordinate host
+ String domainName = Joiner.on('.').join(Iterables.skip(
+ hostName.parts(), hostName.parts().size() - (tld.get().parts().size() + 1)));
+ DomainResource superordinateDomain = loadByForeignKey(DomainResource.class, domainName, now);
+ // Hosts can't be linked if domains import hasn't been run
+ checkState(
+ superordinateDomain != null, "Superordinate domain does not exist: %s", domainName);
+ return Optional.of(superordinateDomain);
+ }
+ }
+
+ private static class HostLinkException extends RuntimeException {
+ HostLinkException(String hostname, String xml, Throwable cause) {
+ super(String.format("Error linking host %s; xml=%s", hostname, xml), cause);
+ }
+ }
+}
diff --git a/java/google/registry/rde/imports/XjcToDomainResourceConverter.java b/java/google/registry/rde/imports/XjcToDomainResourceConverter.java
index a37d9ef2b..fd3a55888 100644
--- a/java/google/registry/rde/imports/XjcToDomainResourceConverter.java
+++ b/java/google/registry/rde/imports/XjcToDomainResourceConverter.java
@@ -20,6 +20,7 @@ import static com.google.common.collect.Iterables.transform;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
import static google.registry.util.DateTimeUtils.END_OF_TIME;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import static google.registry.util.PreconditionsUtils.checkArgumentNotNull;
import com.google.common.base.Ascii;
@@ -97,6 +98,8 @@ final class XjcToDomainResourceConverter extends XjcToEppResourceConverter {
new Function>() {
@Override
public Key apply(String fullyQualifiedHostName) {
+ // host names are always lower case
+ fullyQualifiedHostName = canonicalizeDomainName(fullyQualifiedHostName);
Key key =
ForeignKeyIndex.loadAndGetKey(
HostResource.class, fullyQualifiedHostName, DateTime.now());
@@ -193,7 +196,7 @@ final class XjcToDomainResourceConverter extends XjcToEppResourceConverter {
new GracePeriodConverter(domain, Key.create(autoRenewBillingEvent));
DomainResource.Builder builder =
new DomainResource.Builder()
- .setFullyQualifiedDomainName(domain.getName())
+ .setFullyQualifiedDomainName(canonicalizeDomainName(domain.getName()))
.setRepoId(domain.getRoid())
.setIdnTableName(domain.getIdnTableId())
.setCurrentSponsorClientId(domain.getClID())
diff --git a/java/google/registry/rde/imports/XjcToHostResourceConverter.java b/java/google/registry/rde/imports/XjcToHostResourceConverter.java
index 40253cc17..89f3a71fb 100644
--- a/java/google/registry/rde/imports/XjcToHostResourceConverter.java
+++ b/java/google/registry/rde/imports/XjcToHostResourceConverter.java
@@ -18,6 +18,7 @@ import static com.google.common.base.Predicates.equalTo;
import static com.google.common.base.Predicates.not;
import static google.registry.model.ofy.ObjectifyService.ofy;
import static google.registry.rde.imports.RdeImportUtils.generateTridForImport;
+import static google.registry.util.DomainNameUtils.canonicalizeDomainName;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
@@ -69,7 +70,7 @@ public class XjcToHostResourceConverter extends XjcToEppResourceConverter {
.setParent(Key.create(null, HostResource.class, host.getRoid()))
.build());
return new HostResource.Builder()
- .setFullyQualifiedHostName(host.getName())
+ .setFullyQualifiedHostName(canonicalizeDomainName(host.getName()))
.setRepoId(host.getRoid())
.setCurrentSponsorClientId(host.getClID())
.setLastTransferTime(host.getTrDate())
diff --git a/javatests/google/registry/rde/imports/RdeHostImportActionTest.java b/javatests/google/registry/rde/imports/RdeHostImportActionTest.java
index d1645be59..b26b9d8a6 100644
--- a/javatests/google/registry/rde/imports/RdeHostImportActionTest.java
+++ b/javatests/google/registry/rde/imports/RdeHostImportActionTest.java
@@ -16,6 +16,7 @@ package google.registry.rde.imports;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.testing.DatastoreHelper.createTld;
import static google.registry.testing.DatastoreHelper.getHistoryEntries;
import static google.registry.testing.DatastoreHelper.newHostResource;
import static google.registry.testing.DatastoreHelper.persistResource;
@@ -66,6 +67,7 @@ public class RdeHostImportActionTest extends MapreduceTestCaseabsent(), Optional.absent());
action = new RdeHostImportAction(
diff --git a/javatests/google/registry/rde/imports/RdeHostLinkActionTest.java b/javatests/google/registry/rde/imports/RdeHostLinkActionTest.java
new file mode 100644
index 000000000..9967aa393
--- /dev/null
+++ b/javatests/google/registry/rde/imports/RdeHostLinkActionTest.java
@@ -0,0 +1,108 @@
+// Copyright 2017 The Nomulus Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package google.registry.rde.imports;
+
+import static com.google.common.truth.Truth.assertThat;
+import static google.registry.model.ofy.ObjectifyService.ofy;
+import static google.registry.testing.DatastoreHelper.createTld;
+import static google.registry.testing.DatastoreHelper.newHostResource;
+import static google.registry.testing.DatastoreHelper.persistActiveDomain;
+import static google.registry.testing.DatastoreHelper.persistResource;
+import static google.registry.util.DateTimeUtils.END_OF_TIME;
+
+import com.google.appengine.tools.cloudstorage.GcsFilename;
+import com.google.appengine.tools.cloudstorage.GcsService;
+import com.google.appengine.tools.cloudstorage.GcsServiceFactory;
+import com.google.appengine.tools.cloudstorage.RetryParams;
+import com.google.common.base.Optional;
+import com.google.common.io.ByteSource;
+import com.google.common.io.ByteStreams;
+import com.googlecode.objectify.Key;
+import google.registry.config.RegistryConfig.ConfigModule;
+import google.registry.gcs.GcsUtils;
+import google.registry.mapreduce.MapreduceRunner;
+import google.registry.model.domain.DomainResource;
+import google.registry.model.host.HostResource;
+import google.registry.model.index.ForeignKeyIndex.ForeignKeyDomainIndex;
+import google.registry.request.Response;
+import google.registry.testing.FakeResponse;
+import google.registry.testing.mapreduce.MapreduceTestCase;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link RdeHostLinkAction}. */
+@RunWith(JUnit4.class)
+public class RdeHostLinkActionTest extends MapreduceTestCase {
+
+ private static final ByteSource DEPOSIT_1_HOST = RdeImportsTestData.get("deposit_1_host.xml");
+ private static final String IMPORT_BUCKET_NAME = "import-bucket";
+ private static final String IMPORT_FILE_NAME = "escrow-file.xml";
+
+ private static final GcsService GCS_SERVICE =
+ GcsServiceFactory.createGcsService(RetryParams.getDefaultInstance());
+
+ private MapreduceRunner mrRunner;
+
+ private Response response;
+
+ private final Optional mapShards = Optional.absent();
+
+ @Before
+ public void before() throws Exception {
+ createTld("test");
+ response = new FakeResponse();
+ mrRunner = new MapreduceRunner(Optional.absent(), Optional.absent());
+ action =
+ new RdeHostLinkAction(mrRunner, response, IMPORT_BUCKET_NAME, IMPORT_FILE_NAME, mapShards);
+ }
+
+ @Test
+ public void test_mapreduceSuccessfullyLinksHost() throws Exception {
+ // Create host and domain first
+ persistResource(
+ newHostResource("ns1.example1.test")
+ .asBuilder()
+ .setRepoId("Hns1_example1_test-TEST")
+ .build());
+ DomainResource superordinateDomain = persistActiveDomain("example1.test");
+ ForeignKeyDomainIndex.create(superordinateDomain, END_OF_TIME);
+ Key superOrdinateDomainKey = Key.create(superordinateDomain);
+ pushToGcs(DEPOSIT_1_HOST);
+ runMapreduce();
+ List hosts = ofy().load().type(HostResource.class).list();
+ assertThat(hosts).hasSize(1);
+ assertThat(hosts.get(0).getSuperordinateDomain()).isEqualTo(superOrdinateDomainKey);
+ }
+
+ private void runMapreduce() throws Exception {
+ action.run();
+ executeTasksUntilEmpty("mapreduce");
+ }
+
+ private static void pushToGcs(ByteSource source) throws IOException {
+ try (OutputStream outStream =
+ new GcsUtils(GCS_SERVICE, ConfigModule.provideGcsBufferSize())
+ .openOutputStream(new GcsFilename(IMPORT_BUCKET_NAME, IMPORT_FILE_NAME));
+ InputStream inStream = source.openStream()) {
+ ByteStreams.copy(inStream, outStream);
+ }
+ }
+}
diff --git a/javatests/google/registry/rde/imports/XjcToDomainResourceConverterTest.java b/javatests/google/registry/rde/imports/XjcToDomainResourceConverterTest.java
index 526ce216a..607045a9e 100644
--- a/javatests/google/registry/rde/imports/XjcToDomainResourceConverterTest.java
+++ b/javatests/google/registry/rde/imports/XjcToDomainResourceConverterTest.java
@@ -132,6 +132,16 @@ public class XjcToDomainResourceConverterTest {
assertThat(domain.getAuthInfo().getPw().getValue()).isEqualTo("0123456789abcdef");
}
+ /** Verifies that uppercase domain names are converted to lowercase */
+ @Test
+ public void testConvertDomainResourceUpperCase() throws Exception {
+ persistActiveContact("jd1234");
+ persistActiveContact("sh8013");
+ final XjcRdeDomain xjcDomain = loadDomainFromRdeXml("domain_fragment_ucase.xml");
+ DomainResource domain = convertDomainInTransaction(xjcDomain);
+ assertThat(domain.getFullyQualifiedDomainName()).isEqualTo("example1.example");
+ }
+
@Test
public void testConvertDomainResourceAddPeriod() throws Exception {
persistActiveContact("jd1234");
diff --git a/javatests/google/registry/rde/imports/XjcToHostResourceConverterTest.java b/javatests/google/registry/rde/imports/XjcToHostResourceConverterTest.java
index 76c63540b..271a1ef86 100644
--- a/javatests/google/registry/rde/imports/XjcToHostResourceConverterTest.java
+++ b/javatests/google/registry/rde/imports/XjcToHostResourceConverterTest.java
@@ -51,6 +51,8 @@ import org.junit.runners.JUnit4;
public class XjcToHostResourceConverterTest extends ShardableTestCase {
private static final ByteSource HOST_XML = RdeImportsTestData.get("host_fragment.xml");
+ private static final ByteSource HOST_XML_UCASE =
+ RdeImportsTestData.get("host_fragment_ucase.xml");
// List of packages to initialize JAXBContext
private static final String JAXB_CONTEXT_PACKAGES = Joiner.on(":")
@@ -103,6 +105,14 @@ public class XjcToHostResourceConverterTest extends ShardableTestCase {
assertThat(host.getLastTransferTime()).isEqualTo(DateTime.parse("2008-10-03T09:34:00.0Z"));
}
+ /** Verifies that uppercase host names are converted to lowercase */
+ @Test
+ public void testConvertHostResourceUpperCase() throws Exception {
+ XjcRdeHost xjcHost = loadHostFromRdeXml(HOST_XML_UCASE);
+ HostResource host = convertHostInTransaction(xjcHost);
+ assertThat(host.getFullyQualifiedHostName()).isEqualTo("ns1.example1.test");
+ }
+
@Test
public void testConvertHostResourceHistoryEntry() throws Exception {
XjcRdeHost xjcHost = loadHostFromRdeXml();
@@ -134,7 +144,11 @@ public class XjcToHostResourceConverterTest extends ShardableTestCase {
}
private XjcRdeHost loadHostFromRdeXml() throws Exception {
- try (InputStream ins = HOST_XML.openStream()) {
+ return loadHostFromRdeXml(HOST_XML);
+ }
+
+ private XjcRdeHost loadHostFromRdeXml(ByteSource source) throws Exception {
+ try (InputStream ins = source.openStream()) {
return ((XjcRdeHostElement) unmarshaller.unmarshal(ins)).getValue();
}
}
diff --git a/javatests/google/registry/rde/imports/testdata/domain_fragment_host_objs.xml b/javatests/google/registry/rde/imports/testdata/domain_fragment_host_objs.xml
index 9395c05d7..74ad00dea 100644
--- a/javatests/google/registry/rde/imports/testdata/domain_fragment_host_objs.xml
+++ b/javatests/google/registry/rde/imports/testdata/domain_fragment_host_objs.xml
@@ -9,7 +9,7 @@
sh8013
ns1.example.net
- ns2.example.net
+ NS2.EXAMPLE.NET
RegistrarX
RegistrarX
diff --git a/javatests/google/registry/rde/imports/testdata/domain_fragment_ucase.xml b/javatests/google/registry/rde/imports/testdata/domain_fragment_ucase.xml
new file mode 100644
index 000000000..1cb1f979c
--- /dev/null
+++ b/javatests/google/registry/rde/imports/testdata/domain_fragment_ucase.xml
@@ -0,0 +1,14 @@
+
+ EXAMPLE1.EXAMPLE
+ Dexample1-TEST
+
+ jd1234
+ sh8013
+ sh8013
+ RegistrarX
+ RegistrarX
+ 1999-04-03T22:00:00.0Z
+ 2015-04-03T22:00:00.0Z
+
diff --git a/javatests/google/registry/rde/imports/testdata/host_fragment_ucase.xml b/javatests/google/registry/rde/imports/testdata/host_fragment_ucase.xml
new file mode 100644
index 000000000..6d09021a9
--- /dev/null
+++ b/javatests/google/registry/rde/imports/testdata/host_fragment_ucase.xml
@@ -0,0 +1,15 @@
+
+ NS1.EXAMPLE1.TEST
+ Hns1_example1_test-TEST
+
+
+ 192.0.2.2
+ 192.0.2.29
+ 1080:0:0:0:8:800:200C:417A
+ RegistrarX
+ RegistrarX
+ 1999-05-08T12:10:00.0Z
+ RegistrarX
+ 2009-10-03T09:34:00.0Z
+ 2008-10-03T09:34:00.0Z
+