From b224a90a4c64550833cb774b6e8b74794104b30f Mon Sep 17 00:00:00 2001 From: mountford Date: Sun, 24 Jul 2016 11:02:45 -0400 Subject: [PATCH] RDAP: Display truncation notice for large entity result sets The ICAAN Operational Profile dictates that a notice be added to the RDAP search results response when there are more objects than the server's chosen result set size. This CL (hopefully the last one) handles the fixes for entity (contact and registrar) searches. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=135494283 --- .../registry/rdap/RdapEntitySearchAction.java | 52 ++-- .../rdap/RdapNameserverSearchAction.java | 3 +- .../rdap/RdapEntitySearchActionTest.java | 102 ++++++- .../rdap/testdata/rdap_multiple_contacts.json | 2 +- .../testdata/rdap_multiple_contacts2.json | 2 +- .../testdata/rdap_nontruncated_contacts.json | 227 +++++++++++++++ .../rdap_nontruncated_registrars.json | 255 +++++++++++++++++ .../testdata/rdap_truncated_contacts.json | 235 ++++++++++++++++ .../rdap_truncated_mixed_entities.json | 242 ++++++++++++++++ .../testdata/rdap_truncated_registrars.json | 263 ++++++++++++++++++ 10 files changed, 1354 insertions(+), 29 deletions(-) create mode 100644 javatests/google/registry/rdap/testdata/rdap_nontruncated_contacts.json create mode 100644 javatests/google/registry/rdap/testdata/rdap_nontruncated_registrars.json create mode 100644 javatests/google/registry/rdap/testdata/rdap_truncated_contacts.json create mode 100644 javatests/google/registry/rdap/testdata/rdap_truncated_mixed_entities.json create mode 100644 javatests/google/registry/rdap/testdata/rdap_truncated_registrars.json diff --git a/java/google/registry/rdap/RdapEntitySearchAction.java b/java/google/registry/rdap/RdapEntitySearchAction.java index 82270be6e..bb95f4964 100644 --- a/java/google/registry/rdap/RdapEntitySearchAction.java +++ b/java/google/registry/rdap/RdapEntitySearchAction.java @@ -15,6 +15,7 @@ package google.registry.rdap; import static google.registry.model.ofy.ObjectifyService.ofy; +import static google.registry.rdap.RdapIcannStandardInformation.TRUNCATION_NOTICES; import static google.registry.request.Action.Method.GET; import static google.registry.request.Action.Method.HEAD; import static google.registry.util.DateTimeUtils.END_OF_TIME; @@ -85,7 +86,7 @@ public class RdapEntitySearchAction extends RdapActionBase { if (Booleans.countTrue(fnParam.isPresent(), handleParam.isPresent()) != 1) { throw new BadRequestException("You must specify either fn=XXXX or handle=YYYY"); } - ImmutableList> results; + RdapSearchResults results; if (fnParam.isPresent()) { // syntax: /rdap/entities?fn=Bobby%20Joe* // The name is the contact name or registrar name (not registrar contact name). @@ -95,15 +96,16 @@ public class RdapEntitySearchAction extends RdapActionBase { // The handle is either the contact roid or the registrar clientId. results = searchByHandle(RdapSearchPattern.create(handleParam.get(), false), now); } - if (results.isEmpty()) { + if (results.jsonList().isEmpty()) { throw new NotFoundException("No entities found"); } ImmutableMap.Builder jsonBuilder = new ImmutableMap.Builder<>(); - jsonBuilder.put("entitySearchResults", results); + jsonBuilder.put("entitySearchResults", results.jsonList()); RdapJsonFormatter.addTopLevelEntries( jsonBuilder, BoilerplateType.ENTITY, - ImmutableList.>of(), + results.isTruncated() + ? TRUNCATION_NOTICES : ImmutableList.>of(), ImmutableList.>of(), rdapLinkBase); return jsonBuilder.build(); @@ -123,8 +125,7 @@ public class RdapEntitySearchAction extends RdapActionBase { *

According to RFC 7482 section 6.1, punycode is only used for domain name labels, so we can * assume that entity names are regular unicode. */ - private ImmutableList> - searchByName(final RdapSearchPattern partialStringQuery, DateTime now) { + private RdapSearchResults searchByName(final RdapSearchPattern partialStringQuery, DateTime now) { // Don't allow suffixes in entity name search queries. if (!partialStringQuery.getHasWildcard() && (partialStringQuery.getSuffix() != null)) { throw new UnprocessableEntityException("Suffixes not allowed in entity name searches"); @@ -137,21 +138,23 @@ public class RdapEntitySearchAction extends RdapActionBase { ? ImmutableList.of() : ImmutableList.of(registrar); } else { + // Fetch an additional registrar, so we can detect result set truncation. registrarMatches = ImmutableList.copyOf(Registrar.loadByNameRange( partialStringQuery.getInitialString(), partialStringQuery.getNextInitialString(), - rdapResultSetMaxSize)); + rdapResultSetMaxSize + 1)); } - // Get the contact matches and return the results. + // Get the contact matches and return the results, fetching an additional contact to detect + // truncation. return makeSearchResults( queryUndeleted( - ContactResource.class, "searchName", partialStringQuery, rdapResultSetMaxSize).list(), + ContactResource.class, "searchName", partialStringQuery, rdapResultSetMaxSize + 1).list(), registrarMatches, now); } /** Searches for entities by handle, returning a JSON array of entity info maps. */ - private ImmutableList> searchByHandle( + private RdapSearchResults searchByHandle( final RdapSearchPattern partialStringQuery, DateTime now) { // Handle queries without a wildcard -- load by ID. if (!partialStringQuery.getHasWildcard()) { @@ -170,7 +173,7 @@ public class RdapEntitySearchAction extends RdapActionBase { // worry about deletion times in the future. That allows us to use an equality query for the // deletion time. Because the handle for registrars is the IANA identifier number, don't allow // wildcard searches for registrars, by simply not searching for registrars if a wildcard is - // present. + // present. Fetch an extra contact to detect result set truncation. } else if (partialStringQuery.getSuffix() == null) { return makeSearchResults( ofy().load() @@ -180,7 +183,7 @@ public class RdapEntitySearchAction extends RdapActionBase { .filterKey( "<", Key.create(ContactResource.class, partialStringQuery.getNextInitialString())) .filter("deletionTime", END_OF_TIME) - .limit(rdapResultSetMaxSize) + .limit(rdapResultSetMaxSize + 1) .list(), ImmutableList.of(), now); @@ -199,21 +202,20 @@ public class RdapEntitySearchAction extends RdapActionBase { } catch (NumberFormatException e) { return ImmutableList.of(); } + // Fetch an additional registrar to detect result set truncation. return ImmutableList.copyOf(Registrar.loadByIanaIdentifierRange( - ianaIdentifier, ianaIdentifier + 1, rdapResultSetMaxSize)); + ianaIdentifier, ianaIdentifier + 1, rdapResultSetMaxSize + 1)); } /** Builds a JSON array of entity info maps based on the specified contacts and registrars. */ - private ImmutableList> makeSearchResults( - List contacts, - List registrars, - DateTime now) { + private RdapSearchResults makeSearchResults( + List contacts, List registrars, DateTime now) { // Determine what output data type to use, depending on whether more than one entity will be // returned. int numEntities = contacts.size(); OutputDataType outputDataType; - // If there's more than one contact, then we know already that we need SUMMARY mode. + // If there's more than one contact, then we know already we need SUMMARY mode. if (numEntities > 1) { outputDataType = OutputDataType.SUMMARY; // If there are fewer than two contacts, loop through and compute the total number of contacts @@ -231,11 +233,13 @@ public class RdapEntitySearchAction extends RdapActionBase { } } + // There can be more results than our max size, partially because we have two pools to draw from + // (contacts and registrars), and partially because we try to fetch one more than the max size, + // so we can tell whether to display the truncation notification. List> jsonOutputList = new ArrayList<>(); - // In theory, there could be more results than our max size, so limit the size. for (ContactResource contact : contacts) { if (jsonOutputList.size() >= rdapResultSetMaxSize) { - break; + return RdapSearchResults.create(ImmutableList.copyOf(jsonOutputList), true); } // As per Andy Newton on the regext mailing list, contacts by themselves have no role, since // they are global, and might have different roles for different domains. @@ -249,14 +253,14 @@ public class RdapEntitySearchAction extends RdapActionBase { outputDataType)); } for (Registrar registrar : registrars) { - if (jsonOutputList.size() >= rdapResultSetMaxSize) { - break; - } if (registrar.isActiveAndPubliclyVisible()) { + if (jsonOutputList.size() >= rdapResultSetMaxSize) { + return RdapSearchResults.create(ImmutableList.copyOf(jsonOutputList), true); + } jsonOutputList.add(RdapJsonFormatter.makeRdapJsonForRegistrar( registrar, false, rdapLinkBase, rdapWhoisServer, now, outputDataType)); } } - return ImmutableList.copyOf(jsonOutputList); + return RdapSearchResults.create(ImmutableList.copyOf(jsonOutputList)); } } diff --git a/java/google/registry/rdap/RdapNameserverSearchAction.java b/java/google/registry/rdap/RdapNameserverSearchAction.java index 5838b76d1..65bc5332e 100644 --- a/java/google/registry/rdap/RdapNameserverSearchAction.java +++ b/java/google/registry/rdap/RdapNameserverSearchAction.java @@ -128,8 +128,7 @@ public class RdapNameserverSearchAction extends RdapActionBase { return RdapSearchResults.create( ImmutableList.of( RdapJsonFormatter.makeRdapJsonForHost( - hostResource, false, rdapLinkBase, rdapWhoisServer, now, OutputDataType.FULL)), - false); + hostResource, false, rdapLinkBase, rdapWhoisServer, now, OutputDataType.FULL))); // Handle queries with a wildcard, but no suffix. There are no pending deletes for hosts, so we // can call queryUndeleted. } else if (partialStringQuery.getSuffix() == null) { diff --git a/javatests/google/registry/rdap/RdapEntitySearchActionTest.java b/javatests/google/registry/rdap/RdapEntitySearchActionTest.java index 06cfcdae4..0d5dedc72 100644 --- a/javatests/google/registry/rdap/RdapEntitySearchActionTest.java +++ b/javatests/google/registry/rdap/RdapEntitySearchActionTest.java @@ -17,6 +17,7 @@ package google.registry.rdap; import static com.google.common.truth.Truth.assertThat; import static google.registry.testing.DatastoreHelper.createTld; import static google.registry.testing.DatastoreHelper.persistResource; +import static google.registry.testing.DatastoreHelper.persistResources; import static google.registry.testing.DatastoreHelper.persistSimpleResources; import static google.registry.testing.FullFieldsTestEntityHelper.makeAndPersistContactResource; import static google.registry.testing.FullFieldsTestEntityHelper.makeContactResource; @@ -27,6 +28,7 @@ import static google.registry.testing.TestDataHelper.loadFileWithSubstitutions; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import google.registry.model.ImmutableObject; import google.registry.model.contact.ContactResource; import google.registry.model.ofy.Ofy; import google.registry.model.registrar.Registrar; @@ -34,6 +36,8 @@ import google.registry.testing.AppEngineRule; import google.registry.testing.FakeClock; import google.registry.testing.FakeResponse; import google.registry.testing.InjectRule; +import java.util.List; +import java.util.Map; import javax.annotation.Nullable; import org.joda.time.DateTime; import org.json.simple.JSONValue; @@ -120,7 +124,7 @@ public class RdapEntitySearchActionTest { action.clock = clock; action.requestPath = RdapEntitySearchAction.PATH; action.response = response; - action.rdapResultSetMaxSize = 100; + action.rdapResultSetMaxSize = 4; action.rdapLinkBase = "https://example.com/rdap/"; action.rdapWhoisServer = null; action.fnParam = Optional.absent(); @@ -177,6 +181,38 @@ public class RdapEntitySearchActionTest { return builder.build(); } + private void createManyContactsAndRegistrars(int numContacts, int numRegistrars) { + ImmutableList.Builder resourcesBuilder = new ImmutableList.Builder<>(); + for (int i = 1; i <= numContacts; i++) { + resourcesBuilder.add(makeContactResource( + String.format("contact%d", i), + String.format("Entity %d", i), + String.format("contact%d@gmail.com", i))); + } + persistResources(resourcesBuilder.build()); + for (int i = 1; i <= numRegistrars; i++) { + resourcesBuilder.add( + makeRegistrar( + String.format("registrar%d", i), + String.format("Entity %d", i + numContacts), + Registrar.State.ACTIVE, + 300L + i)); + } + persistResources(resourcesBuilder.build()); + } + + private void checkNumberOfEntitiesInResult(Object obj, int expected) { + assertThat(obj).isInstanceOf(Map.class); + + @SuppressWarnings("unchecked") + Map map = (Map) obj; + + @SuppressWarnings("unchecked") + List domains = (List) map.get("entitySearchResults"); + + assertThat(domains).hasSize(expected); + } + @Test public void testInvalidPath_rejected() throws Exception { action.requestPath = RdapEntitySearchAction.PATH + "/path"; @@ -274,6 +310,62 @@ public class RdapEntitySearchActionTest { assertThat(response.getStatus()).isEqualTo(200); } + @Test + public void testNameMatch_nonTruncatedContacts() throws Exception { + createManyContactsAndRegistrars(4, 0); + assertThat(generateActualJsonWithFullName("Entity *")) + .isEqualTo(generateExpectedJson("rdap_nontruncated_contacts.json")); + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + public void testNameMatch_truncatedContacts() throws Exception { + createManyContactsAndRegistrars(5, 0); + assertThat(generateActualJsonWithFullName("Entity *")) + .isEqualTo(generateExpectedJson("rdap_truncated_contacts.json")); + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + public void testNameMatch_reallyTruncatedContacts() throws Exception { + createManyContactsAndRegistrars(9, 0); + assertThat(generateActualJsonWithFullName("Entity *")) + .isEqualTo(generateExpectedJson("rdap_truncated_contacts.json")); + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + public void testNameMatch_nonTruncatedRegistrars() throws Exception { + createManyContactsAndRegistrars(0, 4); + assertThat(generateActualJsonWithFullName("Entity *")) + .isEqualTo(generateExpectedJson("rdap_nontruncated_registrars.json")); + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + public void testNameMatch_truncatedRegistrars() throws Exception { + createManyContactsAndRegistrars(0, 5); + assertThat(generateActualJsonWithFullName("Entity *")) + .isEqualTo(generateExpectedJson("rdap_truncated_registrars.json")); + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + public void testNameMatch_reallyTruncatedRegistrars() throws Exception { + createManyContactsAndRegistrars(0, 9); + assertThat(generateActualJsonWithFullName("Entity *")) + .isEqualTo(generateExpectedJson("rdap_truncated_registrars.json")); + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + public void testNameMatch_truncatedMixOfContactsAndRegistrars() throws Exception { + createManyContactsAndRegistrars(3, 3); + assertThat(generateActualJsonWithFullName("Entity *")) + .isEqualTo(generateExpectedJson("rdap_truncated_mixed_entities.json")); + assertThat(response.getStatus()).isEqualTo(200); + } + @Test public void testHandleMatch_2roid_found() throws Exception { assertThat(generateActualJsonWithHandle("2-ROID")) @@ -352,4 +444,12 @@ public class RdapEntitySearchActionTest { generateActualJsonWithHandle("3test*"); assertThat(response.getStatus()).isEqualTo(404); } + + @Test + public void testHandleMatch_truncatedEntities() throws Exception { + createManyContactsAndRegistrars(300, 0); + Object obj = generateActualJsonWithHandle("10*"); + assertThat(response.getStatus()).isEqualTo(200); + checkNumberOfEntitiesInResult(obj, 4); + } } diff --git a/javatests/google/registry/rdap/testdata/rdap_multiple_contacts.json b/javatests/google/registry/rdap/testdata/rdap_multiple_contacts.json index a1bc23748..12343c0b4 100644 --- a/javatests/google/registry/rdap/testdata/rdap_multiple_contacts.json +++ b/javatests/google/registry/rdap/testdata/rdap_multiple_contacts.json @@ -102,7 +102,7 @@ } ], "rdapConformance": [ "rdap_level_0" ], - "notices" : + "notices" : [ { "title" : "RDAP Terms of Service", diff --git a/javatests/google/registry/rdap/testdata/rdap_multiple_contacts2.json b/javatests/google/registry/rdap/testdata/rdap_multiple_contacts2.json index f99fa92a6..e1fd79fab 100644 --- a/javatests/google/registry/rdap/testdata/rdap_multiple_contacts2.json +++ b/javatests/google/registry/rdap/testdata/rdap_multiple_contacts2.json @@ -95,7 +95,7 @@ } ], "rdapConformance": [ "rdap_level_0" ], - "notices" : + "notices" : [ { "title" : "RDAP Terms of Service", diff --git a/javatests/google/registry/rdap/testdata/rdap_nontruncated_contacts.json b/javatests/google/registry/rdap/testdata/rdap_nontruncated_contacts.json new file mode 100644 index 000000000..7addc2d91 --- /dev/null +++ b/javatests/google/registry/rdap/testdata/rdap_nontruncated_contacts.json @@ -0,0 +1,227 @@ +{ + "entitySearchResults": + [ + { + "objectClassName" : "entity", + "handle" : "7-ROID", + "status" : ["active"], + "links" : + [ + { + "value" : "https://example.com/rdap/entity/7-ROID", + "rel" : "self", + "href": "https://example.com/rdap/entity/7-ROID", + "type" : "application/rdap+json" + } + ], + "remarks": [ + { + "title": "Incomplete Data", + "description": [ + "Summary data only. For complete data, send a specific query for the object." + ], + "type": "object truncated due to unexplainable reasons" + } + ], + "vcardArray" : + [ + "vcard", + [ + ["version", {}, "text", "4.0"], + ["fn", {}, "text", "Entity 1"], + ["org", {}, "text", "GOOGLE INCORPORATED