mirror of
https://github.com/google/nomulus
synced 2026-09-01 05:37:08 +00:00
Remove URLFetch (#2181)
We previously needed to use URLFetch in some instances where TLS 1.3 is required (mostly when connecting to ICANN servers),and the JDK-bundled SSL engine that came with App Engine runtime did not support TLS 1.3. It appears now that the Java 8 runtime on App Engine supports TLS 1.3 out of the box, which allows us to get rid of URLFetch, which depends on App Engine APIs. Also removed some redundant retry and logging logic, now that we know the HTTP client behaves correctly. TESTED=modified the CannedScriptExecutionAction and deployed to alpha, used the new HTTP client to connect to the three URL endpoints that were problematic before and confirmed that TLS connections can be established. HTTP sessions were rejected in some cases when authentication failed, but that was expected.
This commit is contained in:
Lai Jiang
+45
-61
@@ -18,20 +18,27 @@ import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.loadRegistrar;
|
||||
import static google.registry.testing.DatabaseHelper.persistSimpleResource;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.client.http.HttpResponseException;
|
||||
import com.google.api.client.http.HttpStatusCodes;
|
||||
import com.google.api.client.http.LowLevelHttpRequest;
|
||||
import com.google.api.client.testing.http.MockHttpTransport;
|
||||
import com.google.api.client.testing.http.MockLowLevelHttpRequest;
|
||||
import com.google.api.client.testing.http.MockLowLevelHttpResponse;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import google.registry.model.registrar.Registrar;
|
||||
import google.registry.model.registrar.RegistrarAddress;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.testing.FakeUrlConnectionService;
|
||||
import google.registry.util.UrlConnectionException;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
@@ -61,44 +68,26 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
|
||||
public JpaIntegrationTestExtension jpa =
|
||||
new JpaTestExtensions.Builder().buildIntegrationTestExtension();
|
||||
|
||||
private static class TestHttpTransport extends MockHttpTransport {
|
||||
private MockLowLevelHttpRequest requestSent;
|
||||
private MockLowLevelHttpResponse response;
|
||||
|
||||
void setResponse(MockLowLevelHttpResponse response) {
|
||||
this.response = response;
|
||||
}
|
||||
|
||||
MockLowLevelHttpRequest getRequestSent() {
|
||||
return requestSent;
|
||||
}
|
||||
|
||||
@Override
|
||||
public LowLevelHttpRequest buildRequest(String method, String url) {
|
||||
assertThat(method).isEqualTo("GET");
|
||||
MockLowLevelHttpRequest httpRequest = new MockLowLevelHttpRequest(url);
|
||||
httpRequest.setResponse(response);
|
||||
requestSent = httpRequest;
|
||||
return httpRequest;
|
||||
}
|
||||
}
|
||||
|
||||
private TestHttpTransport httpTransport;
|
||||
private final HttpURLConnection connection = mock(HttpURLConnection.class);
|
||||
private final FakeUrlConnectionService urlConnectionService =
|
||||
new FakeUrlConnectionService(connection);
|
||||
private UpdateRegistrarRdapBaseUrlsAction action;
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
void beforeEach() throws Exception {
|
||||
action = new UpdateRegistrarRdapBaseUrlsAction();
|
||||
httpTransport = new TestHttpTransport();
|
||||
action.httpTransport = httpTransport;
|
||||
setValidResponse();
|
||||
action.urlConnectionService = urlConnectionService;
|
||||
when(connection.getResponseCode()).thenReturn(SC_OK);
|
||||
when(connection.getInputStream())
|
||||
.thenReturn(new ByteArrayInputStream(CSV_REPLY.getBytes(StandardCharsets.UTF_8)));
|
||||
createTld("tld");
|
||||
}
|
||||
|
||||
private void assertCorrectRequestSent() {
|
||||
assertThat(httpTransport.getRequestSent().getUrl())
|
||||
.isEqualTo("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv");
|
||||
assertThat(httpTransport.getRequestSent().getHeaders().get("accept-encoding")).isNull();
|
||||
private void assertCorrectRequestSent() throws Exception {
|
||||
assertThat(urlConnectionService.getConnectedUrls())
|
||||
.containsExactly(
|
||||
new URL("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv"));
|
||||
verify(connection).setRequestProperty("Accept-Encoding", "gzip");
|
||||
}
|
||||
|
||||
private static void persistRegistrar(
|
||||
@@ -119,14 +108,8 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
|
||||
.build());
|
||||
}
|
||||
|
||||
private void setValidResponse() {
|
||||
MockLowLevelHttpResponse csvResponse = new MockLowLevelHttpResponse();
|
||||
csvResponse.setContent(CSV_REPLY);
|
||||
httpTransport.setResponse(csvResponse);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testUnknownIana_cleared() {
|
||||
void testUnknownIana_cleared() throws Exception {
|
||||
// The IANA ID isn't in the CSV reply
|
||||
persistRegistrar("someRegistrar", 4123L, Registrar.Type.REAL, "http://rdap.example/blah");
|
||||
action.run();
|
||||
@@ -135,7 +118,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testKnownIana_changed() {
|
||||
void testKnownIana_changed() throws Exception {
|
||||
// The IANA ID is in the CSV reply
|
||||
persistRegistrar("someRegistrar", 1448L, Registrar.Type.REAL, "http://rdap.example/blah");
|
||||
action.run();
|
||||
@@ -145,7 +128,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testKnownIana_notReal_noChange() {
|
||||
void testKnownIana_notReal_noChange() throws Exception {
|
||||
// The IANA ID is in the CSV reply
|
||||
persistRegistrar("someRegistrar", 9999L, Registrar.Type.INTERNAL, "http://rdap.example/blah");
|
||||
// Real registrars should actually change
|
||||
@@ -159,7 +142,7 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testKnownIana_notReal_nullIANA_noChange() {
|
||||
void testKnownIana_notReal_nullIANA_noChange() throws Exception {
|
||||
persistRegistrar("someRegistrar", null, Registrar.Type.TEST, "http://rdap.example/blah");
|
||||
action.run();
|
||||
assertCorrectRequestSent();
|
||||
@@ -168,29 +151,30 @@ public final class UpdateRegistrarRdapBaseUrlsActionTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFailure_serverErrorResponse() {
|
||||
MockLowLevelHttpResponse badResponse = new MockLowLevelHttpResponse();
|
||||
badResponse.setZeroContent();
|
||||
badResponse.setStatusCode(HttpStatusCodes.STATUS_CODE_SERVER_ERROR);
|
||||
httpTransport.setResponse(badResponse);
|
||||
|
||||
RuntimeException thrown = assertThrows(RuntimeException.class, action::run);
|
||||
void testFailure_serverErrorResponse() throws Exception {
|
||||
when(connection.getResponseCode()).thenReturn(SC_INTERNAL_SERVER_ERROR);
|
||||
when(connection.getInputStream())
|
||||
.thenReturn(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)));
|
||||
InternalServerErrorException thrown =
|
||||
assertThrows(InternalServerErrorException.class, action::run);
|
||||
verify(connection, times(0)).getInputStream();
|
||||
assertThat(thrown).hasMessageThat().isEqualTo("Error when retrieving RDAP base URL CSV file");
|
||||
Throwable cause = thrown.getCause();
|
||||
assertThat(cause).isInstanceOf(HttpResponseException.class);
|
||||
assertThat(cause).isInstanceOf(UrlConnectionException.class);
|
||||
assertThat(cause)
|
||||
.hasMessageThat()
|
||||
.isEqualTo("500\nGET https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv");
|
||||
.contains("https://www.iana.org/assignments/registrar-ids/registrar-ids-1.csv");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFailure_invalidCsv() {
|
||||
MockLowLevelHttpResponse csvResponse = new MockLowLevelHttpResponse();
|
||||
csvResponse.setContent("foo,bar\nbaz,foo");
|
||||
httpTransport.setResponse(csvResponse);
|
||||
void testFailure_invalidCsv() throws Exception {
|
||||
when(connection.getInputStream())
|
||||
.thenReturn(new ByteArrayInputStream("foo,bar\nbaz,foo".getBytes(StandardCharsets.UTF_8)));
|
||||
|
||||
IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, action::run);
|
||||
InternalServerErrorException thrown =
|
||||
assertThrows(InternalServerErrorException.class, action::run);
|
||||
assertThat(thrown)
|
||||
.hasCauseThat()
|
||||
.hasMessageThat()
|
||||
.isEqualTo("Mapping for ID not found, expected one of [foo, bar]");
|
||||
}
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
|
||||
package google.registry.rde;
|
||||
|
||||
import static com.google.appengine.api.urlfetch.HTTPMethod.PUT;
|
||||
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST;
|
||||
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK;
|
||||
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_UNAUTHORIZED;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.model.common.Cursor.CursorType.RDE_REPORT;
|
||||
@@ -24,25 +26,17 @@ import static google.registry.persistence.transaction.TransactionManagerFactory.
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static google.registry.testing.DatabaseHelper.loadByKey;
|
||||
import static google.registry.testing.DatabaseHelper.persistResource;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
|
||||
import static javax.servlet.http.HttpServletResponse.SC_OK;
|
||||
import static org.joda.time.Duration.standardDays;
|
||||
import static org.joda.time.Duration.standardSeconds;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.verifyNoMoreInteractions;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.appengine.api.urlfetch.HTTPHeader;
|
||||
import com.google.appengine.api.urlfetch.HTTPRequest;
|
||||
import com.google.appengine.api.urlfetch.HTTPResponse;
|
||||
import com.google.appengine.api.urlfetch.URLFetchService;
|
||||
import com.google.cloud.storage.BlobId;
|
||||
import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
|
||||
import com.google.common.base.Ascii;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.io.ByteSource;
|
||||
import google.registry.gcs.GcsUtils;
|
||||
import google.registry.model.common.Cursor;
|
||||
@@ -53,25 +47,23 @@ import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationT
|
||||
import google.registry.request.HttpException.InternalServerErrorException;
|
||||
import google.registry.request.HttpException.NoContentException;
|
||||
import google.registry.testing.BouncyCastleProviderExtension;
|
||||
import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeKeyringModule;
|
||||
import google.registry.testing.FakeResponse;
|
||||
import google.registry.testing.FakeSleeper;
|
||||
import google.registry.util.Retrier;
|
||||
import google.registry.testing.FakeUrlConnectionService;
|
||||
import google.registry.util.UrlConnectionException;
|
||||
import google.registry.xjc.XjcXmlTransformer;
|
||||
import google.registry.xjc.rdereport.XjcRdeReportReport;
|
||||
import google.registry.xml.XmlException;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.net.SocketTimeoutException;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
/** Unit tests for {@link RdeReportAction}. */
|
||||
public class RdeReportActionTest {
|
||||
@@ -89,9 +81,11 @@ public class RdeReportActionTest {
|
||||
|
||||
private final FakeResponse response = new FakeResponse();
|
||||
private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class);
|
||||
private final URLFetchService urlFetchService = mock(URLFetchService.class);
|
||||
private final ArgumentCaptor<HTTPRequest> request = ArgumentCaptor.forClass(HTTPRequest.class);
|
||||
private final HTTPResponse httpResponse = mock(HTTPResponse.class);
|
||||
private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
|
||||
private final FakeUrlConnectionService urlConnectionService =
|
||||
new FakeUrlConnectionService(httpUrlConnection);
|
||||
private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream();
|
||||
|
||||
private final PGPPublicKey encryptKey =
|
||||
new FakeKeyringModule().get().getRdeStagingEncryptionKey();
|
||||
private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions());
|
||||
@@ -102,9 +96,8 @@ public class RdeReportActionTest {
|
||||
private RdeReportAction createAction() {
|
||||
RdeReporter reporter = new RdeReporter();
|
||||
reporter.reportUrlPrefix = "https://rde-report.example";
|
||||
reporter.urlFetchService = urlFetchService;
|
||||
reporter.urlConnectionService = urlConnectionService;
|
||||
reporter.password = "foo";
|
||||
reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3);
|
||||
RdeReportAction action = new RdeReportAction();
|
||||
action.gcsUtils = gcsUtils;
|
||||
action.response = response;
|
||||
@@ -126,6 +119,9 @@ public class RdeReportActionTest {
|
||||
persistResource(Cursor.createScoped(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), registry));
|
||||
gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0));
|
||||
when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream);
|
||||
when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_OK);
|
||||
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openBufferedStream());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -142,24 +138,20 @@ public class RdeReportActionTest {
|
||||
|
||||
@Test
|
||||
void testRunWithLock() throws Exception {
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
|
||||
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
|
||||
createAction().runWithLock(loadRdeReportCursor());
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
|
||||
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
|
||||
|
||||
// Verify the HTTP request was correct.
|
||||
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
|
||||
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
|
||||
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
|
||||
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
|
||||
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
|
||||
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
|
||||
verify(httpUrlConnection).setRequestMethod("PUT");
|
||||
assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
|
||||
assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
|
||||
verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
|
||||
verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
|
||||
|
||||
// Verify the payload XML was the same as what's in testdata/report.xml.
|
||||
XjcRdeReportReport report = parseReport(request.getValue().getPayload());
|
||||
XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
|
||||
assertThat(report.getId()).isEqualTo("20101017001");
|
||||
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
|
||||
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
|
||||
@@ -167,9 +159,6 @@ public class RdeReportActionTest {
|
||||
|
||||
@Test
|
||||
void testRunWithLock_withPrefix() throws Exception {
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
|
||||
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
|
||||
RdeReportAction action = createAction();
|
||||
action.runWithLock(loadRdeReportCursor());
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
@@ -177,15 +166,14 @@ public class RdeReportActionTest {
|
||||
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
|
||||
|
||||
// Verify the HTTP request was correct.
|
||||
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
|
||||
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
|
||||
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
|
||||
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
|
||||
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
|
||||
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
|
||||
verify(httpUrlConnection).setRequestMethod("PUT");
|
||||
assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
|
||||
assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
|
||||
verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
|
||||
verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
|
||||
|
||||
// Verify the payload XML was the same as what's in testdata/report.xml.
|
||||
XjcRdeReportReport report = parseReport(request.getValue().getPayload());
|
||||
XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
|
||||
assertThat(report.getId()).isEqualTo("20101017001");
|
||||
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
|
||||
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
|
||||
@@ -200,9 +188,6 @@ public class RdeReportActionTest {
|
||||
|
||||
@Test
|
||||
void testRunWithLock_withoutPrefix() throws Exception {
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
|
||||
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
|
||||
RdeReportAction action = createAction();
|
||||
action.prefix = Optional.empty();
|
||||
gcsUtils.delete(reportFile);
|
||||
@@ -225,15 +210,14 @@ public class RdeReportActionTest {
|
||||
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
|
||||
|
||||
// Verify the HTTP request was correct.
|
||||
assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT);
|
||||
assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https");
|
||||
assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001");
|
||||
Map<String, String> headers = mapifyHeaders(request.getValue().getHeaders());
|
||||
assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml");
|
||||
assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28=");
|
||||
verify(httpUrlConnection).setRequestMethod("PUT");
|
||||
assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https");
|
||||
assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001");
|
||||
verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8");
|
||||
verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28=");
|
||||
|
||||
// Verify the payload XML was the same as what's in testdata/report.xml.
|
||||
XjcRdeReportReport report = parseReport(request.getValue().getPayload());
|
||||
XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray());
|
||||
assertThat(report.getId()).isEqualTo("20101017001");
|
||||
assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z"));
|
||||
assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z"));
|
||||
@@ -246,9 +230,6 @@ public class RdeReportActionTest {
|
||||
PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey();
|
||||
gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey));
|
||||
tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1));
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
|
||||
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
|
||||
createAction().runWithLock(loadRdeReportCursor());
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
}
|
||||
@@ -281,9 +262,8 @@ public class RdeReportActionTest {
|
||||
|
||||
@Test
|
||||
void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception {
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read());
|
||||
when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse);
|
||||
when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST);
|
||||
when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
|
||||
InternalServerErrorException thrown =
|
||||
assertThrows(
|
||||
InternalServerErrorException.class,
|
||||
@@ -292,38 +272,19 @@ public class RdeReportActionTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception {
|
||||
class ExpectedThrownException extends RuntimeException {}
|
||||
when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException());
|
||||
assertThrows(
|
||||
ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRunWithLock_socketTimeout_doesRetry() throws Exception {
|
||||
when(httpResponse.getResponseCode()).thenReturn(SC_OK);
|
||||
when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read());
|
||||
when(urlFetchService.fetch(request.capture()))
|
||||
.thenThrow(new SocketTimeoutException())
|
||||
.thenReturn(httpResponse);
|
||||
createAction().runWithLock(loadRdeReportCursor());
|
||||
assertThat(response.getStatus()).isEqualTo(200);
|
||||
assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8);
|
||||
assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n");
|
||||
void testRunWithLock_notAuthorized() throws Exception {
|
||||
when(httpUrlConnection.getResponseCode()).thenReturn(STATUS_CODE_UNAUTHORIZED);
|
||||
UrlConnectionException thrown =
|
||||
assertThrows(
|
||||
UrlConnectionException.class, () -> createAction().runWithLock(loadRdeReportCursor()));
|
||||
verify(httpUrlConnection, times(0)).getInputStream();
|
||||
assertThat(thrown).hasMessageThat().contains("PUT failed");
|
||||
}
|
||||
|
||||
private DateTime loadRdeReportCursor() {
|
||||
return loadByKey(Cursor.createScopedVKey(RDE_REPORT, registry)).getCursorTime();
|
||||
}
|
||||
|
||||
private static ImmutableMap<String, String> mapifyHeaders(Iterable<HTTPHeader> headers) {
|
||||
ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
|
||||
for (HTTPHeader header : headers) {
|
||||
builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private static XjcRdeReportReport parseReport(byte[] data) {
|
||||
try {
|
||||
return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));
|
||||
|
||||
@@ -14,28 +14,27 @@
|
||||
|
||||
package google.registry.reporting.icann;
|
||||
|
||||
import static com.google.common.net.MediaType.CSV_UTF_8;
|
||||
import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8;
|
||||
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_BAD_REQUEST;
|
||||
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_OK;
|
||||
import static com.google.api.client.http.HttpStatusCodes.STATUS_CODE_SERVER_ERROR;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static google.registry.testing.DatabaseHelper.createTld;
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.api.client.http.HttpResponseException;
|
||||
import com.google.api.client.http.HttpStatusCodes;
|
||||
import com.google.api.client.http.LowLevelHttpRequest;
|
||||
import com.google.api.client.http.LowLevelHttpResponse;
|
||||
import com.google.api.client.testing.http.MockHttpTransport;
|
||||
import com.google.api.client.testing.http.MockLowLevelHttpRequest;
|
||||
import com.google.api.client.testing.http.MockLowLevelHttpResponse;
|
||||
import com.google.api.client.util.Base64;
|
||||
import com.google.api.client.util.StringUtils;
|
||||
import com.google.common.io.BaseEncoding;
|
||||
import com.google.common.io.ByteSource;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions;
|
||||
import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import google.registry.testing.FakeUrlConnectionService;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
@@ -46,103 +45,75 @@ class IcannHttpReporterTest {
|
||||
private static final ByteSource IIRDEA_GOOD_XML = ReportingTestData.loadBytes("iirdea_good.xml");
|
||||
private static final ByteSource IIRDEA_BAD_XML = ReportingTestData.loadBytes("iirdea_bad.xml");
|
||||
private static final byte[] FAKE_PAYLOAD = "test,csv\n1,2".getBytes(UTF_8);
|
||||
private static final IcannHttpReporter reporter = new IcannHttpReporter();
|
||||
|
||||
private MockLowLevelHttpRequest mockRequest;
|
||||
private final HttpURLConnection connection = mock(HttpURLConnection.class);
|
||||
private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
private final FakeUrlConnectionService urlConnectionService =
|
||||
new FakeUrlConnectionService(connection);
|
||||
|
||||
@RegisterExtension
|
||||
final JpaIntegrationTestExtension jpa =
|
||||
new JpaTestExtensions.Builder().buildIntegrationTestExtension();
|
||||
|
||||
private MockHttpTransport createMockTransport(
|
||||
int statusCode, final ByteSource iirdeaResponse) {
|
||||
return new MockHttpTransport() {
|
||||
@Override
|
||||
public LowLevelHttpRequest buildRequest(String method, String url) {
|
||||
mockRequest =
|
||||
new MockLowLevelHttpRequest() {
|
||||
@Override
|
||||
public LowLevelHttpResponse execute() throws IOException {
|
||||
MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
|
||||
response.setStatusCode(statusCode);
|
||||
response.setContentType(PLAIN_TEXT_UTF_8.toString());
|
||||
response.setContent(iirdeaResponse.read());
|
||||
return response;
|
||||
}
|
||||
};
|
||||
mockRequest.setUrl(url);
|
||||
return mockRequest;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private MockHttpTransport createMockTransport(final ByteSource iirdeaResponse) {
|
||||
return createMockTransport(HttpStatusCodes.STATUS_CODE_OK, iirdeaResponse);
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
void beforeEach() {
|
||||
void beforeEach() throws Exception {
|
||||
createTld("test");
|
||||
createTld("xn--abc123");
|
||||
}
|
||||
|
||||
private IcannHttpReporter createReporter() {
|
||||
IcannHttpReporter reporter = new IcannHttpReporter();
|
||||
reporter.httpTransport = createMockTransport(IIRDEA_GOOD_XML);
|
||||
when(connection.getOutputStream()).thenReturn(outputStream);
|
||||
when(connection.getResponseCode()).thenReturn(STATUS_CODE_OK);
|
||||
when(connection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openBufferedStream());
|
||||
reporter.urlConnectionService = urlConnectionService;
|
||||
reporter.password = "fakePass";
|
||||
reporter.icannTransactionsUrl = "https://fake-transactions.url";
|
||||
reporter.icannActivityUrl = "https://fake-activity.url";
|
||||
return reporter;
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess() throws Exception {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv");
|
||||
assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isTrue();
|
||||
|
||||
assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/test/2017-06");
|
||||
Map<String, List<String>> headers = mockRequest.getHeaders();
|
||||
assertThat(urlConnectionService.getConnectedUrls())
|
||||
.containsExactly(new URL("https://fake-transactions.url/test/2017-06"));
|
||||
String userPass = "test_ry:fakePass";
|
||||
String expectedAuth =
|
||||
String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass)));
|
||||
assertThat(headers.get("authorization")).containsExactly(expectedAuth);
|
||||
assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
|
||||
String.format("Basic %s", BaseEncoding.base64().encode(StringUtils.getBytesUtf8(userPass)));
|
||||
verify(connection).setRequestProperty("Authorization", expectedAuth);
|
||||
verify(connection).setRequestProperty("Content-Type", "text/csv; charset=utf-8");
|
||||
assertThat(outputStream.toByteArray()).isEqualTo(FAKE_PAYLOAD);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSuccess_internationalTld() throws Exception {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv");
|
||||
assertThat(reporter.send(FAKE_PAYLOAD, "xn--abc123-transactions-201706.csv")).isTrue();
|
||||
|
||||
assertThat(mockRequest.getUrl()).isEqualTo("https://fake-transactions.url/xn--abc123/2017-06");
|
||||
Map<String, List<String>> headers = mockRequest.getHeaders();
|
||||
assertThat(urlConnectionService.getConnectedUrls())
|
||||
.containsExactly(new URL("https://fake-transactions.url/xn--abc123/2017-06"));
|
||||
String userPass = "xn--abc123_ry:fakePass";
|
||||
String expectedAuth =
|
||||
String.format("Basic %s", Base64.encodeBase64String(StringUtils.getBytesUtf8(userPass)));
|
||||
assertThat(headers.get("authorization")).containsExactly(expectedAuth);
|
||||
assertThat(headers.get("content-type")).containsExactly(CSV_UTF_8.toString());
|
||||
String.format("Basic %s", BaseEncoding.base64().encode(StringUtils.getBytesUtf8(userPass)));
|
||||
verify(connection).setRequestProperty("Authorization", expectedAuth);
|
||||
verify(connection).setRequestProperty("Content-Type", "text/csv; charset=utf-8");
|
||||
assertThat(outputStream.toByteArray()).isEqualTo(FAKE_PAYLOAD);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFail_BadIirdeaResponse() throws Exception {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
reporter.httpTransport =
|
||||
createMockTransport(HttpStatusCodes.STATUS_CODE_BAD_REQUEST, IIRDEA_BAD_XML);
|
||||
when(connection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openBufferedStream());
|
||||
when(connection.getResponseCode()).thenReturn(STATUS_CODE_BAD_REQUEST);
|
||||
assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse();
|
||||
verify(connection).getInputStream();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFail_transportException() {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
reporter.httpTransport =
|
||||
createMockTransport(HttpStatusCodes.STATUS_CODE_FORBIDDEN, ByteSource.empty());
|
||||
assertThrows(
|
||||
HttpResponseException.class,
|
||||
() -> reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv"));
|
||||
void testFail_OtherBadHttpResponse() throws Exception {
|
||||
when(connection.getResponseCode()).thenReturn(STATUS_CODE_SERVER_ERROR);
|
||||
assertThat(reporter.send(FAKE_PAYLOAD, "test-transactions-201706.csv")).isFalse();
|
||||
verify(connection, times(0)).getInputStream();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFail_invalidFilename_nonSixDigitYearMonth() {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
@@ -156,7 +127,6 @@ class IcannHttpReporterTest {
|
||||
|
||||
@Test
|
||||
void testFail_invalidFilename_notActivityOrTransactions() {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
@@ -169,7 +139,6 @@ class IcannHttpReporterTest {
|
||||
|
||||
@Test
|
||||
void testFail_invalidFilename_invalidTldName() {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
@@ -183,7 +152,6 @@ class IcannHttpReporterTest {
|
||||
|
||||
@Test
|
||||
void testFail_invalidFilename_tldDoesntExist() {
|
||||
IcannHttpReporter reporter = createReporter();
|
||||
IllegalArgumentException thrown =
|
||||
assertThrows(
|
||||
IllegalArgumentException.class,
|
||||
|
||||
@@ -16,6 +16,7 @@ package google.registry.testing;
|
||||
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import google.registry.request.UrlConnectionService;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
@@ -26,15 +27,10 @@ import java.util.List;
|
||||
public class FakeUrlConnectionService implements UrlConnectionService {
|
||||
|
||||
private final HttpURLConnection mockConnection;
|
||||
private final List<URL> connectedUrls;
|
||||
private final List<URL> connectedUrls = new ArrayList<>();
|
||||
|
||||
public FakeUrlConnectionService(HttpURLConnection mockConnection) {
|
||||
this(mockConnection, new ArrayList<>());
|
||||
}
|
||||
|
||||
public FakeUrlConnectionService(HttpURLConnection mockConnection, List<URL> connectedUrls) {
|
||||
this.mockConnection = mockConnection;
|
||||
this.connectedUrls = connectedUrls;
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -43,4 +39,8 @@ public class FakeUrlConnectionService implements UrlConnectionService {
|
||||
when(mockConnection.getURL()).thenReturn(url);
|
||||
return mockConnection;
|
||||
}
|
||||
|
||||
public ImmutableList<URL> getConnectedUrls() {
|
||||
return ImmutableList.copyOf(connectedUrls);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,9 +25,7 @@ import google.registry.testing.FakeClock;
|
||||
import google.registry.testing.FakeUrlConnectionService;
|
||||
import google.registry.testing.TestCacheExtension;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.extension.RegisterExtension;
|
||||
@@ -55,9 +53,8 @@ abstract class TmchActionTestCase {
|
||||
final Marksdb marksdb = new Marksdb();
|
||||
|
||||
protected final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class);
|
||||
protected final ArrayList<URL> connectedUrls = new ArrayList<>();
|
||||
protected FakeUrlConnectionService urlConnectionService =
|
||||
new FakeUrlConnectionService(httpUrlConnection, connectedUrls);
|
||||
new FakeUrlConnectionService(httpUrlConnection);
|
||||
|
||||
@BeforeEach
|
||||
public void beforeEachTmchActionTestCase() throws Exception {
|
||||
|
||||
@@ -56,7 +56,8 @@ class TmchCrlActionTest extends TmchActionTestCase {
|
||||
readResourceBytes(TmchCertificateAuthority.class, "icann-tmch-pilot.crl").read()));
|
||||
newTmchCrlAction(TmchCaMode.PILOT).run();
|
||||
verify(httpUrlConnection).getInputStream();
|
||||
assertThat(connectedUrls).containsExactly(new URL("https://sloth.lol/tmch.crl"));
|
||||
assertThat(urlConnectionService.getConnectedUrls())
|
||||
.containsExactly(new URL("https://sloth.lol/tmch.crl"));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@@ -49,7 +49,10 @@ class TmchDnlActionTest extends TmchActionTestCase {
|
||||
.thenReturn(new ByteArrayInputStream(TmchTestData.loadBytes("dnl/dnl-latest.sig").read()));
|
||||
newTmchDnlAction().run();
|
||||
verify(httpUrlConnection, times(2)).getInputStream();
|
||||
assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
|
||||
assertThat(
|
||||
urlConnectionService.getConnectedUrls().stream()
|
||||
.map(URL::toString)
|
||||
.collect(toImmutableList()))
|
||||
.containsExactly(MARKSDB_URL + "/dnl/dnl-latest.csv", MARKSDB_URL + "/dnl/dnl-latest.sig");
|
||||
|
||||
// Make sure the contents of testdata/dnl-latest.csv got inserted into the database.
|
||||
|
||||
@@ -50,7 +50,10 @@ class TmchSmdrlActionTest extends TmchActionTestCase {
|
||||
.thenReturn(new ByteArrayInputStream(loadBytes("smdrl/smdrl-latest.sig").read()));
|
||||
newTmchSmdrlAction().run();
|
||||
verify(httpUrlConnection, times(2)).getInputStream();
|
||||
assertThat(connectedUrls.stream().map(URL::toString).collect(toImmutableList()))
|
||||
assertThat(
|
||||
urlConnectionService.getConnectedUrls().stream()
|
||||
.map(URL::toString)
|
||||
.collect(toImmutableList()))
|
||||
.containsExactly(
|
||||
MARKSDB_URL + "/smdrl/smdrl-latest.csv", MARKSDB_URL + "/smdrl/smdrl-latest.sig");
|
||||
smdrl = SignedMarkRevocationList.get();
|
||||
|
||||
@@ -6,7 +6,7 @@ PATH CLASS
|
||||
/_dr/task/deleteExpiredDomains DeleteExpiredDomainsAction GET n API APP ADMIN
|
||||
/_dr/task/deleteLoadTestData DeleteLoadTestDataAction POST n API APP ADMIN
|
||||
/_dr/task/deleteProberData DeleteProberDataAction POST n API APP ADMIN
|
||||
/_dr/task/executeCannedScript CannedScriptExecutionAction POST y API APP ADMIN
|
||||
/_dr/task/executeCannedScript CannedScriptExecutionAction POST,GET y API APP ADMIN
|
||||
/_dr/task/expandBillingRecurrences ExpandBillingRecurrencesAction GET n API APP ADMIN
|
||||
/_dr/task/exportDomainLists ExportDomainListsAction POST n API APP ADMIN
|
||||
/_dr/task/exportPremiumTerms ExportPremiumTermsAction POST n API APP ADMIN
|
||||
|
||||
Reference in New Issue
Block a user