diff --git a/core/src/main/java/google/registry/module/backend/BackendComponent.java b/core/src/main/java/google/registry/module/backend/BackendComponent.java index 042a4e715..9bb3ce2d1 100644 --- a/core/src/main/java/google/registry/module/backend/BackendComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendComponent.java @@ -44,6 +44,7 @@ import google.registry.request.Modules.DatastoreServiceModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.NetHttpTransportModule; import google.registry.request.Modules.UrlConnectionServiceModule; +import google.registry.request.Modules.UrlFetchServiceModule; import google.registry.request.Modules.UrlFetchTransportModule; import google.registry.request.Modules.UserServiceModule; import google.registry.request.auth.AuthModule; @@ -81,6 +82,7 @@ import javax.inject.Singleton; SheetsServiceModule.class, StackdriverModule.class, UrlConnectionServiceModule.class, + UrlFetchServiceModule.class, UrlFetchTransportModule.class, UserServiceModule.class, VoidDnsWriterModule.class, diff --git a/core/src/main/java/google/registry/rde/RdeReporter.java b/core/src/main/java/google/registry/rde/RdeReporter.java index f4fdd3811..7fc5d647c 100644 --- a/core/src/main/java/google/registry/rde/RdeReporter.java +++ b/core/src/main/java/google/registry/rde/RdeReporter.java @@ -14,24 +14,25 @@ package google.registry.rde; -import static google.registry.request.UrlConnectionUtils.getResponseBytes; -import static google.registry.request.UrlConnectionUtils.setBasicAuth; -import static google.registry.request.UrlConnectionUtils.setPayload; +import static com.google.appengine.api.urlfetch.FetchOptions.Builder.validateCertificate; +import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; +import static com.google.common.io.BaseEncoding.base64; +import static com.google.common.net.HttpHeaders.AUTHORIZATION; +import static com.google.common.net.HttpHeaders.CONTENT_TYPE; import static google.registry.util.DomainNameUtils.canonicalizeDomainName; import static java.nio.charset.StandardCharsets.UTF_8; import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST; import static javax.servlet.http.HttpServletResponse.SC_OK; -import com.google.api.client.http.HttpMethods; +import com.google.appengine.api.urlfetch.HTTPHeader; +import com.google.appengine.api.urlfetch.HTTPRequest; import com.google.appengine.api.urlfetch.HTTPResponse; +import com.google.appengine.api.urlfetch.URLFetchService; import com.google.common.flogger.FluentLogger; -import com.google.common.net.MediaType; import google.registry.config.RegistryConfig.Config; import google.registry.keyring.api.KeyModule.Key; import google.registry.request.HttpException.InternalServerErrorException; -import google.registry.request.UrlConnectionService; import google.registry.util.Retrier; -import google.registry.util.UrlConnectionException; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.iirdea.XjcIirdeaResponseElement; import google.registry.xjc.iirdea.XjcIirdeaResult; @@ -39,7 +40,6 @@ import google.registry.xjc.rdeheader.XjcRdeHeader; import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xml.XmlException; import java.io.ByteArrayInputStream; -import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.SocketTimeoutException; import java.net.URL; @@ -58,10 +58,10 @@ public class RdeReporter { * @see * ICANN Registry Interfaces - Interface details */ - private static final MediaType MEDIA_TYPE = MediaType.XML_UTF_8; + private static final String REPORT_MIME = "text/xml"; @Inject Retrier retrier; - @Inject UrlConnectionService urlConnectionService; + @Inject URLFetchService urlFetchService; @Inject @Config("rdeReportUrlPrefix") String reportUrlPrefix; @Inject @Key("icannReportingPassword") String password; @@ -76,24 +76,29 @@ public class RdeReporter { // Send a PUT request to ICANN's HTTPS server. URL url = makeReportUrl(header.getTld(), report.getId()); String username = header.getTld() + "_ry"; + String token = base64().encode(String.format("%s:%s", username, password).getBytes(UTF_8)); + final HTTPRequest req = new HTTPRequest(url, PUT, validateCertificate().setDeadline(60d)); + req.addHeader(new HTTPHeader(CONTENT_TYPE, REPORT_MIME)); + req.addHeader(new HTTPHeader(AUTHORIZATION, "Basic " + token)); + req.setPayload(reportBytes); logger.atInfo().log("Sending report:\n%s", new String(reportBytes, UTF_8)); - byte[] responseBytes = + HTTPResponse rsp = retrier.callWithRetry( () -> { - HttpURLConnection connection = urlConnectionService.createConnection(url); - connection.setRequestMethod(HttpMethods.PUT); - setBasicAuth(connection, username, password); - setPayload(connection, reportBytes, MEDIA_TYPE.toString()); - int responseCode = connection.getResponseCode(); - if (responseCode == SC_OK || responseCode == SC_BAD_REQUEST) { - return getResponseBytes(connection); + HTTPResponse rsp1 = urlFetchService.fetch(req); + switch (rsp1.getResponseCode()) { + case SC_OK: + case SC_BAD_REQUEST: + break; + default: + throw new RuntimeException("PUT failed"); } - throw new UrlConnectionException("PUT failed", connection); + return rsp1; }, SocketTimeoutException.class); // Ensure the XML response is valid. - XjcIirdeaResult result = parseResult(responseBytes); + XjcIirdeaResult result = parseResult(rsp.getContent()); if (result.getCode().getValue() != 1000) { logger.atWarning().log( "PUT rejected: %d %s\n%s", diff --git a/core/src/main/java/google/registry/request/Modules.java b/core/src/main/java/google/registry/request/Modules.java index feed5ab5b..5c84e8b22 100644 --- a/core/src/main/java/google/registry/request/Modules.java +++ b/core/src/main/java/google/registry/request/Modules.java @@ -23,14 +23,14 @@ import com.google.api.client.http.javanet.NetHttpTransport; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.urlfetch.URLFetchService; +import com.google.appengine.api.urlfetch.URLFetchServiceFactory; import com.google.appengine.api.users.UserService; import com.google.appengine.api.users.UserServiceFactory; import dagger.Module; import dagger.Provides; import java.net.HttpURLConnection; import javax.inject.Singleton; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; /** Dagger modules for App Engine services and other vendor classes. */ public final class Modules { @@ -51,16 +51,18 @@ public final class Modules { public static final class UrlConnectionServiceModule { @Provides static UrlConnectionService provideUrlConnectionService() { - return url -> { - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - if (connection instanceof HttpsURLConnection) { - HttpsURLConnection httpsConnection = (HttpsURLConnection) connection; - SSLContext tls13Context = SSLContext.getInstance("TLSv1.3"); - tls13Context.init(null, null, null); - httpsConnection.setSSLSocketFactory(tls13Context.getSocketFactory()); - } - return connection; - }; + return url -> (HttpURLConnection) url.openConnection(); + } + } + + /** Dagger module for {@link URLFetchService}. */ + @Module + public static final class UrlFetchServiceModule { + private static final URLFetchService fetchService = URLFetchServiceFactory.getURLFetchService(); + + @Provides + static URLFetchService provideUrlFetchService() { + return fetchService; } } diff --git a/core/src/main/java/google/registry/tools/RegistryToolComponent.java b/core/src/main/java/google/registry/tools/RegistryToolComponent.java index 042a97578..d882505a2 100644 --- a/core/src/main/java/google/registry/tools/RegistryToolComponent.java +++ b/core/src/main/java/google/registry/tools/RegistryToolComponent.java @@ -40,6 +40,7 @@ import google.registry.rde.RdeModule; import google.registry.request.Modules.DatastoreServiceModule; import google.registry.request.Modules.Jackson2Module; import google.registry.request.Modules.UrlConnectionServiceModule; +import google.registry.request.Modules.UrlFetchServiceModule; import google.registry.request.Modules.UserServiceModule; import google.registry.tools.AuthModule.LocalCredentialModule; import google.registry.tools.javascrap.CompareEscrowDepositsCommand; @@ -80,6 +81,7 @@ import javax.inject.Singleton; RequestFactoryModule.class, SecretManagerModule.class, UrlConnectionServiceModule.class, + UrlFetchServiceModule.class, UserServiceModule.class, UtilsModule.class, VoidDnsWriterModule.class, diff --git a/core/src/test/java/google/registry/rde/RdeReportActionTest.java b/core/src/test/java/google/registry/rde/RdeReportActionTest.java index 3d77cad01..b672ee4c1 100644 --- a/core/src/test/java/google/registry/rde/RdeReportActionTest.java +++ b/core/src/test/java/google/registry/rde/RdeReportActionTest.java @@ -14,6 +14,7 @@ package google.registry.rde; +import static com.google.appengine.api.urlfetch.HTTPMethod.PUT; import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; import static com.google.common.truth.Truth.assertThat; import static google.registry.model.common.Cursor.CursorType.RDE_REPORT; @@ -28,13 +29,20 @@ import static javax.servlet.http.HttpServletResponse.SC_OK; import static org.joda.time.Duration.standardDays; import static org.joda.time.Duration.standardSeconds; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.google.appengine.api.urlfetch.HTTPHeader; +import com.google.appengine.api.urlfetch.HTTPRequest; +import com.google.appengine.api.urlfetch.HTTPResponse; +import com.google.appengine.api.urlfetch.URLFetchService; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; import google.registry.gcs.GcsUtils; import google.registry.model.common.Cursor; @@ -49,21 +57,20 @@ import google.registry.testing.FakeClock; import google.registry.testing.FakeKeyringModule; import google.registry.testing.FakeResponse; import google.registry.testing.FakeSleeper; -import google.registry.testing.FakeUrlConnectionService; import google.registry.testing.TestOfyAndSql; import google.registry.util.Retrier; import google.registry.xjc.XjcXmlTransformer; import google.registry.xjc.rdereport.XjcRdeReportReport; import google.registry.xml.XmlException; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.net.HttpURLConnection; import java.net.SocketTimeoutException; +import java.util.Map; import java.util.Optional; import org.bouncycastle.openpgp.PGPPublicKey; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.ArgumentCaptor; /** Unit tests for {@link RdeReportAction}. */ @DualDatabaseTest @@ -82,21 +89,20 @@ public class RdeReportActionTest { private final FakeResponse response = new FakeResponse(); private final EscrowTaskRunner runner = mock(EscrowTaskRunner.class); + private final URLFetchService urlFetchService = mock(URLFetchService.class); + private final ArgumentCaptor request = ArgumentCaptor.forClass(HTTPRequest.class); + private final HTTPResponse httpResponse = mock(HTTPResponse.class); private final PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); private final GcsUtils gcsUtils = new GcsUtils(LocalStorageHelper.getOptions()); private final BlobId reportFile = BlobId.of("tub", "test_2006-06-06_full_S1_R0-report.xml.ghostryde"); - private final HttpURLConnection httpUrlConnection = mock(HttpURLConnection.class); - private final FakeUrlConnectionService urlConnectionService = - new FakeUrlConnectionService(httpUrlConnection); - private final ByteArrayOutputStream connectionOutputStream = new ByteArrayOutputStream(); private RdeReportAction createAction() { RdeReporter reporter = new RdeReporter(); reporter.reportUrlPrefix = "https://rde-report.example"; + reporter.urlFetchService = urlFetchService; reporter.password = "foo"; - reporter.urlConnectionService = urlConnectionService; reporter.retrier = new Retrier(new FakeSleeper(new FakeClock()), 3); RdeReportAction action = new RdeReportAction(); action.gcsUtils = gcsUtils; @@ -121,7 +127,6 @@ public class RdeReportActionTest { Cursor.create(RDE_UPLOAD, DateTime.parse("2006-06-07TZ"), Registry.get("test"))); gcsUtils.createFromBytes(reportFile, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 0)); - when(httpUrlConnection.getOutputStream()).thenReturn(connectionOutputStream); } @TestOfyAndSql @@ -137,22 +142,24 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock() throws Exception { - when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); - when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); + when(httpResponse.getResponseCode()).thenReturn(SC_OK); + when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); + when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8); assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); // Verify the HTTP request was correct. - verify(httpUrlConnection).setRequestMethod("PUT"); - assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https"); - assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001"); - verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8"); - verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28="); + assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); + assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); + assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); + Map headers = mapifyHeaders(request.getValue().getHeaders()); + assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); + assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28="); // Verify the payload XML was the same as what's in testdata/report.xml. - XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray()); + XjcRdeReportReport report = parseReport(request.getValue().getPayload()); assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); @@ -160,8 +167,9 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_withPrefix() throws Exception { - when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); - when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); + when(httpResponse.getResponseCode()).thenReturn(SC_OK); + when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); + when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); RdeReportAction action = createAction(); action.prefix = Optional.of("job-name/"); gcsUtils.delete(reportFile); @@ -174,14 +182,15 @@ public class RdeReportActionTest { assertThat(response.getPayload()).isEqualTo("OK test 2006-06-06T00:00:00.000Z\n"); // Verify the HTTP request was correct. - verify(httpUrlConnection).setRequestMethod("PUT"); - assertThat(httpUrlConnection.getURL().getProtocol()).isEqualTo("https"); - assertThat(httpUrlConnection.getURL().getPath()).endsWith("/test/20101017001"); - verify(httpUrlConnection).setRequestProperty("Content-Type", "text/xml; charset=utf-8"); - verify(httpUrlConnection).setRequestProperty("Authorization", "Basic dGVzdF9yeTpmb28="); + assertThat(request.getValue().getMethod()).isSameInstanceAs(PUT); + assertThat(request.getValue().getURL().getProtocol()).isEqualTo("https"); + assertThat(request.getValue().getURL().getPath()).endsWith("/test/20101017001"); + Map headers = mapifyHeaders(request.getValue().getHeaders()); + assertThat(headers).containsEntry("CONTENT_TYPE", "text/xml"); + assertThat(headers).containsEntry("AUTHORIZATION", "Basic dGVzdF9yeTpmb28="); // Verify the payload XML was the same as what's in testdata/report.xml. - XjcRdeReportReport report = parseReport(connectionOutputStream.toByteArray()); + XjcRdeReportReport report = parseReport(request.getValue().getPayload()); assertThat(report.getId()).isEqualTo("20101017001"); assertThat(report.getCrDate()).isEqualTo(DateTime.parse("2010-10-17T00:15:00.0Z")); assertThat(report.getWatermark()).isEqualTo(DateTime.parse("2010-10-17T00:00:00Z")); @@ -194,8 +203,9 @@ public class RdeReportActionTest { PGPPublicKey encryptKey = new FakeKeyringModule().get().getRdeStagingEncryptionKey(); gcsUtils.createFromBytes(newReport, Ghostryde.encode(REPORT_XML.read(), encryptKey)); tm().transact(() -> RdeRevision.saveRevision("test", DateTime.parse("2006-06-06TZ"), FULL, 1)); - when(httpUrlConnection.getResponseCode()).thenReturn(SC_OK); - when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); + when(httpResponse.getResponseCode()).thenReturn(SC_OK); + when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); + when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); } @@ -228,8 +238,9 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_badRequest_throws500WithErrorInfo() throws Exception { - when(httpUrlConnection.getResponseCode()).thenReturn(SC_BAD_REQUEST); - when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_BAD_XML.openStream()); + when(httpResponse.getResponseCode()).thenReturn(SC_BAD_REQUEST); + when(httpResponse.getContent()).thenReturn(IIRDEA_BAD_XML.read()); + when(urlFetchService.fetch(request.capture())).thenReturn(httpResponse); InternalServerErrorException thrown = assertThrows( InternalServerErrorException.class, @@ -240,17 +251,18 @@ public class RdeReportActionTest { @TestOfyAndSql void testRunWithLock_fetchFailed_throwsRuntimeException() throws Exception { class ExpectedThrownException extends RuntimeException {} - when(httpUrlConnection.getResponseCode()).thenThrow(new ExpectedThrownException()); + when(urlFetchService.fetch(any(HTTPRequest.class))).thenThrow(new ExpectedThrownException()); assertThrows( ExpectedThrownException.class, () -> createAction().runWithLock(loadRdeReportCursor())); } @TestOfyAndSql void testRunWithLock_socketTimeout_doesRetry() throws Exception { - when(httpUrlConnection.getInputStream()).thenReturn(IIRDEA_GOOD_XML.openStream()); - when(httpUrlConnection.getResponseCode()) + when(httpResponse.getResponseCode()).thenReturn(SC_OK); + when(httpResponse.getContent()).thenReturn(IIRDEA_GOOD_XML.read()); + when(urlFetchService.fetch(request.capture())) .thenThrow(new SocketTimeoutException()) - .thenReturn(SC_OK); + .thenReturn(httpResponse); createAction().runWithLock(loadRdeReportCursor()); assertThat(response.getStatus()).isEqualTo(200); assertThat(response.getContentType()).isEqualTo(PLAIN_TEXT_UTF_8); @@ -261,6 +273,14 @@ public class RdeReportActionTest { return loadByKey(Cursor.createVKey(RDE_REPORT, "test")).getCursorTime(); } + private static ImmutableMap mapifyHeaders(Iterable headers) { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + for (HTTPHeader header : headers) { + builder.put(Ascii.toUpperCase(header.getName().replace('-', '_')), header.getValue()); + } + return builder.build(); + } + private static XjcRdeReportReport parseReport(byte[] data) { try { return XjcXmlTransformer.unmarshal(XjcRdeReportReport.class, new ByteArrayInputStream(data));