1
0
mirror of https://github.com/google/nomulus synced 2026-07-19 06:22:33 +00:00

Handle too-large requests/responses (#3147)

100 MB is kind of arbitrary, but it's as good as any other limit.

D.1 numbers 1, 8, 9
This commit is contained in:
gbrodman
2026-07-17 12:10:20 -04:00
committed by GitHub
parent 71e9bc95a7
commit 6b74924067
12 changed files with 206 additions and 85 deletions
@@ -119,6 +119,18 @@ public abstract class HttpException extends RuntimeException {
}
}
/** Exception that causes a 413 response. */
public static final class PayloadTooLargeException extends HttpException {
public PayloadTooLargeException(String message) {
super(HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE, message, null);
}
@Override
public String getResponseCodeString() {
return "Payload Too Large";
}
}
/** Exception that causes a 404 response. */
public static final class NotFoundException extends HttpException {
public NotFoundException() {
@@ -32,7 +32,6 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.ByteStreams;
import com.google.common.io.CharStreams;
import com.google.common.net.MediaType;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
@@ -42,6 +41,7 @@ import com.google.protobuf.ByteString;
import dagger.Module;
import dagger.Provides;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.PayloadTooLargeException;
import google.registry.request.HttpException.UnsupportedMediaTypeException;
import google.registry.request.auth.AuthResult;
import google.registry.request.lock.LockHandler;
@@ -50,6 +50,7 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.servlet.http.HttpSession;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Map;
import java.util.Optional;
@@ -57,6 +58,8 @@ import java.util.Optional;
@Module
public final class RequestModule {
public static final int MAX_PAYLOAD_BYTES = 50 * 1024 * 1024;
private final HttpServletRequest req;
private final HttpServletResponse rsp;
private final AuthResult authResult;
@@ -167,9 +170,37 @@ public final class RequestModule {
@Provides
@Payload
static String providePayloadAsString(HttpServletRequest req) {
public static String providePayloadAsString(
@Payload byte[] payloadBytes, HttpServletRequest req) {
String charsetName = req.getCharacterEncoding();
Charset charset;
try {
return CharStreams.toString(req.getReader());
charset = (charsetName != null) ? Charset.forName(charsetName) : UTF_8;
} catch (IllegalArgumentException e) {
throw new UnsupportedMediaTypeException("Unsupported charset: " + charsetName, e);
}
return new String(payloadBytes, charset);
}
@Provides
@Payload
public static byte[] providePayloadAsBytes(HttpServletRequest req) {
try {
if (req.getContentLengthLong() > MAX_PAYLOAD_BYTES) {
throw new PayloadTooLargeException(
String.format(
"Payload size %d exceeds limit of %d bytes",
req.getContentLengthLong(), MAX_PAYLOAD_BYTES));
}
if (req.getInputStream() == null) {
return new byte[0];
}
byte[] bytes =
ByteStreams.toByteArray(ByteStreams.limit(req.getInputStream(), MAX_PAYLOAD_BYTES + 1));
if (bytes.length > MAX_PAYLOAD_BYTES) {
throw new PayloadTooLargeException("Payload exceeds maximum allowed size");
}
return bytes;
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -177,22 +208,8 @@ public final class RequestModule {
@Provides
@Payload
static byte[] providePayloadAsBytes(HttpServletRequest req) {
try {
return ByteStreams.toByteArray(req.getInputStream());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Provides
@Payload
static ByteString providePayloadAsByteString(HttpServletRequest req) {
try {
return ByteString.copyFrom(ByteStreams.toByteArray(req.getInputStream()));
} catch (IOException e) {
throw new RuntimeException(e);
}
public static ByteString providePayloadAsByteString(@Payload byte[] payloadBytes) {
return ByteString.copyFrom(payloadBytes);
}
@Provides
@@ -251,12 +268,10 @@ public final class RequestModule {
@Provides
@OptionalJsonPayload
public static Optional<JsonElement> provideJsonBody(HttpServletRequest req, Gson gson) {
try {
// GET requests return a null reader and thus a null JsonObject, which is fine
return Optional.ofNullable(gson.fromJson(req.getReader(), JsonElement.class));
} catch (IOException e) {
public static Optional<JsonElement> provideJsonBody(@Payload String payloadString, Gson gson) {
if (payloadString.isEmpty()) {
return Optional.empty();
}
return Optional.ofNullable(gson.fromJson(payloadString, JsonElement.class));
}
}
@@ -22,6 +22,7 @@ import static com.google.common.net.HttpHeaders.CONTENT_LENGTH;
import static com.google.common.net.HttpHeaders.CONTENT_TYPE;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.flogger.FluentLogger;
import com.google.common.io.ByteStreams;
import com.google.common.net.MediaType;
import java.io.ByteArrayInputStream;
@@ -37,6 +38,11 @@ import org.apache.commons.compress.utils.IOUtils;
/** Utilities for common functionality relating to {@link URLConnection}s. */
public final class UrlConnectionUtils {
private static final FluentLogger logger = FluentLogger.forEnclosingClass();
// 100 MB is arbitrary, not specifically selected for any reason
public static final int MAX_RESPONSE_PAYLOAD_BYTES = 100 * 1024 * 1024;
private UrlConnectionUtils() {}
/**
@@ -48,10 +54,26 @@ public final class UrlConnectionUtils {
* @see HttpURLConnection#getErrorStream()
*/
public static byte[] getResponseBytes(HttpURLConnection connection) throws IOException {
int responseCode = connection.getResponseCode();
try (InputStream is =
responseCode < 400 ? connection.getInputStream() : connection.getErrorStream()) {
return ByteStreams.toByteArray(is);
try {
if (connection.getContentLengthLong() > MAX_RESPONSE_PAYLOAD_BYTES) {
throw new IOException(
String.format(
"Response size %d exceeds limit of %d bytes",
connection.getContentLengthLong(), MAX_RESPONSE_PAYLOAD_BYTES));
} else if ((double) connection.getContentLengthLong() / MAX_RESPONSE_PAYLOAD_BYTES >= 0.9) {
logger.atSevere().log(
"Response from %s was within 90%% of the maximum response size", connection.getURL());
}
int responseCode = connection.getResponseCode();
try (InputStream is =
responseCode < 400 ? connection.getInputStream() : connection.getErrorStream()) {
byte[] bytes =
ByteStreams.toByteArray(ByteStreams.limit(is, MAX_RESPONSE_PAYLOAD_BYTES + 1));
if (bytes.length > MAX_RESPONSE_PAYLOAD_BYTES) {
throw new IOException("Response exceeds maximum allowed size");
}
return bytes;
}
} catch (NullPointerException e) {
return new byte[] {};
}
@@ -15,14 +15,23 @@
package google.registry.request;
import static com.google.common.truth.Truth.assertThat;
import static google.registry.request.RequestModule.provideJsonBody;
import static google.registry.request.RequestModule.provideJsonPayload;
import static google.registry.request.RequestModule.providePayloadAsBytes;
import static google.registry.request.RequestModule.providePayloadAsString;
import static google.registry.security.JsonHttpTestUtils.createServletInputStream;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import com.google.common.net.MediaType;
import com.google.gson.Gson;
import google.registry.request.HttpException.BadRequestException;
import google.registry.request.HttpException.PayloadTooLargeException;
import google.registry.request.HttpException.UnsupportedMediaTypeException;
import google.registry.tools.GsonUtils;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
/** Unit tests for {@link RequestModule}. */
@@ -72,4 +81,46 @@ final class RequestModuleTest {
UnsupportedMediaTypeException.class,
() -> provideJsonPayload(MediaType.JSON_UTF_8.withParameter("omg", "handel"), "{}", GSON));
}
@Test
void testProvidePayloadAsBytes_contentLengthExceedsLimit_throws413() {
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getContentLengthLong()).thenReturn((long) RequestModule.MAX_PAYLOAD_BYTES + 1);
PayloadTooLargeException thrown =
assertThrows(PayloadTooLargeException.class, () -> providePayloadAsBytes(req));
assertThat(thrown).hasMessageThat().contains("exceeds limit");
}
@Test
void testProvidePayloadAsBytes_streamExceedsLimit_throws413() throws Exception {
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getContentLengthLong()).thenReturn(-1L);
when(req.getInputStream())
.thenReturn(createServletInputStream(new byte[RequestModule.MAX_PAYLOAD_BYTES + 1]));
PayloadTooLargeException thrown =
assertThrows(PayloadTooLargeException.class, () -> providePayloadAsBytes(req));
assertThat(thrown).hasMessageThat().contains("exceeds maximum allowed size");
}
@Test
void testProvidePayloadAsString_invalidCharset_throws415() {
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getCharacterEncoding()).thenReturn("invalid-charset-name");
UnsupportedMediaTypeException thrown =
assertThrows(
UnsupportedMediaTypeException.class,
() -> providePayloadAsString("hello".getBytes(UTF_8), req));
assertThat(thrown).hasMessageThat().contains("Unsupported charset: invalid-charset-name");
}
@Test
void testProvideJsonBody_contentLengthExceedsLimit_throws413() {
HttpServletRequest req = mock(HttpServletRequest.class);
when(req.getContentLengthLong()).thenReturn((long) RequestModule.MAX_PAYLOAD_BYTES + 1);
PayloadTooLargeException thrown =
assertThrows(
PayloadTooLargeException.class,
() -> provideJsonBody(providePayloadAsString(providePayloadAsBytes(req), req), GSON));
assertThat(thrown).hasMessageThat().contains("exceeds limit");
}
}
@@ -31,6 +31,7 @@ import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
@@ -80,13 +81,13 @@ public class UrlConnectionUtilsTest {
"294");
String payload =
"""
--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r
Content-Disposition: form-data; name="lol"; filename="cat"\r
Content-Type: text/csv; charset=utf-8\r
\r
The nice people at the store say hello. ヘ(◕。◕ヘ)\r
--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r
""";
--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\r
Content-Disposition: form-data; name="lol"; filename="cat"\r
Content-Type: text/csv; charset=utf-8\r
\r
The nice people at the store say hello. ヘ(◕。◕ヘ)\r
--------------------------------AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA--\r
""";
verify(connection).setDoOutput(true);
verify(connection).getOutputStream();
assertThat(connectionOutputStream.toByteArray()).isEqualTo(payload.getBytes(UTF_8));
@@ -124,4 +125,27 @@ public class UrlConnectionUtilsTest {
when(connection.getResponseCode()).thenReturn(400);
assertThat(UrlConnectionUtils.getResponseBytes(connection)).isEmpty();
}
@Test
void testGetResponseBytes_contentLengthExceedsLimit_throwsException() {
HttpsURLConnection connection = mock(HttpsURLConnection.class);
when(connection.getContentLengthLong())
.thenReturn((long) UrlConnectionUtils.MAX_RESPONSE_PAYLOAD_BYTES + 1);
IOException thrown =
assertThrows(IOException.class, () -> UrlConnectionUtils.getResponseBytes(connection));
assertThat(thrown).hasMessageThat().contains("exceeds limit");
}
@Test
void testGetResponseBytes_streamExceedsLimit_throwsException() throws Exception {
HttpsURLConnection connection = mock(HttpsURLConnection.class);
when(connection.getContentLengthLong()).thenReturn(-1L);
when(connection.getResponseCode()).thenReturn(200);
when(connection.getInputStream())
.thenReturn(
new ByteArrayInputStream(new byte[UrlConnectionUtils.MAX_RESPONSE_PAYLOAD_BYTES + 1]));
IOException thrown =
assertThrows(IOException.class, () -> UrlConnectionUtils.getResponseBytes(connection));
assertThat(thrown).hasMessageThat().contains("exceeds maximum allowed size");
}
}
@@ -24,7 +24,10 @@ import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import google.registry.tools.GsonUtils;
import jakarta.servlet.ReadListener;
import jakarta.servlet.ServletInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.Map;
@@ -84,4 +87,32 @@ public final class JsonHttpTestUtils {
final StringWriter writer) {
return memoize(() -> getJsonResponse(writer));
}
public static ServletInputStream createServletInputStream(byte[] data) {
ByteArrayInputStream bais = new ByteArrayInputStream(data);
return new ServletInputStream() {
@Override
public boolean isFinished() {
return bais.available() == 0;
}
@Override
public boolean isReady() {
return true;
}
@Override
public void setReadListener(ReadListener listener) {}
@Override
public int read() {
return bais.read();
}
@Override
public int read(byte[] b, int off, int len) {
return bais.read(b, off, len);
}
};
}
}
@@ -21,10 +21,13 @@ import com.google.gson.Gson;
import google.registry.model.console.User;
import google.registry.persistence.transaction.JpaTestExtensions;
import google.registry.request.auth.AuthResult;
import google.registry.security.JsonHttpTestUtils;
import google.registry.testing.ConsoleApiParamsUtils;
import google.registry.testing.FakeClock;
import google.registry.testing.FakeResponse;
import google.registry.tools.GsonUtils;
import jakarta.servlet.ServletInputStream;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -51,4 +54,8 @@ public abstract class ConsoleActionBaseTestCase {
consoleApiParams = ConsoleApiParamsUtils.createFake(authResult);
response = (FakeResponse) consoleApiParams.response();
}
protected static ServletInputStream createServletInputStream(String data) {
return JsonHttpTestUtils.createServletInputStream(data.getBytes(StandardCharsets.UTF_8));
}
}
@@ -23,7 +23,6 @@ import static jakarta.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static jakarta.servlet.http.HttpServletResponse.SC_FORBIDDEN;
import static jakarta.servlet.http.HttpServletResponse.SC_OK;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -47,9 +46,7 @@ import google.registry.ui.server.console.ConsoleEppPasswordAction.EppPasswordDat
import google.registry.util.EmailMessage;
import jakarta.mail.internet.AddressException;
import jakarta.mail.internet.InternetAddress;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -167,16 +164,13 @@ class ConsoleEppPasswordActionTest extends ConsoleActionBaseTestCase {
AuthenticatedRegistrarAccessor.createForTesting(
ImmutableSetMultimap.of("TheRegistrar", OWNER));
when(consoleApiParams.request().getMethod()).thenReturn(Action.Method.POST.toString());
doReturn(
new BufferedReader(
new StringReader(
String.format(
eppPostData, registrarId, oldPassword, newPassword, newPasswordRepeat))))
.when(consoleApiParams.request())
.getReader();
Optional<EppPasswordData> maybePasswordChangeRequest =
ConsoleModule.provideEppPasswordChangeRequest(
GSON, RequestModule.provideJsonBody(consoleApiParams.request(), GSON));
GSON,
RequestModule.provideJsonBody(
String.format(
eppPostData, registrarId, oldPassword, newPassword, newPasswordRepeat),
GSON));
return new ConsoleEppPasswordAction(
consoleApiParams, authenticatedRegistrarAccessor, maybePasswordChangeRequest);
@@ -21,7 +21,6 @@ import static google.registry.testing.DatabaseHelper.loadSingleton;
import static google.registry.testing.DatabaseHelper.persistResource;
import static jakarta.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static jakarta.servlet.http.HttpServletResponse.SC_OK;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -43,9 +42,7 @@ import google.registry.util.EmailMessage;
import google.registry.util.RegistryEnvironment;
import jakarta.mail.internet.AddressException;
import jakarta.mail.internet.InternetAddress;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.time.Instant;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
@@ -219,12 +216,8 @@ class ConsoleUpdateRegistrarActionTest extends ConsoleActionBaseTestCase {
private ConsoleUpdateRegistrarAction createAction(String requestData) throws IOException {
when(consoleApiParams.request().getMethod()).thenReturn(Action.Method.POST.toString());
doReturn(new BufferedReader(new StringReader(requestData)))
.when(consoleApiParams.request())
.getReader();
Optional<Registrar> maybeRegistrarUpdateData =
ConsoleModule.provideRegistrar(
GSON, RequestModule.provideJsonBody(consoleApiParams.request(), GSON));
ConsoleModule.provideRegistrar(GSON, RequestModule.provideJsonBody(requestData, GSON));
return new ConsoleUpdateRegistrarAction(consoleApiParams, maybeRegistrarUpdateData);
}
}
@@ -23,7 +23,6 @@ import static google.registry.testing.DatabaseHelper.persistResource;
import static google.registry.testing.SqlHelper.saveRegistrar;
import static jakarta.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static jakarta.servlet.http.HttpServletResponse.SC_OK;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
@@ -42,9 +41,6 @@ import google.registry.testing.ConsoleApiParamsUtils;
import google.registry.testing.DeterministicStringGenerator;
import google.registry.testing.FakeResponse;
import google.registry.util.StringGenerator;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -222,20 +218,9 @@ class RegistrarsActionTest extends ConsoleActionBaseTestCase {
return new RegistrarsAction(
consoleApiParams, Optional.ofNullable(null), passwordGenerator, passcodeGenerator);
} else {
try {
doReturn(new BufferedReader(new StringReader(registrarParamMap.toString())))
.when(consoleApiParams.request())
.getReader();
} catch (IOException e) {
return new RegistrarsAction(
consoleApiParams,
Optional.ofNullable(null),
passwordGenerator,
passcodeGenerator);
}
Optional<Registrar> maybeRegistrar =
ConsoleModule.provideRegistrar(
GSON, RequestModule.provideJsonBody(consoleApiParams.request(), GSON));
GSON, RequestModule.provideJsonBody(registrarParamMap.toString(), GSON));
return new RegistrarsAction(
consoleApiParams, maybeRegistrar, passwordGenerator, passcodeGenerator);
}
@@ -19,7 +19,6 @@ import static google.registry.model.ImmutableObjectSubject.assertAboutImmutableO
import static google.registry.testing.DatabaseHelper.loadSingleton;
import static jakarta.servlet.http.HttpServletResponse.SC_FORBIDDEN;
import static jakarta.servlet.http.HttpServletResponse.SC_OK;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableList;
@@ -41,9 +40,7 @@ import google.registry.testing.DatabaseHelper;
import google.registry.testing.FakeResponse;
import google.registry.ui.server.console.ConsoleActionBaseTestCase;
import google.registry.ui.server.console.ConsoleModule;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
import org.junit.jupiter.api.Test;
@@ -143,13 +140,10 @@ public class RdapRegistrarFieldsActionTest extends ConsoleActionBaseTestCase {
consoleApiParams = ConsoleApiParamsUtils.createFake(AuthResult.createUser(user));
response = (FakeResponse) consoleApiParams.response();
when(consoleApiParams.request().getMethod()).thenReturn(Action.Method.POST.toString());
doReturn(new BufferedReader(new StringReader(uiRegistrarMap.toString())))
.when(consoleApiParams.request())
.getReader();
return new RdapRegistrarFieldsAction(
consoleApiParams,
registrarAccessor,
ConsoleModule.provideRegistrar(
GSON, RequestModule.provideJsonBody(consoleApiParams.request(), GSON)));
GSON, RequestModule.provideJsonBody(uiRegistrarMap.toString(), GSON)));
}
}
@@ -22,7 +22,6 @@ import static google.registry.testing.SqlHelper.saveRegistrar;
import static google.registry.util.DateTimeUtils.START_INSTANT;
import static jakarta.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
import static jakarta.servlet.http.HttpServletResponse.SC_OK;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableSet;
@@ -37,9 +36,7 @@ import google.registry.request.auth.AuthenticatedRegistrarAccessor;
import google.registry.testing.FakeResponse;
import google.registry.ui.server.console.ConsoleActionBaseTestCase;
import google.registry.ui.server.console.ConsoleModule;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.time.Instant;
import java.util.Optional;
import org.junit.jupiter.api.BeforeEach;
@@ -170,12 +167,8 @@ class SecurityActionTest extends ConsoleActionBaseTestCase {
String registrarId, String jsonBody, CertificateChecker certificateChecker)
throws IOException {
when(consoleApiParams.request().getMethod()).thenReturn(Action.Method.POST.toString());
doReturn(new BufferedReader(new StringReader(jsonBody)))
.when(consoleApiParams.request())
.getReader();
Optional<Registrar> maybeRegistrar =
ConsoleModule.provideRegistrar(
GSON, RequestModule.provideJsonBody(consoleApiParams.request(), GSON));
ConsoleModule.provideRegistrar(GSON, RequestModule.provideJsonBody(jsonBody, GSON));
return new SecurityAction(
consoleApiParams, certificateChecker, registrarAccessor, registrarId, maybeRegistrar);
}