WebDAV range request support is back!

This commit is contained in:
Sebastian Stenzel
2016-02-17 17:35:05 +01:00
parent 50e8a9e429
commit 57b40675ac
6 changed files with 438 additions and 2 deletions

View File

@@ -0,0 +1,76 @@
package org.cryptomator.frontend.webdav.jackrabbitservlet;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.util.Objects;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.jackrabbit.webdav.DavException;
import org.apache.jackrabbit.webdav.DavSession;
import org.apache.jackrabbit.webdav.io.OutputContext;
import org.apache.jackrabbit.webdav.lock.LockManager;
import org.cryptomator.filesystem.ReadableFile;
import org.cryptomator.filesystem.jackrabbit.FileLocator;
import org.eclipse.jetty.http.HttpHeader;
import com.google.common.io.ByteStreams;
/**
* Delivers only the requested range of bytes from a file.
*
* @see {@link https://tools.ietf.org/html/rfc7233#section-4}
*/
public class DavFileWithRange extends DavFile {
private final Pair<String, String> requestRange;
public DavFileWithRange(FilesystemResourceFactory factory, LockManager lockManager, DavSession session, FileLocator node, Pair<String, String> requestRange) throws DavException {
super(factory, lockManager, session, node);
this.requestRange = Objects.requireNonNull(requestRange);
}
@Override
public void spool(OutputContext outputContext) throws IOException {
outputContext.setModificationTime(node.lastModified().toEpochMilli());
if (!outputContext.hasStream()) {
return;
}
try (ReadableFile src = node.openReadable(); OutputStream out = outputContext.getOutputStream()) {
final long contentLength = src.size();
final Pair<Long, Long> range = getEffectiveRange(contentLength);
assert range.getLeft() >= 0;
assert range.getLeft() <= range.getRight();
assert range.getRight() <= contentLength;
final Long rangeLength = range.getRight() - range.getLeft() + 1;
outputContext.setContentLength(rangeLength);
outputContext.setProperty(HttpHeader.CONTENT_RANGE.asString(), contentRangeResponseHeader(range.getLeft(), range.getRight(), contentLength));
src.position(range.getLeft());
InputStream limitedIn = ByteStreams.limit(Channels.newInputStream(src), rangeLength);
ByteStreams.copy(limitedIn, out);
}
}
private String contentRangeResponseHeader(long firstByte, long lastByte, long completeLength) {
return String.format("bytes %d-%d/%d", firstByte, lastByte, completeLength);
}
private Pair<Long, Long> getEffectiveRange(long contentLength) {
try {
final Long lower = requestRange.getLeft().isEmpty() ? null : Long.valueOf(requestRange.getLeft());
final Long upper = requestRange.getRight().isEmpty() ? null : Long.valueOf(requestRange.getRight());
if (lower == null) {
return new ImmutablePair<Long, Long>(contentLength - upper, contentLength - 1);
} else if (upper == null) {
return new ImmutablePair<Long, Long>(lower, contentLength - 1);
} else {
return new ImmutablePair<Long, Long>(lower, Math.min(upper, contentLength - 1));
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid byte range: " + requestRange, e);
}
}
}

View File

@@ -0,0 +1,42 @@
package org.cryptomator.frontend.webdav.jackrabbitservlet;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.channels.Channels;
import org.apache.jackrabbit.webdav.DavException;
import org.apache.jackrabbit.webdav.DavSession;
import org.apache.jackrabbit.webdav.io.OutputContext;
import org.apache.jackrabbit.webdav.lock.LockManager;
import org.cryptomator.filesystem.ReadableFile;
import org.cryptomator.filesystem.jackrabbit.FileLocator;
import org.eclipse.jetty.http.HttpHeader;
import com.google.common.io.ByteStreams;
/**
* Sends the full file in reaction to an unsatisfiable range.
*
* @see {@link https://tools.ietf.org/html/rfc7233#section-4.2}
*/
public class DavFileWithUnsatisfiableRange extends DavFile {
public DavFileWithUnsatisfiableRange(FilesystemResourceFactory factory, LockManager lockManager, DavSession session, FileLocator node) throws DavException {
super(factory, lockManager, session, node);
}
@Override
public void spool(OutputContext outputContext) throws IOException {
outputContext.setModificationTime(node.lastModified().toEpochMilli());
if (!outputContext.hasStream()) {
return;
}
try (ReadableFile src = node.openReadable(); OutputStream out = outputContext.getOutputStream()) {
final long contentLength = src.size();
outputContext.setContentLength(contentLength);
outputContext.setProperty(HttpHeader.CONTENT_RANGE.asString(), "bytes */" + contentLength);
ByteStreams.copy(src, Channels.newChannel(out));
}
}
}

View File

@@ -8,7 +8,15 @@
*******************************************************************************/
package org.cryptomator.frontend.webdav.jackrabbitservlet;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.jackrabbit.webdav.DavException;
import org.apache.jackrabbit.webdav.DavMethods;
import org.apache.jackrabbit.webdav.DavResource;
import org.apache.jackrabbit.webdav.DavResourceFactory;
import org.apache.jackrabbit.webdav.DavResourceLocator;
@@ -18,9 +26,14 @@ import org.apache.jackrabbit.webdav.DavSession;
import org.apache.jackrabbit.webdav.lock.LockManager;
import org.cryptomator.filesystem.jackrabbit.FileLocator;
import org.cryptomator.filesystem.jackrabbit.FolderLocator;
import org.eclipse.jetty.http.HttpHeader;
class FilesystemResourceFactory implements DavResourceFactory {
private static final String RANGE_BYTE_PREFIX = "bytes=";
private static final char RANGE_SET_SEP = ',';
private static final char RANGE_SEP = '-';
private final LockManager lockManager;
public FilesystemResourceFactory() {
@@ -29,7 +42,11 @@ class FilesystemResourceFactory implements DavResourceFactory {
@Override
public DavResource createResource(DavResourceLocator locator, DavServletRequest request, DavServletResponse response) throws DavException {
return createResource(locator, request.getDavSession());
if (locator instanceof FileLocator && DavMethods.METHOD_GET.equals(request.getMethod()) && request.getHeader(HttpHeader.RANGE.asString()) != null) {
return createFileRange((FileLocator) locator, request.getDavSession(), request, response);
} else {
return createResource(locator, request.getDavSession());
}
}
@Override
@@ -53,4 +70,80 @@ class FilesystemResourceFactory implements DavResourceFactory {
return new DavFile(this, lockManager, session, file);
}
private DavFile createFileRange(FileLocator file, DavSession session, DavServletRequest request, DavServletResponse response) throws DavException {
// 404 for non-existing resources:
if (!file.exists()) {
throw new DavException(DavServletResponse.SC_NOT_FOUND);
}
// 200 for "normal" resources, if if-range is not satisified:
final String ifRangeHeader = request.getHeader(HttpHeader.IF_RANGE.asString());
if (!isIfRangeHeaderSatisfied(file, ifRangeHeader)) {
return createFile(file, session);
}
final String rangeHeader = request.getHeader(HttpHeader.RANGE.asString());
try {
// 206 for ranged resources:
final Pair<String, String> parsedRange = parseRangeRequestHeader(rangeHeader);
response.setStatus(DavServletResponse.SC_PARTIAL_CONTENT);
return new DavFileWithRange(this, lockManager, session, file, parsedRange);
} catch (DavException ex) {
if (ex.getErrorCode() == DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE) {
// 416 for unsatisfiable ranges:
response.setStatus(DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
return new DavFileWithUnsatisfiableRange(this, lockManager, session, file);
} else {
throw new DavException(ex.getErrorCode(), ex);
}
}
}
/**
* Processes the given range header field, if it is supported. Only headers containing a single byte range are supported.<br/>
* <code>
* bytes=100-200<br/>
* bytes=-500<br/>
* bytes=1000-
* </code>
*
* @return Tuple of lower and upper range.
* @throws DavException HTTP statuscode 400 for malformed requests. 416 if requested range is not supported.
*/
private Pair<String, String> parseRangeRequestHeader(String rangeHeader) throws DavException {
assert rangeHeader != null;
if (!rangeHeader.startsWith(RANGE_BYTE_PREFIX)) {
throw new DavException(DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
}
final String byteRangeSet = StringUtils.removeStartIgnoreCase(rangeHeader, RANGE_BYTE_PREFIX);
final String[] byteRanges = StringUtils.split(byteRangeSet, RANGE_SET_SEP);
if (byteRanges.length != 1) {
throw new DavException(DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
}
final String byteRange = byteRanges[0];
final String[] bytePos = StringUtils.splitPreserveAllTokens(byteRange, RANGE_SEP);
if (bytePos.length != 2 || bytePos[0].isEmpty() && bytePos[1].isEmpty()) {
throw new DavException(DavServletResponse.SC_BAD_REQUEST, "malformed range header: " + rangeHeader);
}
return new ImmutablePair<>(bytePos[0], bytePos[1]);
}
/**
* @return <code>true</code> if a partial response should be generated according to an If-Range precondition.
*/
private boolean isIfRangeHeaderSatisfied(FileLocator file, String ifRangeHeader) throws DavException {
if (ifRangeHeader == null) {
// no header set -> satisfied implicitly
return true;
} else {
try {
Instant expectedTime = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(ifRangeHeader));
Instant actualTime = file.lastModified();
return expectedTime.compareTo(actualTime) == 0;
} catch (DateTimeParseException e) {
throw new DavException(DavServletResponse.SC_BAD_REQUEST, "Unsupported If-Range header: " + ifRangeHeader);
}
}
}
}

View File

@@ -27,11 +27,17 @@ import org.apache.jackrabbit.webdav.lock.Type;
import org.apache.jackrabbit.webdav.server.AbstractWebdavServlet;
import org.cryptomator.filesystem.Folder;
import org.cryptomator.filesystem.jackrabbit.FileSystemResourceLocatorFactory;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.io.EofException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class WebDavServlet extends AbstractWebdavServlet {
private static final long serialVersionUID = -6632687979352625020L;
private static final Logger LOG = LoggerFactory.getLogger(WebDavServlet.class);
private final DavSessionProvider davSessionProvider;
private final DavLocatorFactory davLocatorFactory;
private final DavResourceFactory davResourceFactory;
@@ -77,6 +83,23 @@ public class WebDavServlet extends AbstractWebdavServlet {
throw new UnsupportedOperationException("Setting resourceFactory not supported.");
}
/* GET stuff */
@Override
protected void doGet(WebdavRequest request, WebdavResponse response, DavResource resource) throws IOException, DavException {
if (request.getHeader(HttpHeader.RANGE.asString()) != null) {
try {
super.doGet(request, response, resource);
} catch (EofException e) {
if (LOG.isDebugEnabled()) {
LOG.trace("Unexpected end of stream during delivery of partial content (client hung up).");
}
}
} else {
super.doGet(request, response, resource);
}
}
/* LOCK stuff */
@Override

View File

@@ -33,7 +33,7 @@ public class InMemoryWebDavServer {
server.setPort(8080);
server.start();
FileSystem fileSystem = cryptoFileSystem();
FileSystem fileSystem = inMemoryFileSystem();
ServletContextHandler servlet = server.addServlet(fileSystem, URI.create("http://localhost:8080/foo"));
servlet.addFilter(LoggingHttpFilter.class, "/*", EnumSet.of(DispatcherType.REQUEST));
servlet.start();

View File

@@ -15,8 +15,15 @@ import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import javax.xml.parsers.ParserConfigurationException;
@@ -27,6 +34,7 @@ import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.methods.ByteArrayRequestEntity;
import org.apache.commons.httpclient.methods.EntityEnclosingMethod;
import org.apache.commons.httpclient.methods.GetMethod;
@@ -51,11 +59,14 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
public class WebDavServerTest {
private static final WebDavServer SERVER = DaggerWebDavComponent.create().server();
private static final Logger LOG = LoggerFactory.getLogger(WebDavServerTest.class);
private String servletRoot;
private FileSystem fs;
private ServletContextHandler servlet;
@@ -311,4 +322,195 @@ public class WebDavServerTest {
Assert.assertFalse(fs.folder("dstFile").exists());
}
/* Range requests */
@Test
public void testGetWithUnsatisfiableRange() throws IOException {
final HttpClient client = new HttpClient();
// write test content:
final byte[] testContent = "hello world".getBytes();
try (WritableFile w = fs.file("foo.txt").openWritable()) {
w.write(ByteBuffer.wrap(testContent));
}
// check get response body:
final HttpMethod getMethod = new GetMethod(servletRoot + "/foo.txt");
getMethod.addRequestHeader("Range", "chunks=1-2");
final int statusCode = client.executeMethod(getMethod);
Assert.assertEquals(416, statusCode);
Assert.assertArrayEquals(testContent, getMethod.getResponseBody());
getMethod.releaseConnection();
}
@Test
public void testMultipleGetWithRangeAsync() throws IOException, URISyntaxException, InterruptedException {
final String testResourceUrl = servletRoot + "/foo.txt";
// prepare 8MiB test data:
final byte[] plaintextData = new byte[2097152 * Integer.BYTES];
final ByteBuffer plaintextDataByteBuffer = ByteBuffer.wrap(plaintextData);
for (int i = 0; i < 2097152; i++) {
plaintextDataByteBuffer.putInt(i);
}
try (WritableFile w = fs.file("foo.txt").openWritable()) {
plaintextDataByteBuffer.flip();
w.write(plaintextDataByteBuffer);
}
final MultiThreadedHttpConnectionManager cm = new MultiThreadedHttpConnectionManager();
cm.getParams().setDefaultMaxConnectionsPerHost(50);
final HttpClient client = new HttpClient(cm);
// multiple async range requests:
final List<ForkJoinTask<?>> tasks = new ArrayList<>();
final Random generator = new Random(System.currentTimeMillis());
final AtomicBoolean success = new AtomicBoolean(true);
// 10 full interrupted requests:
for (int i = 0; i < 10; i++) {
final ForkJoinTask<?> task = ForkJoinTask.adapt(() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl);
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 200) {
LOG.error("Invalid status code for interrupted full request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 crappy interrupted range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length);
final ForkJoinTask<?> task = ForkJoinTask.adapt(() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl);
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 206) {
LOG.error("Invalid status code for interrupted range request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 normal open range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length - 512);
final int upper = plaintextData.length - 1;
final ForkJoinTask<?> task = ForkJoinTask.adapt(() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl);
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 10];
final int bytesRead = IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for open range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for open range request");
success.set(false);
} else if (!Arrays.equals(expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for open range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 200 normal closed range requests:
for (int i = 0; i < 200; i++) {
final int pos1 = generator.nextInt(plaintextData.length - 512);
final int pos2 = pos1 + 512;
final ForkJoinTask<?> task = ForkJoinTask.adapt(() -> {
try {
final int lower = Math.min(pos1, pos2);
final int upper = Math.max(pos1, pos2);
final HttpMethod getMethod = new GetMethod(testResourceUrl);
getMethod.addRequestHeader("Range", "bytes=" + lower + "-" + upper);
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 1];
final int bytesRead = IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for closed range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for closed range request");
success.set(false);
} else if (!Arrays.equals(expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for closed range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
Collections.shuffle(tasks, generator);
final ForkJoinPool pool = new ForkJoinPool(4);
for (ForkJoinTask<?> task : tasks) {
pool.execute(task);
}
for (ForkJoinTask<?> task : tasks) {
task.join();
}
pool.shutdown();
cm.shutdown();
Assert.assertTrue(success.get());
}
@Test
public void testUnsatisfiableRangeRequest() throws IOException, URISyntaxException {
final String testResourceUrl = servletRoot + "/unsatisfiableRangeRequestTestFile.txt";
final HttpClient client = new HttpClient();
// prepare file content:
final byte[] fileContent = "This is some test file content.".getBytes();
// put request:
final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString());
putMethod.setRequestEntity(new ByteArrayRequestEntity(fileContent));
final int putResponse = client.executeMethod(putMethod);
putMethod.releaseConnection();
Assert.assertEquals(201, putResponse);
// get request:
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "chunks=1-2");
final int getResponse = client.executeMethod(getMethod);
final byte[] response = new byte[fileContent.length];
IOUtils.read(getMethod.getResponseBodyAsStream(), response);
getMethod.releaseConnection();
Assert.assertEquals(416, getResponse);
Assert.assertArrayEquals(fileContent, response);
}
}