diff --git a/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/DavFileWithRange.java b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/DavFileWithRange.java new file mode 100644 index 000000000..283a5ce59 --- /dev/null +++ b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/DavFileWithRange.java @@ -0,0 +1,76 @@ +package org.cryptomator.frontend.webdav.jackrabbitservlet; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.util.Objects; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.jackrabbit.webdav.DavException; +import org.apache.jackrabbit.webdav.DavSession; +import org.apache.jackrabbit.webdav.io.OutputContext; +import org.apache.jackrabbit.webdav.lock.LockManager; +import org.cryptomator.filesystem.ReadableFile; +import org.cryptomator.filesystem.jackrabbit.FileLocator; +import org.eclipse.jetty.http.HttpHeader; + +import com.google.common.io.ByteStreams; + +/** + * Delivers only the requested range of bytes from a file. + * + * @see {@link https://tools.ietf.org/html/rfc7233#section-4} + */ +public class DavFileWithRange extends DavFile { + + private final Pair requestRange; + + public DavFileWithRange(FilesystemResourceFactory factory, LockManager lockManager, DavSession session, FileLocator node, Pair requestRange) throws DavException { + super(factory, lockManager, session, node); + this.requestRange = Objects.requireNonNull(requestRange); + } + + @Override + public void spool(OutputContext outputContext) throws IOException { + outputContext.setModificationTime(node.lastModified().toEpochMilli()); + if (!outputContext.hasStream()) { + return; + } + try (ReadableFile src = node.openReadable(); OutputStream out = outputContext.getOutputStream()) { + final long contentLength = src.size(); + final Pair range = getEffectiveRange(contentLength); + assert range.getLeft() >= 0; + assert range.getLeft() <= range.getRight(); + assert range.getRight() <= contentLength; + final Long rangeLength = range.getRight() - range.getLeft() + 1; + outputContext.setContentLength(rangeLength); + outputContext.setProperty(HttpHeader.CONTENT_RANGE.asString(), contentRangeResponseHeader(range.getLeft(), range.getRight(), contentLength)); + src.position(range.getLeft()); + InputStream limitedIn = ByteStreams.limit(Channels.newInputStream(src), rangeLength); + ByteStreams.copy(limitedIn, out); + } + } + + private String contentRangeResponseHeader(long firstByte, long lastByte, long completeLength) { + return String.format("bytes %d-%d/%d", firstByte, lastByte, completeLength); + } + + private Pair getEffectiveRange(long contentLength) { + try { + final Long lower = requestRange.getLeft().isEmpty() ? null : Long.valueOf(requestRange.getLeft()); + final Long upper = requestRange.getRight().isEmpty() ? null : Long.valueOf(requestRange.getRight()); + if (lower == null) { + return new ImmutablePair(contentLength - upper, contentLength - 1); + } else if (upper == null) { + return new ImmutablePair(lower, contentLength - 1); + } else { + return new ImmutablePair(lower, Math.min(upper, contentLength - 1)); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid byte range: " + requestRange, e); + } + } + +} diff --git a/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/DavFileWithUnsatisfiableRange.java b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/DavFileWithUnsatisfiableRange.java new file mode 100644 index 000000000..9e9e24b4d --- /dev/null +++ b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/DavFileWithUnsatisfiableRange.java @@ -0,0 +1,42 @@ +package org.cryptomator.frontend.webdav.jackrabbitservlet; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; + +import org.apache.jackrabbit.webdav.DavException; +import org.apache.jackrabbit.webdav.DavSession; +import org.apache.jackrabbit.webdav.io.OutputContext; +import org.apache.jackrabbit.webdav.lock.LockManager; +import org.cryptomator.filesystem.ReadableFile; +import org.cryptomator.filesystem.jackrabbit.FileLocator; +import org.eclipse.jetty.http.HttpHeader; + +import com.google.common.io.ByteStreams; + +/** + * Sends the full file in reaction to an unsatisfiable range. + * + * @see {@link https://tools.ietf.org/html/rfc7233#section-4.2} + */ +public class DavFileWithUnsatisfiableRange extends DavFile { + + public DavFileWithUnsatisfiableRange(FilesystemResourceFactory factory, LockManager lockManager, DavSession session, FileLocator node) throws DavException { + super(factory, lockManager, session, node); + } + + @Override + public void spool(OutputContext outputContext) throws IOException { + outputContext.setModificationTime(node.lastModified().toEpochMilli()); + if (!outputContext.hasStream()) { + return; + } + try (ReadableFile src = node.openReadable(); OutputStream out = outputContext.getOutputStream()) { + final long contentLength = src.size(); + outputContext.setContentLength(contentLength); + outputContext.setProperty(HttpHeader.CONTENT_RANGE.asString(), "bytes */" + contentLength); + ByteStreams.copy(src, Channels.newChannel(out)); + } + } + +} diff --git a/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/FilesystemResourceFactory.java b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/FilesystemResourceFactory.java index accca7915..829121c2b 100644 --- a/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/FilesystemResourceFactory.java +++ b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/FilesystemResourceFactory.java @@ -8,7 +8,15 @@ *******************************************************************************/ package org.cryptomator.frontend.webdav.jackrabbitservlet; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.jackrabbit.webdav.DavException; +import org.apache.jackrabbit.webdav.DavMethods; import org.apache.jackrabbit.webdav.DavResource; import org.apache.jackrabbit.webdav.DavResourceFactory; import org.apache.jackrabbit.webdav.DavResourceLocator; @@ -18,9 +26,14 @@ import org.apache.jackrabbit.webdav.DavSession; import org.apache.jackrabbit.webdav.lock.LockManager; import org.cryptomator.filesystem.jackrabbit.FileLocator; import org.cryptomator.filesystem.jackrabbit.FolderLocator; +import org.eclipse.jetty.http.HttpHeader; class FilesystemResourceFactory implements DavResourceFactory { + private static final String RANGE_BYTE_PREFIX = "bytes="; + private static final char RANGE_SET_SEP = ','; + private static final char RANGE_SEP = '-'; + private final LockManager lockManager; public FilesystemResourceFactory() { @@ -29,7 +42,11 @@ class FilesystemResourceFactory implements DavResourceFactory { @Override public DavResource createResource(DavResourceLocator locator, DavServletRequest request, DavServletResponse response) throws DavException { - return createResource(locator, request.getDavSession()); + if (locator instanceof FileLocator && DavMethods.METHOD_GET.equals(request.getMethod()) && request.getHeader(HttpHeader.RANGE.asString()) != null) { + return createFileRange((FileLocator) locator, request.getDavSession(), request, response); + } else { + return createResource(locator, request.getDavSession()); + } } @Override @@ -53,4 +70,80 @@ class FilesystemResourceFactory implements DavResourceFactory { return new DavFile(this, lockManager, session, file); } + private DavFile createFileRange(FileLocator file, DavSession session, DavServletRequest request, DavServletResponse response) throws DavException { + // 404 for non-existing resources: + if (!file.exists()) { + throw new DavException(DavServletResponse.SC_NOT_FOUND); + } + + // 200 for "normal" resources, if if-range is not satisified: + final String ifRangeHeader = request.getHeader(HttpHeader.IF_RANGE.asString()); + if (!isIfRangeHeaderSatisfied(file, ifRangeHeader)) { + return createFile(file, session); + } + + final String rangeHeader = request.getHeader(HttpHeader.RANGE.asString()); + try { + // 206 for ranged resources: + final Pair parsedRange = parseRangeRequestHeader(rangeHeader); + response.setStatus(DavServletResponse.SC_PARTIAL_CONTENT); + return new DavFileWithRange(this, lockManager, session, file, parsedRange); + } catch (DavException ex) { + if (ex.getErrorCode() == DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE) { + // 416 for unsatisfiable ranges: + response.setStatus(DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); + return new DavFileWithUnsatisfiableRange(this, lockManager, session, file); + } else { + throw new DavException(ex.getErrorCode(), ex); + } + } + } + + /** + * Processes the given range header field, if it is supported. Only headers containing a single byte range are supported.
+ * + * bytes=100-200
+ * bytes=-500
+ * bytes=1000- + * + * + * @return Tuple of lower and upper range. + * @throws DavException HTTP statuscode 400 for malformed requests. 416 if requested range is not supported. + */ + private Pair parseRangeRequestHeader(String rangeHeader) throws DavException { + assert rangeHeader != null; + if (!rangeHeader.startsWith(RANGE_BYTE_PREFIX)) { + throw new DavException(DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); + } + final String byteRangeSet = StringUtils.removeStartIgnoreCase(rangeHeader, RANGE_BYTE_PREFIX); + final String[] byteRanges = StringUtils.split(byteRangeSet, RANGE_SET_SEP); + if (byteRanges.length != 1) { + throw new DavException(DavServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); + } + final String byteRange = byteRanges[0]; + final String[] bytePos = StringUtils.splitPreserveAllTokens(byteRange, RANGE_SEP); + if (bytePos.length != 2 || bytePos[0].isEmpty() && bytePos[1].isEmpty()) { + throw new DavException(DavServletResponse.SC_BAD_REQUEST, "malformed range header: " + rangeHeader); + } + return new ImmutablePair<>(bytePos[0], bytePos[1]); + } + + /** + * @return true if a partial response should be generated according to an If-Range precondition. + */ + private boolean isIfRangeHeaderSatisfied(FileLocator file, String ifRangeHeader) throws DavException { + if (ifRangeHeader == null) { + // no header set -> satisfied implicitly + return true; + } else { + try { + Instant expectedTime = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(ifRangeHeader)); + Instant actualTime = file.lastModified(); + return expectedTime.compareTo(actualTime) == 0; + } catch (DateTimeParseException e) { + throw new DavException(DavServletResponse.SC_BAD_REQUEST, "Unsupported If-Range header: " + ifRangeHeader); + } + } + } + } diff --git a/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/WebDavServlet.java b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/WebDavServlet.java index 41ba9c3f7..7fd04ba7c 100644 --- a/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/WebDavServlet.java +++ b/main/frontend-webdav/src/main/java/org/cryptomator/frontend/webdav/jackrabbitservlet/WebDavServlet.java @@ -27,11 +27,17 @@ import org.apache.jackrabbit.webdav.lock.Type; import org.apache.jackrabbit.webdav.server.AbstractWebdavServlet; import org.cryptomator.filesystem.Folder; import org.cryptomator.filesystem.jackrabbit.FileSystemResourceLocatorFactory; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.io.EofException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class WebDavServlet extends AbstractWebdavServlet { private static final long serialVersionUID = -6632687979352625020L; + private static final Logger LOG = LoggerFactory.getLogger(WebDavServlet.class); + private final DavSessionProvider davSessionProvider; private final DavLocatorFactory davLocatorFactory; private final DavResourceFactory davResourceFactory; @@ -77,6 +83,23 @@ public class WebDavServlet extends AbstractWebdavServlet { throw new UnsupportedOperationException("Setting resourceFactory not supported."); } + /* GET stuff */ + + @Override + protected void doGet(WebdavRequest request, WebdavResponse response, DavResource resource) throws IOException, DavException { + if (request.getHeader(HttpHeader.RANGE.asString()) != null) { + try { + super.doGet(request, response, resource); + } catch (EofException e) { + if (LOG.isDebugEnabled()) { + LOG.trace("Unexpected end of stream during delivery of partial content (client hung up)."); + } + } + } else { + super.doGet(request, response, resource); + } + } + /* LOCK stuff */ @Override diff --git a/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/InMemoryWebDavServer.java b/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/InMemoryWebDavServer.java index 35f12fd24..7c649a6ac 100644 --- a/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/InMemoryWebDavServer.java +++ b/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/InMemoryWebDavServer.java @@ -33,7 +33,7 @@ public class InMemoryWebDavServer { server.setPort(8080); server.start(); - FileSystem fileSystem = cryptoFileSystem(); + FileSystem fileSystem = inMemoryFileSystem(); ServletContextHandler servlet = server.addServlet(fileSystem, URI.create("http://localhost:8080/foo")); servlet.addFilter(LoggingHttpFilter.class, "/*", EnumSet.of(DispatcherType.REQUEST)); servlet.start(); diff --git a/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/WebDavServerTest.java b/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/WebDavServerTest.java index 33cdce038..27cb245c3 100644 --- a/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/WebDavServerTest.java +++ b/main/frontend-webdav/src/test/java/org/cryptomator/frontend/webdav/WebDavServerTest.java @@ -15,8 +15,15 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.ForkJoinTask; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import javax.xml.parsers.ParserConfigurationException; @@ -27,6 +34,7 @@ import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.ByteArrayRequestEntity; import org.apache.commons.httpclient.methods.EntityEnclosingMethod; import org.apache.commons.httpclient.methods.GetMethod; @@ -51,11 +59,14 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.xml.sax.SAXException; public class WebDavServerTest { private static final WebDavServer SERVER = DaggerWebDavComponent.create().server(); + private static final Logger LOG = LoggerFactory.getLogger(WebDavServerTest.class); private String servletRoot; private FileSystem fs; private ServletContextHandler servlet; @@ -311,4 +322,195 @@ public class WebDavServerTest { Assert.assertFalse(fs.folder("dstFile").exists()); } + /* Range requests */ + + @Test + public void testGetWithUnsatisfiableRange() throws IOException { + final HttpClient client = new HttpClient(); + + // write test content: + final byte[] testContent = "hello world".getBytes(); + try (WritableFile w = fs.file("foo.txt").openWritable()) { + w.write(ByteBuffer.wrap(testContent)); + } + + // check get response body: + final HttpMethod getMethod = new GetMethod(servletRoot + "/foo.txt"); + getMethod.addRequestHeader("Range", "chunks=1-2"); + final int statusCode = client.executeMethod(getMethod); + Assert.assertEquals(416, statusCode); + Assert.assertArrayEquals(testContent, getMethod.getResponseBody()); + getMethod.releaseConnection(); + } + + @Test + public void testMultipleGetWithRangeAsync() throws IOException, URISyntaxException, InterruptedException { + final String testResourceUrl = servletRoot + "/foo.txt"; + + // prepare 8MiB test data: + final byte[] plaintextData = new byte[2097152 * Integer.BYTES]; + final ByteBuffer plaintextDataByteBuffer = ByteBuffer.wrap(plaintextData); + for (int i = 0; i < 2097152; i++) { + plaintextDataByteBuffer.putInt(i); + } + try (WritableFile w = fs.file("foo.txt").openWritable()) { + plaintextDataByteBuffer.flip(); + w.write(plaintextDataByteBuffer); + } + + final MultiThreadedHttpConnectionManager cm = new MultiThreadedHttpConnectionManager(); + cm.getParams().setDefaultMaxConnectionsPerHost(50); + final HttpClient client = new HttpClient(cm); + + // multiple async range requests: + final List> tasks = new ArrayList<>(); + final Random generator = new Random(System.currentTimeMillis()); + + final AtomicBoolean success = new AtomicBoolean(true); + + // 10 full interrupted requests: + for (int i = 0; i < 10; i++) { + final ForkJoinTask task = ForkJoinTask.adapt(() -> { + try { + final HttpMethod getMethod = new GetMethod(testResourceUrl); + final int statusCode = client.executeMethod(getMethod); + if (statusCode != 200) { + LOG.error("Invalid status code for interrupted full request"); + success.set(false); + } + getMethod.getResponseBodyAsStream().read(); + getMethod.getResponseBodyAsStream().close(); + getMethod.releaseConnection(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + tasks.add(task); + } + + // 50 crappy interrupted range requests: + for (int i = 0; i < 50; i++) { + final int lower = generator.nextInt(plaintextData.length); + final ForkJoinTask task = ForkJoinTask.adapt(() -> { + try { + final HttpMethod getMethod = new GetMethod(testResourceUrl); + getMethod.addRequestHeader("Range", "bytes=" + lower + "-"); + final int statusCode = client.executeMethod(getMethod); + if (statusCode != 206) { + LOG.error("Invalid status code for interrupted range request"); + success.set(false); + } + getMethod.getResponseBodyAsStream().read(); + getMethod.getResponseBodyAsStream().close(); + getMethod.releaseConnection(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + tasks.add(task); + } + + // 50 normal open range requests: + for (int i = 0; i < 50; i++) { + final int lower = generator.nextInt(plaintextData.length - 512); + final int upper = plaintextData.length - 1; + final ForkJoinTask task = ForkJoinTask.adapt(() -> { + try { + final HttpMethod getMethod = new GetMethod(testResourceUrl); + getMethod.addRequestHeader("Range", "bytes=" + lower + "-"); + final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1); + final int statusCode = client.executeMethod(getMethod); + final byte[] responseBody = new byte[upper - lower + 10]; + final int bytesRead = IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody); + getMethod.releaseConnection(); + if (statusCode != 206) { + LOG.error("Invalid status code for open range request"); + success.set(false); + } else if (upper - lower + 1 != bytesRead) { + LOG.error("Invalid response length for open range request"); + success.set(false); + } else if (!Arrays.equals(expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) { + LOG.error("Invalid response body for open range request"); + success.set(false); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + tasks.add(task); + } + + // 200 normal closed range requests: + for (int i = 0; i < 200; i++) { + final int pos1 = generator.nextInt(plaintextData.length - 512); + final int pos2 = pos1 + 512; + final ForkJoinTask task = ForkJoinTask.adapt(() -> { + try { + final int lower = Math.min(pos1, pos2); + final int upper = Math.max(pos1, pos2); + final HttpMethod getMethod = new GetMethod(testResourceUrl); + getMethod.addRequestHeader("Range", "bytes=" + lower + "-" + upper); + final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1); + final int statusCode = client.executeMethod(getMethod); + final byte[] responseBody = new byte[upper - lower + 1]; + final int bytesRead = IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody); + getMethod.releaseConnection(); + if (statusCode != 206) { + LOG.error("Invalid status code for closed range request"); + success.set(false); + } else if (upper - lower + 1 != bytesRead) { + LOG.error("Invalid response length for closed range request"); + success.set(false); + } else if (!Arrays.equals(expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) { + LOG.error("Invalid response body for closed range request"); + success.set(false); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + tasks.add(task); + } + + Collections.shuffle(tasks, generator); + + final ForkJoinPool pool = new ForkJoinPool(4); + for (ForkJoinTask task : tasks) { + pool.execute(task); + } + for (ForkJoinTask task : tasks) { + task.join(); + } + pool.shutdown(); + cm.shutdown(); + + Assert.assertTrue(success.get()); + } + + @Test + public void testUnsatisfiableRangeRequest() throws IOException, URISyntaxException { + final String testResourceUrl = servletRoot + "/unsatisfiableRangeRequestTestFile.txt"; + final HttpClient client = new HttpClient(); + + // prepare file content: + final byte[] fileContent = "This is some test file content.".getBytes(); + + // put request: + final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString()); + putMethod.setRequestEntity(new ByteArrayRequestEntity(fileContent)); + final int putResponse = client.executeMethod(putMethod); + putMethod.releaseConnection(); + Assert.assertEquals(201, putResponse); + + // get request: + final HttpMethod getMethod = new GetMethod(testResourceUrl.toString()); + getMethod.addRequestHeader("Range", "chunks=1-2"); + final int getResponse = client.executeMethod(getMethod); + final byte[] response = new byte[fileContent.length]; + IOUtils.read(getMethod.getResponseBodyAsStream(), response); + getMethod.releaseConnection(); + Assert.assertEquals(416, getResponse); + Assert.assertArrayEquals(fileContent, response); + } + }