From 48f544ef91e97b03d8734e1eaa12ae300b4d63f5 Mon Sep 17 00:00:00 2001 From: Sebastian Stenzel Date: Sun, 21 Jun 2015 22:11:15 +0200 Subject: [PATCH] - support for http range requests in new schema --- .../webdav/jackrabbit/EncryptedFile.java | 7 +- .../crypto/aes256/Aes256Cryptor.java | 113 +++++++++++++----- .../aes256/AesCryptographicConfiguration.java | 5 + .../aes256/LengthObfuscationInputStream.java | 31 ++--- .../crypto/aes256/MacInputStream.java | 44 ------- .../crypto/aes256/MacOutputStream.java | 37 ------ .../crypto/aes256/Aes256CryptorTest.java | 8 +- 7 files changed, 114 insertions(+), 131 deletions(-) delete mode 100644 main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacInputStream.java delete mode 100644 main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacOutputStream.java diff --git a/main/core/src/main/java/org/cryptomator/webdav/jackrabbit/EncryptedFile.java b/main/core/src/main/java/org/cryptomator/webdav/jackrabbit/EncryptedFile.java index 21630a677..fab70b343 100644 --- a/main/core/src/main/java/org/cryptomator/webdav/jackrabbit/EncryptedFile.java +++ b/main/core/src/main/java/org/cryptomator/webdav/jackrabbit/EncryptedFile.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; import java.nio.channels.OverlappingFileLockException; -import java.nio.channels.SeekableByteChannel; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; @@ -96,13 +95,13 @@ class EncryptedFile extends AbstractEncryptedNode implements FileConstants { if (Files.isRegularFile(filePath)) { outputContext.setModificationTime(Files.getLastModifiedTime(filePath).toMillis()); outputContext.setProperty(HttpHeader.ACCEPT_RANGES.asString(), HttpHeaderValue.BYTES.asString()); - try (final SeekableByteChannel channel = Files.newByteChannel(filePath, StandardOpenOption.READ)) { - final Long contentLength = cryptor.decryptedContentLength(channel); + try (final FileChannel c = FileChannel.open(filePath, StandardOpenOption.READ); final FileLock lock = c.lock(0L, Long.MAX_VALUE, true)) { + final Long contentLength = cryptor.decryptedContentLength(c); if (contentLength != null) { outputContext.setContentLength(contentLength); } if (outputContext.hasStream()) { - cryptor.decryptFile(channel, outputContext.getOutputStream()); + cryptor.decryptFile(c, outputContext.getOutputStream()); } } catch (EOFException e) { LOG.warn("Unexpected end of stream (possibly client hung up)."); diff --git a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/Aes256Cryptor.java b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/Aes256Cryptor.java index d231c511e..d18b77c84 100644 --- a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/Aes256Cryptor.java +++ b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/Aes256Cryptor.java @@ -23,8 +23,6 @@ import java.util.Arrays; import javax.crypto.BadPaddingException; import javax.crypto.Cipher; -import javax.crypto.CipherInputStream; -import javax.crypto.CipherOutputStream; import javax.crypto.IllegalBlockSizeException; import javax.crypto.Mac; import javax.crypto.NoSuchPaddingException; @@ -418,7 +416,7 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration { // reading ciphered input and MACs interleaved: long bytesDecrypted = 0; final InputStream in = new SeekableByteChannelInputStream(encryptedFile); - byte[] buffer = new byte[1024 * 1024 + 32]; + byte[] buffer = new byte[CONTENT_MAC_BLOCK + 32]; int n = 0; while ((n = IOUtils.read(in, buffer)) > 0) { if (n < 32) { @@ -447,32 +445,93 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration { @Override public Long decryptRange(SeekableByteChannel encryptedFile, OutputStream plaintextFile, long pos, long length) throws IOException, DecryptFailedException { - // read iv: + // read header: encryptedFile.position(0l); - final ByteBuffer countingIv = ByteBuffer.allocate(AES_BLOCK_LENGTH); - final int numIvBytesRead = encryptedFile.read(countingIv); - - // check validity of header: - if (numIvBytesRead != AES_BLOCK_LENGTH) { + final ByteBuffer headerBuf = ByteBuffer.allocate(96); + final int headerBytesRead = encryptedFile.read(headerBuf); + if (headerBytesRead != headerBuf.capacity()) { throw new IOException("Failed to read file header."); } - // seek relevant position and update iv: - long firstRelevantBlock = pos / AES_BLOCK_LENGTH; // cut of fraction! - long beginOfFirstRelevantBlock = firstRelevantBlock * AES_BLOCK_LENGTH; - long offsetInsideFirstRelevantBlock = pos - beginOfFirstRelevantBlock; - countingIv.putInt(AES_BLOCK_LENGTH - Integer.BYTES, (int) firstRelevantBlock); // int-cast is possible, as max file size is 64GiB + // read iv: + final byte[] iv = new byte[AES_BLOCK_LENGTH]; + headerBuf.position(0); + headerBuf.get(iv); - // fast forward stream: - encryptedFile.position(96l + beginOfFirstRelevantBlock); + // read content key: + final byte[] encryptedContentKeyBytes = new byte[32]; + headerBuf.position(32); + headerBuf.get(encryptedContentKeyBytes); + final byte[] contentKeyBytes = decryptHeaderData(encryptedContentKeyBytes, iv); - // generate cipher: - final Cipher cipher = this.aesCtrCipher(primaryMasterKey, countingIv.array(), Cipher.DECRYPT_MODE); + // read header mac: + final byte[] storedHeaderMac = new byte[32]; + headerBuf.position(64); + headerBuf.get(storedHeaderMac); - // read content - final InputStream in = new SeekableByteChannelInputStream(encryptedFile); - final InputStream cipheredIn = new CipherInputStream(in, cipher); - return IOUtils.copyLarge(cipheredIn, plaintextFile, offsetInsideFirstRelevantBlock, length); + // calculate mac over first 64 bytes of header: + final Mac headerMac = this.hmacSha256(hMacMasterKey); + headerBuf.position(0); + headerBuf.limit(64); + headerMac.update(headerBuf); + + // check header integrity: + if (!MessageDigest.isEqual(storedHeaderMac, headerMac.doFinal())) { + throw new MacAuthenticationFailedException("Header MAC authentication failed."); + } + + // find first relevant block: + final long startBlock = pos / CONTENT_MAC_BLOCK; // floor + final long startByte = startBlock * (CONTENT_MAC_BLOCK + 32) + 96l; + final long offsetFromFirstBlock = pos - startBlock * CONTENT_MAC_BLOCK; + + // derive nonce used in counter mode from IV by setting last 64bit to 0: + final ByteBuffer nonceBuf = ByteBuffer.wrap(iv.clone()); + nonceBuf.putLong(AES_BLOCK_LENGTH - Long.BYTES, startBlock * CONTENT_MAC_BLOCK / AES_BLOCK_LENGTH); + final byte[] nonce = nonceBuf.array(); + + // content decryption: + encryptedFile.position(startByte); + final SecretKey contentKey = new SecretKeySpec(contentKeyBytes, AES_KEY_ALGORITHM); + final Cipher cipher = this.aesCtrCipher(contentKey, nonce, Cipher.DECRYPT_MODE); + final Mac contentMac = this.hmacSha256(hMacMasterKey); + + try { + + // reading ciphered input and MACs interleaved: + long bytesWritten = 0; + final InputStream in = new SeekableByteChannelInputStream(encryptedFile); + byte[] buffer = new byte[CONTENT_MAC_BLOCK + 32]; + int n = 0; + while ((n = IOUtils.read(in, buffer)) > 0 && bytesWritten < length) { + if (n < 32) { + throw new DecryptFailedException("Invalid file content, missing MAC."); + } + + // check MAC of current block: + contentMac.update(buffer, 0, n - 32); + final byte[] calculatedMac = contentMac.doFinal(); + final byte[] storedMac = new byte[32]; + System.arraycopy(buffer, n - 32, storedMac, 0, 32); + if (!MessageDigest.isEqual(calculatedMac, storedMac)) { + throw new MacAuthenticationFailedException("Content MAC authentication failed."); + } + + // decrypt block: + final byte[] plaintext = cipher.update(buffer, 0, n - 32); + final int offset = (bytesWritten == 0) ? (int) offsetFromFirstBlock : 0; + final long pending = length - bytesWritten; + final int available = plaintext.length - offset; + final int currentBatch = (int) Math.min(pending, available); + + plaintextFile.write(plaintext, offset, currentBatch); + bytesWritten += currentBatch; + } + + return bytesWritten; + } finally { + destroyQuietly(contentKey); + } } /** @@ -507,16 +566,16 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration { final SecretKey contentKey = new SecretKeySpec(contentKeyBytes, AES_KEY_ALGORITHM); final Cipher cipher = this.aesCtrCipher(contentKey, nonce, Cipher.ENCRYPT_MODE); final Mac contentMac = this.hmacSha256(hMacMasterKey); - final OutputStream out = new SeekableByteChannelOutputStream(encryptedFile); - final OutputStream macOut = new MacOutputStream(out, contentMac); @SuppressWarnings("resource") - final OutputStream cipheredOut = new CipherOutputStream(macOut, cipher); + final OutputStream out = new SeekableByteChannelOutputStream(encryptedFile); // writing ciphered output and MACs interleaved: - byte[] buffer = new byte[1024 * 1024]; + final byte[] buffer = new byte[CONTENT_MAC_BLOCK]; int n = 0; while ((n = IOUtils.read(in, buffer)) > 0) { - cipheredOut.write(buffer, 0, n); + final byte[] ciphertext = cipher.update(buffer, 0, n); + out.write(ciphertext); + contentMac.update(ciphertext); final byte[] mac = contentMac.doFinal(); out.write(mac); } diff --git a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/AesCryptographicConfiguration.java b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/AesCryptographicConfiguration.java index 09a3e2f0a..6cf7408fc 100644 --- a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/AesCryptographicConfiguration.java +++ b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/AesCryptographicConfiguration.java @@ -81,6 +81,11 @@ interface AesCryptographicConfiguration { */ int AES_BLOCK_LENGTH = 16; + /** + * Number of bytes, a content block over which a MAC is calculated consists of. + */ + int CONTENT_MAC_BLOCK = 5 * 1024 * 1024; + /** * How to encode the encrypted file names safely. Base32 uses only alphanumeric characters and is case-insensitive. */ diff --git a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/LengthObfuscationInputStream.java b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/LengthObfuscationInputStream.java index 0b1defce2..90f741d8d 100644 --- a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/LengthObfuscationInputStream.java +++ b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/LengthObfuscationInputStream.java @@ -4,6 +4,8 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import org.apache.commons.io.IOUtils; + /** * Not thread-safe! */ @@ -25,7 +27,7 @@ public class LengthObfuscationInputStream extends FilterInputStream { private void choosePaddingLengthOnce() { if (paddingLength == -1) { - long upperBound = Math.min(inputBytesRead / 10, 16 * 1024 * 1024); // 10% of original bytes, but not more than 16MiBs + long upperBound = Math.min(Math.max(inputBytesRead / 10, 4096), 16 * 1024 * 1024); // 10% of original bytes (at least 4KiB), but not more than 16MiBs paddingLength = (int) (Math.random() * upperBound); } } @@ -59,8 +61,7 @@ public class LengthObfuscationInputStream extends FilterInputStream { @Override public int read(byte[] b, int off, int len) throws IOException { - final int n = in.read(b, 0, len); - final int bytesRead = Math.max(0, n); // EOF -> 0 + final int bytesRead = IOUtils.read(in, b, off, len); // 0 on EOF inputBytesRead += bytesRead; if (bytesRead == len) { @@ -68,16 +69,21 @@ public class LengthObfuscationInputStream extends FilterInputStream { } else if (bytesRead < len) { choosePaddingLengthOnce(); final int additionalBytesNeeded = len - bytesRead; - final int m = readFromPadding(b, bytesRead, additionalBytesNeeded); - final int additionalBytesRead = Math.max(0, m); // EOF -> 0 - return (n == -1 && m == -1) ? -1 : bytesRead + additionalBytesRead; + final int additionalBytesRead = readFromPadding(b, off + bytesRead, additionalBytesNeeded); + return (bytesRead == 0 && additionalBytesRead == 0) ? -1 : bytesRead + additionalBytesRead; } else { // bytesRead > len: - throw new IllegalStateException("read more bytes than requested."); + throw new IllegalStateException("Read more bytes than requested."); } } + /** + * @return bytes read from padding (0, if fully read) + */ private int readFromPadding(byte[] b, int off, int len) { + if (len < 0) { + throw new IllegalArgumentException("Length must not be negative"); + } if (paddingLength == -1) { throw new IllegalStateException("No padding length chosen yet."); } @@ -86,20 +92,15 @@ public class LengthObfuscationInputStream extends FilterInputStream { if (remainingPadding > len) { // padding available: for (int i = 0; i < len; i++) { - b[off + i] = padding[paddingBytesRead + i % padding.length]; + b[off + i] = padding[paddingBytesRead++ % padding.length]; } - paddingBytesRead += len; return len; - } else if (remainingPadding > 0) { + } else { // partly available: for (int i = 0; i < remainingPadding; i++) { - b[off + i] = padding[paddingBytesRead + i % padding.length]; + b[off + i] = padding[paddingBytesRead++ % padding.length]; } - paddingBytesRead += remainingPadding; return remainingPadding; - } else { - // end of stream AND padding - return -1; } } diff --git a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacInputStream.java b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacInputStream.java deleted file mode 100644 index 483214be5..000000000 --- a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacInputStream.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.cryptomator.crypto.aes256; - -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; - -import javax.crypto.Mac; - -/** - * Updates a {@link Mac} with the bytes read from this stream. - */ -@Deprecated -class MacInputStream extends FilterInputStream { - - private final Mac mac; - - /** - * @param in Stream from which to read contents, which will update the Mac. - * @param mac Mac to be updated during writes. - */ - public MacInputStream(InputStream in, Mac mac) { - super(in); - this.mac = mac; - } - - @Override - public int read() throws IOException { - int b = in.read(); - if (b != -1) { - mac.update((byte) b); - } - return b; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int read = in.read(b, off, len); - if (read > 0) { - mac.update(b, off, read); - } - return read; - } - -} diff --git a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacOutputStream.java b/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacOutputStream.java deleted file mode 100644 index 785478197..000000000 --- a/main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/MacOutputStream.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.cryptomator.crypto.aes256; - -import java.io.FilterOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -import javax.crypto.Mac; - -/** - * Updates a {@link Mac} with the bytes written to this stream. - */ -class MacOutputStream extends FilterOutputStream { - - private final Mac mac; - - /** - * @param out Stream to redirect contents to after updating the mac. - * @param mac Mac to be updated during writes. - */ - public MacOutputStream(OutputStream out, Mac mac) { - super(out); - this.mac = mac; - } - - @Override - public void write(int b) throws IOException { - mac.update((byte) b); - out.write(b); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - mac.update(b, off, len); - out.write(b, off, len); - } - -} diff --git a/main/crypto-aes/src/test/java/org/cryptomator/crypto/aes256/Aes256CryptorTest.java b/main/crypto-aes/src/test/java/org/cryptomator/crypto/aes256/Aes256CryptorTest.java index d4338d9f6..e0d5b4778 100644 --- a/main/crypto-aes/src/test/java/org/cryptomator/crypto/aes256/Aes256CryptorTest.java +++ b/main/crypto-aes/src/test/java/org/cryptomator/crypto/aes256/Aes256CryptorTest.java @@ -140,9 +140,9 @@ public class Aes256CryptorTest { @Test public void testPartialDecryption() throws IOException, DecryptFailedException, WrongPasswordException, UnsupportedKeyLengthException, EncryptFailedException { // our test plaintext data: - final byte[] plaintextData = new byte[65536 * Integer.BYTES]; + final byte[] plaintextData = new byte[524288 * Integer.BYTES]; final ByteBuffer bbIn = ByteBuffer.wrap(plaintextData); - for (int i = 0; i < 65536; i++) { + for (int i = 0; i < 524288; i++) { bbIn.putInt(i); } final InputStream plaintextIn = new ByteArrayInputStream(plaintextData); @@ -162,14 +162,14 @@ public class Aes256CryptorTest { // decrypt: final SeekableByteChannel encryptedIn = new ByteBufferBackedSeekableChannel(encryptedData); final ByteArrayOutputStream plaintextOut = new ByteArrayOutputStream(); - final Long numDecryptedBytes = cryptor.decryptRange(encryptedIn, plaintextOut, 25000 * Integer.BYTES, 30000 * Integer.BYTES); + final Long numDecryptedBytes = cryptor.decryptRange(encryptedIn, plaintextOut, 260000 * Integer.BYTES, 4000 * Integer.BYTES); IOUtils.closeQuietly(encryptedIn); IOUtils.closeQuietly(plaintextOut); Assert.assertTrue(numDecryptedBytes > 0); // check decrypted data: final byte[] result = plaintextOut.toByteArray(); - final byte[] expected = Arrays.copyOfRange(plaintextData, 25000 * Integer.BYTES, 55000 * Integer.BYTES); + final byte[] expected = Arrays.copyOfRange(plaintextData, 260000 * Integer.BYTES, 264000 * Integer.BYTES); Assert.assertArrayEquals(expected, result); }