mirrors/cryptomator
1
0
Fork 0
mirror of https://github.com/cryptomator/cryptomator.git synced 2026-05-14 08:41:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

- mutlithreaded encryption

Browse Source 
- moved to bytebuffer (experimental)
This commit is contained in:
Sebastian Stenzel
2015-07-27 22:25:24 +02:00
parent cdcc1626ce
commit aac9ead633
6 changed files with 175 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
main/core/src/main/java/org/cryptomator/webdav/jackrabbit/WebDavServlet.java
View File

@@ -89,6 +89,16 @@ public class WebDavServlet extends AbstractWebdavServlet {
this.davResourceFactory = resourceFactory;
}
@Override
protected void doPut(WebdavRequest request, WebdavResponse response, DavResource resource) throws IOException, DavException {
long t0 = System.nanoTime();
super.doPut(request, response, resource);
if (LOG.isDebugEnabled()) {
long t1 = System.nanoTime();
LOG.debug("PUT TIME: " + (t1 - t0) / 1000 / 1000.0 + " ms");
}
}
@Override
protected void doGet(WebdavRequest request, WebdavResponse response, DavResource resource) throws IOException, DavException {
long t0 = System.nanoTime();
@@ -101,7 +111,7 @@ public class WebDavServlet extends AbstractWebdavServlet {
}
if (LOG.isDebugEnabled()) {
long t1 = System.nanoTime();
LOG.debug("REQUEST TIME: " + (t1 - t0) / 1000 / 1000.0 + " ms");
LOG.debug("GET TIME: " + (t1 - t0) / 1000 / 1000.0 + " ms");
}
}

136
main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/Aes256Cryptor.java
View File

@@ -12,6 +12,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.SeekableByteChannel;
import java.nio.charset.StandardCharsets;
import java.security.InvalidAlgorithmParameterException;
@@ -41,6 +42,7 @@ import javax.crypto.IllegalBlockSizeException;
import javax.crypto.Mac;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.ShortBufferException;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import javax.security.auth.DestroyFailedException;
@@ -56,12 +58,15 @@ import org.cryptomator.crypto.exceptions.UnsupportedKeyLengthException;
import org.cryptomator.crypto.exceptions.UnsupportedVaultException;
import org.cryptomator.crypto.exceptions.WrongPasswordException;
import org.cryptomator.crypto.io.SeekableByteChannelInputStream;
import org.cryptomator.crypto.io.SeekableByteChannelOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration {
private static final Logger LOG = LoggerFactory.getLogger(Aes256Cryptor.class);
/**
* Defined in static initializer. Defaults to 256, but falls back to maximum value possible, if JCE Unlimited Strength Jurisdiction Policy Files isn't installed. Those files can be downloaded
* here: http://www.oracle.com/technetwork/java/javase/downloads/.
@@ -454,26 +459,35 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration {
final ExecutorService executorService = Executors.newFixedThreadPool(numWorkers);
final CompletionService<Void> completionService = new ExecutorCompletionService<>(executorService);
for (int i = 0; i < numWorkers; i++) {
final DecryptWorker worker = new DecryptWorker(lock, blockDone, currentBlock, inputQueue, authenticate, paddingRemovingOutputStream) {
final DecryptWorker worker = new DecryptWorker(lock, blockDone, currentBlock, inputQueue, authenticate, Channels.newChannel(paddingRemovingOutputStream)) {
private final Mac mac = hmacSha256(hMacMasterKey);
@Override
protected byte[] decrypt(Block block) {
protected ByteBuffer decrypt(Block block) {
final ByteBuffer nonceAndCounterBuf = ByteBuffer.allocate(AES_BLOCK_LENGTH);
nonceAndCounterBuf.put(nonce);
nonceAndCounterBuf.putLong(block.blockNumber * CONTENT_MAC_BLOCK / AES_BLOCK_LENGTH);
final byte[] nonceAndCounter = nonceAndCounterBuf.array();
final ByteBuffer input = ByteBuffer.wrap(block.buffer);
input.limit(block.buffer.length - mac.getMacLength());
final Cipher cipher = aesCtrCipher(fileKey, nonceAndCounter, Cipher.DECRYPT_MODE);
return cipher.update(block.buffer, 0, block.numBytes - mac.getMacLength());
try {
assert cipher.getOutputSize(block.buffer.length) == block.buffer.length;
final ByteBuffer output = ByteBuffer.allocate(input.limit());
cipher.update(input, output);
return output;
} catch (ShortBufferException e) {
throw new IllegalStateException("Output buffer size too short, even though outlen = inlen in CTR mode.", e);
}
}
@Override
protected void checkMac(Block block) throws MacAuthenticationFailedException {
mac.update(iv);
mac.update(longToByteArray(block.blockNumber));
mac.update(block.buffer, 0, block.numBytes - mac.getMacLength());
mac.update(block.buffer, 0, block.buffer.length - mac.getMacLength());
final byte[] calculatedMac = mac.doFinal();
final byte[] storedMac = Arrays.copyOfRange(block.buffer, block.numBytes - mac.getMacLength(), block.numBytes);
final byte[] storedMac = Arrays.copyOfRange(block.buffer, block.buffer.length - mac.getMacLength(), block.buffer.length);
if (!MessageDigest.isEqual(calculatedMac, storedMac)) {
throw new MacAuthenticationFailedException("Content MAC authentication failed.");
}
@@ -490,11 +504,11 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration {
final InputStream in = new SeekableByteChannelInputStream(encryptedFile);
final byte[] buffer = new byte[CONTENT_MAC_BLOCK + 32];
int n = 0;
int blockNumber = 0;
long blockNumber = 0;
try {
// read as many blocks from file as possible, but wait if queue is full:
while ((n = IOUtils.read(in, buffer)) > 0) {
final boolean consumedInTime = inputQueue.offer(new Block(n, Arrays.copyOf(buffer, n), blockNumber++), 1, TimeUnit.SECONDS);
final boolean consumedInTime = inputQueue.offer(new Block(Arrays.copyOf(buffer, n), blockNumber++), 1, TimeUnit.SECONDS);
if (!consumedInTime) {
// interrupt read loop and make room for some poisons:
inputQueue.clear();
@@ -506,8 +520,7 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration {
inputQueue.put(CryptoWorker.POISON);
}
} catch (InterruptedException e) {
// TODO
e.printStackTrace();
LOG.error("Thread interrupted", e);
}
// wait for decryption workers to finish:
@@ -519,10 +532,13 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration {
final Throwable cause = e.getCause();
if (cause instanceof IOException) {
throw (IOException) cause;
} else if (cause instanceof RuntimeException) {
throw (RuntimeException) cause;
} else {
LOG.error("Unexpected exception", e);
}
} catch (InterruptedException e) {
// TODO
e.printStackTrace();
LOG.error("Thread interrupted", e);
} finally {
// shutdown either after normal decryption or if ANY worker threw an exception:
executorService.shutdownNow();
@@ -653,10 +669,6 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration {
// chosse 8 byte random nonce and 8 byte counter set to zero:
final byte[] nonce = randomData(8);
final ByteBuffer nonceAndCounterBuf = ByteBuffer.allocate(AES_BLOCK_LENGTH);
nonceAndCounterBuf.put(nonce);
nonceAndCounterBuf.putLong(0L);
final byte[] nonceAndCounter = nonceAndCounterBuf.array();
// choose a random content key:
final byte[] fileKeyBytes = randomData(32);
@@ -672,24 +684,88 @@ public class Aes256Cryptor implements Cryptor, AesCryptographicConfiguration {
// content encryption:
final SecretKey fileKey = new SecretKeySpec(fileKeyBytes, AES_KEY_ALGORITHM);
final Cipher cipher = this.aesCtrCipher(fileKey, nonceAndCounter, Cipher.ENCRYPT_MODE);
final Mac contentMac = this.hmacSha256(hMacMasterKey);
@SuppressWarnings("resource")
final OutputStream out = new SeekableByteChannelOutputStream(encryptedFile);
// prepare some crypto workers:
final int numWorkers = Runtime.getRuntime().availableProcessors();
final Lock lock = new ReentrantLock();
final Condition blockDone = lock.newCondition();
final AtomicLong currentBlock = new AtomicLong();
final BlockingQueue<Block> inputQueue = new LinkedBlockingQueue<>(numWorkers);
final List<EncryptWorker> workers = new ArrayList<>();
final ExecutorService executorService = Executors.newFixedThreadPool(numWorkers);
final CompletionService<Void> completionService = new ExecutorCompletionService<>(executorService);
for (int i = 0; i < numWorkers; i++) {
final EncryptWorker worker = new EncryptWorker(lock, blockDone, currentBlock, inputQueue, encryptedFile) {
private final Mac mac = hmacSha256(hMacMasterKey);
@Override
protected void encrypt(Block block, ByteBuffer output) {
final ByteBuffer nonceAndCounterBuf = ByteBuffer.allocate(AES_BLOCK_LENGTH);
nonceAndCounterBuf.put(nonce);
nonceAndCounterBuf.putLong(block.blockNumber * CONTENT_MAC_BLOCK / AES_BLOCK_LENGTH);
final byte[] nonceAndCounter = nonceAndCounterBuf.array();
final Cipher cipher = aesCtrCipher(fileKey, nonceAndCounter, Cipher.ENCRYPT_MODE);
try {
assert cipher.getOutputSize(block.buffer.length) == block.buffer.length;
cipher.update(ByteBuffer.wrap(block.buffer), output);
} catch (ShortBufferException e) {
throw new IllegalStateException("Output buffer size too short, even though outlen = inlen in CTR mode.", e);
}
}
@Override
protected byte[] mac(long blockNumber, ByteBuffer ciphertext) {
mac.update(iv);
mac.update(longToByteArray(blockNumber));
mac.update(ciphertext);
return mac.doFinal();
}
};
workers.add(worker);
completionService.submit(worker);
}
// writing ciphered output and MACs interleaved:
final byte[] buffer = new byte[CONTENT_MAC_BLOCK];
int n = 0;
long blockNum = 0;
while ((n = IOUtils.read(in, buffer)) > 0) {
final byte[] ciphertext = cipher.update(buffer, 0, n);
out.write(ciphertext);
contentMac.update(iv);
contentMac.update(longToByteArray(blockNum));
contentMac.update(ciphertext);
final byte[] mac = contentMac.doFinal();
out.write(mac);
blockNum++;
long blockNumber = 0;
try {
// read as many blocks from file as possible, but wait if queue is full:
while ((n = IOUtils.read(in, buffer)) > 0) {
final boolean consumedInTime = inputQueue.offer(new Block(Arrays.copyOf(buffer, n), blockNumber++), 1, TimeUnit.SECONDS);
if (!consumedInTime) {
// interrupt read loop and make room for some poisons:
inputQueue.clear();
break;
}
}
// each worker has to swallow some poison:
for (int i = 0; i < numWorkers; i++) {
inputQueue.put(CryptoWorker.POISON);
}
} catch (InterruptedException e) {
LOG.error("Thread interrupted", e);
}
// wait for encryption workers to finish:
try {
for (int i = 0; i < numWorkers; i++) {
completionService.take().get();
}
} catch (ExecutionException e) {
final Throwable cause = e.getCause();
if (cause instanceof IOException) {
throw (IOException) cause;
} else if (cause instanceof RuntimeException) {
throw (RuntimeException) cause;
} else {
LOG.error("Unexpected exception", e);
}
} catch (InterruptedException e) {
LOG.error("Thread interrupted", e);
} finally {
// shutdown either after normal encryption or if ANY worker threw an exception:
executorService.shutdownNow();
}
destroyQuietly(fileKey);

4
main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/Block.java
View File

@@ -2,12 +2,10 @@ package org.cryptomator.crypto.aes256;
class Block {
final int numBytes;
final byte[] buffer;
final long blockNumber;
Block(int numBytes, byte[] buffer, long blockNumber) {
this.numBytes = numBytes;
Block(byte[] buffer, long blockNumber) {
this.buffer = buffer;
this.blockNumber = blockNumber;
}

9
main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/CryptoWorker.java
View File

@@ -1,6 +1,7 @@
package org.cryptomator.crypto.aes256;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicLong;
@@ -11,7 +12,7 @@ import org.cryptomator.crypto.exceptions.CryptingException;
abstract class CryptoWorker implements Callable<Void> {
static final Block POISON = new Block(0, new byte[0], -1L);
static final Block POISON = new Block(new byte[0], -1L);
final Lock lock;
final Condition blockDone;
@@ -34,7 +35,7 @@ abstract class CryptoWorker implements Callable<Void> {
// put poison back in for other threads:
break;
}
final byte[] processedBytes = this.process(block);
final ByteBuffer processedBytes = this.process(block);
lock.lock();
try {
while (currentBlock.get() != block.blockNumber) {
@@ -56,8 +57,8 @@ abstract class CryptoWorker implements Callable<Void> {
return null;
}
protected abstract byte[] process(Block block) throws CryptingException;
protected abstract ByteBuffer process(Block block) throws CryptingException;
protected abstract void write(byte[] processedBytes) throws IOException;
protected abstract void write(ByteBuffer processedBytes) throws IOException;
}

16
main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/DecryptWorker.java
View File

@@ -1,7 +1,8 @@
package org.cryptomator.crypto.aes256;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
@@ -14,17 +15,17 @@ import org.cryptomator.crypto.exceptions.MacAuthenticationFailedException;
abstract class DecryptWorker extends CryptoWorker implements AesCryptographicConfiguration {
private final boolean shouldAuthenticate;
private final OutputStream out;
private final WritableByteChannel out;
public DecryptWorker(Lock lock, Condition blockDone, AtomicLong currentBlock, BlockingQueue<Block> queue, boolean shouldAuthenticate, OutputStream out) {
public DecryptWorker(Lock lock, Condition blockDone, AtomicLong currentBlock, BlockingQueue<Block> queue, boolean shouldAuthenticate, WritableByteChannel out) {
super(lock, blockDone, currentBlock, queue);
this.shouldAuthenticate = shouldAuthenticate;
this.out = out;
}
@Override
protected byte[] process(Block block) throws CryptingException {
if (block.numBytes < 32) {
protected ByteBuffer process(Block block) throws CryptingException {
if (block.buffer.length < 32) {
throw new DecryptFailedException("Invalid file content, missing MAC.");
}
@@ -38,12 +39,13 @@ abstract class DecryptWorker extends CryptoWorker implements AesCryptographicCon
}
@Override
protected void write(byte[] processedBytes) throws IOException {
protected void write(ByteBuffer processedBytes) throws IOException {
processedBytes.flip();
out.write(processedBytes);
}
protected abstract void checkMac(Block block) throws MacAuthenticationFailedException;
protected abstract byte[] decrypt(Block block);
protected abstract ByteBuffer decrypt(Block block);
}

43
main/crypto-aes/src/main/java/org/cryptomator/crypto/aes256/EncryptWorker.java Normal file
View File

@@ -0,0 +1,43 @@
package org.cryptomator.crypto.aes256;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import org.cryptomator.crypto.exceptions.CryptingException;
abstract class EncryptWorker extends CryptoWorker implements AesCryptographicConfiguration {
private final WritableByteChannel out;
public EncryptWorker(Lock lock, Condition blockDone, AtomicLong currentBlock, BlockingQueue<Block> queue, WritableByteChannel out) {
super(lock, blockDone, currentBlock, queue);
this.out = out;
}
@Override
protected ByteBuffer process(Block block) throws CryptingException {
final ByteBuffer buf = ByteBuffer.allocateDirect(block.buffer.length + 32);
encrypt(block, buf);
final ByteBuffer ciphertextBuffer = buf.duplicate();
ciphertextBuffer.flip();
final byte[] mac = mac(block.blockNumber, ciphertextBuffer);
buf.put(mac);
return buf;
}
@Override
protected void write(ByteBuffer processedBytes) throws IOException {
processedBytes.flip();
out.write(processedBytes);
}
protected abstract byte[] mac(long blockNumber, ByteBuffer ciphertext);
protected abstract void encrypt(Block block, ByteBuffer ciphertext);
}