Changes to filesystem API and nio implementation

* Partial implementation of nio filesystem
* Removed timeouts from openReadable and openWritable
* Added convenience methods for copying
* Added utility to support deadlock safe opening of multiple files
This commit is contained in:
Markus Kreusch
2015-12-17 23:46:58 +01:00
parent 58524e5099
commit 25eed3dc4a
29 changed files with 725 additions and 165 deletions

View File

@@ -10,8 +10,6 @@ package org.cryptomator.crypto.fs;
import java.io.UncheckedIOException;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.cryptomator.crypto.engine.Cryptor;
import org.cryptomator.filesystem.File;
@@ -37,13 +35,13 @@ public class CryptoFile extends CryptoNode implements File {
}
@Override
public ReadableFile openReadable(long timeout, TimeUnit unit) throws TimeoutException {
public ReadableFile openReadable() {
// TODO Auto-generated method stub
return null;
}
@Override
public WritableFile openWritable(long timeout, TimeUnit unit) throws TimeoutException {
public WritableFile openWritable() {
// TODO Auto-generated method stub
return null;
}
@@ -53,4 +51,9 @@ public class CryptoFile extends CryptoNode implements File {
return parent.toString() + name;
}
@Override
public int compareTo(File o) {
return toString().compareTo(o.toString());
}
}

View File

@@ -8,12 +8,8 @@
*******************************************************************************/
package org.cryptomator.crypto.fs;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.cryptomator.crypto.engine.Cryptor;
import org.cryptomator.filesystem.File;
@@ -50,12 +46,13 @@ public class CryptoFileSystem extends CryptoFolder implements FileSystem {
}
assert masterkeyFile.exists() : "A CryptoFileSystem can not exist without a masterkey file.";
final File backupFile = physicalRoot.file(MASTERKEY_BACKUP_FILENAME);
backupMasterKeyFileSilently(masterkeyFile, backupFile);
masterkeyFile.copyTo(backupFile);
}
private static boolean decryptMasterKeyFile(Cryptor cryptor, File masterkeyFile, CharSequence passphrase) {
try (ReadableFile file = masterkeyFile.openReadable(1, TimeUnit.SECONDS)) {
// TODO we need to read the whole file but can not be sure about the buffer size:
try (ReadableFile file = masterkeyFile.openReadable()) {
// TODO we need to read the whole file but can not be sure about the
// buffer size:
final ByteBuffer bigEnoughBuffer = ByteBuffer.allocate(500);
file.read(bigEnoughBuffer);
bigEnoughBuffer.flip();
@@ -63,25 +60,13 @@ public class CryptoFileSystem extends CryptoFolder implements FileSystem {
final byte[] fileContents = new byte[bigEnoughBuffer.remaining()];
bigEnoughBuffer.get(fileContents);
return cryptor.readKeysFromMasterkeyFile(fileContents, passphrase);
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock masterkey file in time. " + masterkeyFile, e));
}
}
private static void encryptMasterKeyFile(Cryptor cryptor, File masterkeyFile, CharSequence passphrase) {
try (WritableFile file = masterkeyFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile file = masterkeyFile.openWritable()) {
final byte[] fileContents = cryptor.writeKeysToMasterkeyFile(passphrase);
file.write(ByteBuffer.wrap(fileContents));
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock masterkey file in time. " + masterkeyFile, e));
}
}
private static void backupMasterKeyFileSilently(File masterkeyFile, File backupFile) {
try (ReadableFile src = masterkeyFile.openReadable(1, TimeUnit.SECONDS); WritableFile dst = backupFile.openWritable(1, TimeUnit.SECONDS)) {
src.copyTo(dst);
} catch (TimeoutException e) {
LOG.warn("Failed to lock masterkey file (" + masterkeyFile + ") or backup file (" + backupFile + ") in time. Skipping backup.");
}
}
@@ -115,11 +100,9 @@ public class CryptoFileSystem extends CryptoFolder implements FileSystem {
physicalDataRoot().create(mode);
final File dirFile = physicalFile();
final String directoryId = getDirectoryId();
try (WritableFile writable = dirFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile writable = dirFile.openWritable()) {
final ByteBuffer buf = ByteBuffer.wrap(directoryId.getBytes());
writable.write(buf);
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock directory file in time. " + dirFile, e));
}
physicalFolder().create(FolderCreateMode.INCLUDING_PARENTS);
}

View File

@@ -9,13 +9,10 @@
package org.cryptomator.crypto.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
@@ -46,15 +43,13 @@ class CryptoFolder extends CryptoNode implements Folder {
if (directoryId.get() == null) {
File dirFile = physicalFile();
if (dirFile.exists()) {
try (ReadableFile readable = dirFile.openReadable(1, TimeUnit.SECONDS)) {
try (ReadableFile readable = dirFile.openReadable()) {
final ByteBuffer buf = ByteBuffer.allocate(64);
readable.read(buf);
buf.flip();
byte[] bytes = new byte[buf.remaining()];
buf.get(bytes);
directoryId.set(new String(bytes));
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock directory file in time." + dirFile, e));
}
} else {
directoryId.compareAndSet(null, UUID.randomUUID().toString());
@@ -125,11 +120,9 @@ class CryptoFolder extends CryptoNode implements Folder {
}
assert parent.exists();
final String directoryId = getDirectoryId();
try (WritableFile writable = dirFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile writable = dirFile.openWritable()) {
final ByteBuffer buf = ByteBuffer.wrap(directoryId.getBytes());
writable.write(buf);
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock directory file in time." + dirFile, e));
}
physicalFolder().create(FolderCreateMode.INCLUDING_PARENTS);
}
@@ -150,12 +143,11 @@ class CryptoFolder extends CryptoNode implements Folder {
target.physicalFile().parent().get().create(FolderCreateMode.INCLUDING_PARENTS);
assert target.physicalFile().parent().get().exists();
try (WritableFile src = this.physicalFile().openWritable(1, TimeUnit.SECONDS); WritableFile dst = target.physicalFile().openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile src = this.physicalFile().openWritable(); WritableFile dst = target.physicalFile().openWritable()) {
src.moveTo(dst);
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock file for moving (src: " + this + ", dst: " + target + ")", e));
}
// directoryId is now used by target, we must no longer use the same id (we'll generate a new one when needed)
// directoryId is now used by target, we must no longer use the same id
// (we'll generate a new one when needed)
directoryId.set(null);
}

View File

@@ -0,0 +1,37 @@
package org.cryptomator.filesystem;
class Copier {
public static void copy(Folder source, Folder destination) {
assertFoldersAreNotNested(source, destination);
destination.delete();
destination.create(FolderCreateMode.INCLUDING_PARENTS);
source.files().forEach(sourceFile -> {
File destinationFile = destination.file(sourceFile.name());
copy(sourceFile, destinationFile);
});
source.folders().forEach(sourceFolder -> {
Folder destinationFolder = destination.folder(sourceFolder.name());
sourceFolder.copyTo(destinationFolder);
});
}
private static void assertFoldersAreNotNested(Folder source, Folder destination) {
if (source.isAncestorOf(destination)) {
throw new IllegalArgumentException("Can not copy parent to child directory (src: " + source + ", dst: " + destination + ")");
}
if (destination.isAncestorOf(source)) {
throw new IllegalArgumentException("Can not copy child to parent directory (src: " + source + ", dst: " + destination + ")");
}
}
public static void copy(File source, File destination) {
try (OpenFiles openFiles = DeadlockSafeFileOpener.withReadable(source).andWritable(destination).open()) {
openFiles.readable(source).copyTo(openFiles.writable(destination));
}
}
}

View File

@@ -0,0 +1,61 @@
package org.cryptomator.filesystem;
import static java.lang.String.format;
import java.util.HashMap;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.function.Consumer;
public class DeadlockSafeFileOpener {
public static DeadlockSafeFileOpener withReadable(File file) {
return new DeadlockSafeFileOpener().andReadable(file);
}
public static DeadlockSafeFileOpener withWritable(File file) {
return new DeadlockSafeFileOpener().andWritable(file);
}
private final SortedMap<File, Consumer<File>> filesWithOperation = new TreeMap<>();
private final Map<File, ReadableFile> readableFiles = new HashMap<>();
private final Map<File, WritableFile> writableFiles = new HashMap<>();
private DeadlockSafeFileOpener() {
}
public DeadlockSafeFileOpener andReadable(File file) {
if (filesWithOperation.put(file, this::openReadable) != null) {
throw new IllegalArgumentException(format("File %s already marked for opening", file));
}
return this;
}
public DeadlockSafeFileOpener andWritable(File file) {
if (filesWithOperation.put(file, this::openWritable) != null) {
throw new IllegalArgumentException(format("File %s already marked for opening", file));
}
return this;
}
private void openReadable(File file) {
readableFiles.put(file, file.openReadable());
}
private void openWritable(File file) {
writableFiles.put(file, file.openWritable());
}
public OpenFiles open() {
try {
filesWithOperation.forEach((file, openAction) -> openAction.accept(file));
} catch (RuntimeException e) {
OpenFiles.cleanup(readableFiles.values(), writableFiles.values());
throw e;
}
return new OpenFiles(readableFiles, writableFiles);
}
}

View File

@@ -7,15 +7,13 @@ package org.cryptomator.filesystem;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* A {@link File} in a {@link FileSystem}.
*
* @author Markus Kreusch
*/
public interface File extends Node {
public interface File extends Node, Comparable<File> {
/**
* <p>
@@ -34,17 +32,13 @@ public interface File extends Node {
* In addition implementations may block to lock the required IO resources
* to read the file.
*
* @param timeout
* the timeout to wait until failing with a
* {@link TimeoutException}
* @param unit
* the {@link TimeUnit} of the timeout value
* @return a {@link ReadableFile} to work with
* @throws UncheckedIOException
* if an {@link IOException} occurs while opening the file, the
* file does not exist or is a directory
*/
ReadableFile openReadable(long timeout, TimeUnit unit) throws UncheckedIOException, TimeoutException;
ReadableFile openReadable() throws UncheckedIOException;
/**
* <p>
@@ -54,8 +48,9 @@ public interface File extends Node {
* <p>
* An implementation guarantees, that per {@link FileSystem} and
* {@code File} only one {@link WritableFile} is open at a time. A
* {@link WritableFile} is open when returned from this method and not yet
* closed using {@link WritableFile#close()}.<br>
* {@code WritableFile} is open when returned from this method and not yet
* closed using {@link WritableFile#close()} or
* {@link WritableFile#delete()}.<br>
* In addition while a {@code WritableFile} is open no {@link ReadableFile}
* can be open and vice versa.
* <p>
@@ -65,16 +60,15 @@ public interface File extends Node {
* In addition implementations may block to lock the required IO resources
* to read the file.
*
* @param timeout
* the timeout to wait until failing with a
* {@link TimeoutException}
* @param unit
* the {@link TimeUnit} of the timeout value
* @return a {@link WritableFile} to work with
* @throws UncheckedIOException
* if an {@link IOException} occurs while opening the file or
* the file is a directory
*/
WritableFile openWritable(long timeout, TimeUnit unit) throws UncheckedIOException, TimeoutException;
WritableFile openWritable() throws UncheckedIOException;
default void copyTo(File destination) {
Copier.copy(this, destination);
}
}

View File

@@ -8,8 +8,6 @@ package org.cryptomator.filesystem;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Stream;
/**
@@ -58,59 +56,53 @@ public interface Folder extends Node {
Folder folder(String name) throws UncheckedIOException;
/**
* Creates the directory, if it doesn't exist yet. No effect, if folder already exists. After successful invocation {@link #exists()} will return <code>true</code>.
* Creates the directory, if it doesn't exist yet. No effect, if folder
* already exists.
*
* @param mode Depending on this option either the attempt is made to recursively create all parent directories or an exception is thrown if the parent doesn't exist yet.
* @throws UncheckedIOException wrapping an {@link FileNotFoundException}, if mode is {@link FolderCreateMode#FAIL_IF_PARENT_IS_MISSING FAIL_IF_PARENT_IS_MISSING} and parent doesn't exist.
* @param mode
* Depending on this option either the attempt is made to
* recursively create all parent directories or an exception is
* thrown if the parent doesn't exist yet.
* @throws UncheckedIOException
* wrapping an {@link FileNotFoundException}, if mode is
* {@link FolderCreateMode#FAIL_IF_PARENT_IS_MISSING
* FAIL_IF_PARENT_IS_MISSING} and parent doesn't exist.
*/
void create(FolderCreateMode mode) throws UncheckedIOException;
/**
* Recusively copies this directory and all its contents to (not into) the given destination, creating nonexisting parent directories.
* If the target exists it is deleted before performing the copy.
* Recusively copies this directory and all its contents to (not into) the
* given destination, creating nonexisting parent directories. If the target
* exists it is deleted before performing the copy.
*
* @param target Destination folder. Must not be a descendant of this folder.
* @param target
* Destination folder. Must not be a descendant of this folder.
*/
default void copyTo(Folder target) throws UncheckedIOException {
if (this.isAncestorOf(target)) {
throw new IllegalArgumentException("Can not copy parent to child directory (src: " + this + ", dst: " + target + ")");
}
// remove previous contents:
if (target.exists()) {
target.delete();
}
// make sure target directory exists:
target.create(FolderCreateMode.INCLUDING_PARENTS);
assert target.exists();
// copy files:
files().forEach(srcFile -> {
try (ReadableFile src = srcFile.openReadable(1, TimeUnit.SECONDS)) {
final File dstFile = target.file(srcFile.name());
try (WritableFile dst = dstFile.openWritable(1, TimeUnit.MILLISECONDS)) {
src.copyTo(dst);
} catch (TimeoutException e) {
throw new IllegalStateException("Destination file (" + dstFile + ") must not exist yet, thus can't be locked.");
}
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock source file (" + srcFile + ") in time.", e));
}
});
// copy subdirectories:
folders().forEach(folder -> folder.copyTo(target.folder(folder.name())));
Copier.copy(this, target);
}
/**
* Deletes the directory including all child elements. Afterwards {@link #exists()} will return <code>false</code>.
* <p>
* Deletes the directory including all child elements.
* <p>
* If the directory does not exist this method does nothing.
*/
void delete() throws UncheckedIOException;
default void delete() throws UncheckedIOException {
if (!exists()) {
return;
}
folders().forEach(Folder::delete);
files().forEach(file -> {
try (WritableFile writableFile = file.openWritable()) {
writableFile.delete();
}
});
}
/**
* Moves this directory and its contents to the given destination. If the target exists it is deleted before performing the move.
* Afterwards {@link #exists()} will return <code>false</code> for this folder and any child nodes.
* Moves this directory and its contents to the given destination. If the
* target exists it is deleted before performing the move.
*/
void moveTo(Folder target);
@@ -135,9 +127,11 @@ public interface Folder extends Node {
}
/**
* Recursively checks whether this folder or any subfolder contains the given node.
* Recursively checks whether this folder or any subfolder contains the
* given node.
*
* @param node Potential child, grandchild, ...
* @param node
* Potential child, grandchild, ...
* @return <code>true</code> if this folder is an ancestor of the node.
*/
default boolean isAncestorOf(Node node) {

View File

@@ -0,0 +1,63 @@
package org.cryptomator.filesystem;
import java.io.UncheckedIOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class OpenFiles implements AutoCloseable {
private final static Logger LOG = LoggerFactory.getLogger(OpenFiles.class);
private final Map<File, ReadableFile> readableFiles;
private final Map<File, WritableFile> writableFiles;
public OpenFiles(Map<File, ReadableFile> readableFiles, Map<File, WritableFile> writableFiles) {
this.readableFiles = readableFiles;
this.writableFiles = writableFiles;
}
@Override
public void close() throws UncheckedIOException {
OpenFiles.cleanup(readableFiles.values(), writableFiles.values());
}
public ReadableFile readable(File file) {
return readableFiles.computeIfAbsent(file, fileNotOpenForReading -> {
throw new IllegalArgumentException(String.format("File %s is not open for reading", fileNotOpenForReading));
});
}
public WritableFile writable(File file) {
return writableFiles.computeIfAbsent(file, fileNotOpenForWriting -> {
throw new IllegalArgumentException(String.format("File %s is not open for writing", fileNotOpenForWriting));
});
}
static void cleanup(Collection<ReadableFile> readableFiles, Collection<WritableFile> writableFiles) {
Iterator<AutoCloseable> iterator = Stream.concat(readableFiles.stream(), writableFiles.stream()).iterator();
UncheckedIOException firstException = null;
while (iterator.hasNext()) {
AutoCloseable openFile = iterator.next();
try {
openFile.close();
} catch (UncheckedIOException e) {
if (firstException == null) {
firstException = e;
} else {
firstException.addSuppressed(e);
}
} catch (Exception e) {
LOG.error("Unexpected exception during close on " + openFile.getClass().getSimpleName(), e);
}
}
if (firstException != null) {
throw firstException;
}
}
}

View File

@@ -7,7 +7,7 @@ package org.cryptomator.filesystem;
import java.io.UncheckedIOException;
public interface ReadableFile extends File, ReadableBytes, AutoCloseable {
public interface ReadableFile extends ReadableBytes, AutoCloseable {
void copyTo(WritableFile other) throws UncheckedIOException;

View File

@@ -8,16 +8,33 @@ package org.cryptomator.filesystem;
import java.io.UncheckedIOException;
import java.time.Instant;
public interface WritableFile extends File, WritableBytes, AutoCloseable {
public interface WritableFile extends WritableBytes, AutoCloseable {
void moveTo(WritableFile other) throws UncheckedIOException;
void setLastModified(Instant instant) throws UncheckedIOException;
/**
* <p>
* Deletes this file from the file system.
* <p>
* Deleting a file causes it to be {@link WritableFile#close() closed}.
*/
void delete() throws UncheckedIOException;
void truncate() throws UncheckedIOException;
/**
* <p>
* Closes this {@code WritableFile} which finally commits all operations
* performed on it to the underlying file system.
* <p>
* After a {@code WritableFile} has been closed all other operations will
* throw an {@link UncheckedIOException}.
* <p>
* Invoking this method on a {@link WritableFile} which has already been
* closed does nothing.
*/
@Override
void close() throws UncheckedIOException;

View File

@@ -12,14 +12,13 @@ import java.io.FileNotFoundException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.cryptomator.filesystem.File;
import org.cryptomator.filesystem.ReadableFile;
import org.cryptomator.filesystem.WritableFile;
class InMemoryFile extends InMemoryNode implements ReadableFile, WritableFile {
class InMemoryFile extends InMemoryNode implements File, ReadableFile, WritableFile {
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private ByteBuffer content = ByteBuffer.wrap(new byte[0]);
@@ -29,29 +28,17 @@ class InMemoryFile extends InMemoryNode implements ReadableFile, WritableFile {
}
@Override
public ReadableFile openReadable(long timeout, TimeUnit unit) throws TimeoutException {
public ReadableFile openReadable() {
if (!exists()) {
throw new UncheckedIOException(new FileNotFoundException(this.name() + " does not exist"));
}
try {
if (!lock.readLock().tryLock(timeout, unit)) {
throw new TimeoutException("Failed to open " + name() + " for reading within time limit.");
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
lock.readLock().lock();
return this;
}
@Override
public WritableFile openWritable(long timeout, TimeUnit unit) throws TimeoutException {
try {
if (!lock.writeLock().tryLock(timeout, unit)) {
throw new TimeoutException("Failed to open " + name() + " for writing within time limit.");
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
public WritableFile openWritable() {
lock.writeLock().lock();
final InMemoryFolder parent = parent().get();
parent.children.compute(this.name(), (k, v) -> {
if (v != null && v != this) {
@@ -123,7 +110,7 @@ class InMemoryFile extends InMemoryNode implements ReadableFile, WritableFile {
// returning null removes the entry.
return null;
});
assert!this.exists();
assert !this.exists();
}
@Override
@@ -141,4 +128,9 @@ class InMemoryFile extends InMemoryNode implements ReadableFile, WritableFile {
return parent.toString() + name;
}
@Override
public int compareTo(File o) {
return toString().compareTo(o.toString());
}
}

View File

@@ -64,7 +64,7 @@ public class InMemoryFileSystemTest {
Thread.sleep(1);
// write "hello world" to foo
try (WritableFile writable = fooFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile writable = fooFile.openWritable()) {
writable.write(ByteBuffer.wrap("hello world".getBytes()));
}
Assert.assertTrue(fooFile.exists());
@@ -79,7 +79,7 @@ public class InMemoryFileSystemTest {
Thread.sleep(1);
// write "dlrow olleh" to foo
try (WritableFile writable = fooFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile writable = fooFile.openWritable()) {
writable.write(ByteBuffer.wrap("dlrow olleh".getBytes()));
}
Assert.assertTrue(fooFile.exists());
@@ -98,7 +98,7 @@ public class InMemoryFileSystemTest {
Assert.assertEquals(0, fs.files().count());
// write "hello world" to foo
try (WritableFile writable = fooFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile writable = fooFile.openWritable()) {
writable.write(ByteBuffer.wrap("hello".getBytes()));
writable.write(ByteBuffer.wrap(" ".getBytes()));
writable.write(ByteBuffer.wrap("world".getBytes()));
@@ -107,8 +107,8 @@ public class InMemoryFileSystemTest {
// copy foo to bar
File barFile = fs.file("bar.txt");
try (WritableFile writable = barFile.openWritable(1, TimeUnit.SECONDS)) {
try (ReadableFile readable = fooFile.openReadable(1, TimeUnit.SECONDS)) {
try (WritableFile writable = barFile.openWritable()) {
try (ReadableFile readable = fooFile.openReadable()) {
readable.copyTo(writable);
}
}
@@ -117,8 +117,8 @@ public class InMemoryFileSystemTest {
// move bar to baz
File bazFile = fs.file("baz.txt");
try (WritableFile src = barFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile dst = bazFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile src = barFile.openWritable()) {
try (WritableFile dst = bazFile.openWritable()) {
src.moveTo(dst);
}
}
@@ -127,7 +127,7 @@ public class InMemoryFileSystemTest {
// read "hello world" from baz
final ByteBuffer readBuf = ByteBuffer.allocate(5);
try (ReadableFile readable = bazFile.openReadable(1, TimeUnit.SECONDS)) {
try (ReadableFile readable = bazFile.openReadable()) {
readable.read(readBuf, 6);
}
Assert.assertEquals("world", new String(readBuf.array()));
@@ -143,8 +143,8 @@ public class InMemoryFileSystemTest {
fooBarFolder.create(FolderCreateMode.INCLUDING_PARENTS);
// create some files inside foo/bar/
try (WritableFile writable1 = test1File.openWritable(1, TimeUnit.SECONDS); //
WritableFile writable2 = test2File.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile writable1 = test1File.openWritable(); //
WritableFile writable2 = test2File.openWritable()) {
writable1.write(ByteBuffer.wrap("hello".getBytes()));
writable2.write(ByteBuffer.wrap("world".getBytes()));
}

1
main/filesystem-nio/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
/target/

View File

@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Copyright (c) 2015 Markus Kreusch This file is licensed under the terms
of the MIT license. See the LICENSE.txt file for more info. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.cryptomator</groupId>
<artifactId>main</artifactId>
<version>0.11.0-SNAPSHOT</version>
</parent>
<artifactId>filesystem-nio</artifactId>
<dependencies>
<dependency>
<groupId>org.cryptomator</groupId>
<artifactId>filesystem-api</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<description>FileSystem implementation to access the real file system of an operating system</description>
<name>Cryptomator NIO Filesystem</name>
</project>

View File

@@ -0,0 +1,18 @@
package org.cryptomator.filesystem.nio;
import java.nio.file.Path;
import java.util.Optional;
class DefaultNioNodeFactory implements NioNodeFactory {
@Override
public NioFile file(Optional<NioFolder> parent, Path path) {
return new NioFile(parent, path, this);
}
@Override
public NioFolder folder(Optional<NioFolder> parent, Path path) {
return new NioFolder(parent, path, this);
}
}

View File

@@ -0,0 +1,102 @@
package org.cryptomator.filesystem.nio;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.time.Instant;
import java.util.Optional;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.cryptomator.filesystem.File;
import org.cryptomator.filesystem.ReadableFile;
import org.cryptomator.filesystem.WritableFile;
class NioFile extends NioNode implements File {
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
public NioFile(Optional<NioFolder> parent, Path path, NioNodeFactory nodeFactory) {
super(parent, path, nodeFactory);
}
@Override
public ReadableFile openReadable() throws UncheckedIOException {
if (lock.getWriteHoldCount() > 0) {
throw new IllegalStateException("Current thread is currently reading this file");
}
lock.readLock().lock();
return new ReadableView();
}
@Override
public WritableFile openWritable() throws UncheckedIOException {
if (lock.getReadHoldCount() > 0) {
throw new IllegalStateException("Current thread is currently reading this file");
}
lock.readLock().lock();
return new WritableView();
}
private class ReadableView implements ReadableFile {
@Override
public void read(ByteBuffer target) throws UncheckedIOException {
}
@Override
public void read(ByteBuffer target, int position) throws UncheckedIOException {
}
@Override
public void copyTo(WritableFile other) throws UncheckedIOException {
}
@Override
public void close() throws UncheckedIOException {
}
}
private class WritableView implements WritableFile {
@Override
public void write(ByteBuffer source) throws UncheckedIOException {
}
@Override
public void write(ByteBuffer source, int position) throws UncheckedIOException {
}
@Override
public void moveTo(WritableFile other) throws UncheckedIOException {
}
@Override
public void setLastModified(Instant instant) throws UncheckedIOException {
}
@Override
public void delete() throws UncheckedIOException {
}
@Override
public void truncate() throws UncheckedIOException {
}
@Override
public void close() throws UncheckedIOException {
}
}
@Override
public int compareTo(File o) {
if (belongsToSameFilesystem(o)) {
assert o instanceof NioNode;
return path.compareTo(((NioFile) o).path);
} else {
throw new IllegalArgumentException("Can not mix File objects from different file systems");
}
}
}

View File

@@ -0,0 +1,18 @@
package org.cryptomator.filesystem.nio;
import java.nio.file.Path;
import java.util.Optional;
import org.cryptomator.filesystem.FileSystem;
public class NioFileSystem extends NioFolder implements FileSystem {
public static NioFileSystem rootedAt(Path root) {
return new NioFileSystem(root, new DefaultNioNodeFactory());
}
NioFileSystem(Path root, NioNodeFactory nodeFactory) {
super(Optional.empty(), root, nodeFactory);
}
}

View File

@@ -0,0 +1,85 @@
package org.cryptomator.filesystem.nio;
import static org.cryptomator.filesystem.FolderCreateMode.INCLUDING_PARENTS;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Optional;
import java.util.stream.Stream;
import org.cryptomator.filesystem.File;
import org.cryptomator.filesystem.Folder;
import org.cryptomator.filesystem.FolderCreateMode;
import org.cryptomator.filesystem.Node;
class NioFolder extends NioNode implements Folder {
private final WeakValuedCache<Path, NioFolder> folders = WeakValuedCache.usingLoader(this::folderFromPath);
private final WeakValuedCache<Path, NioFile> files = WeakValuedCache.usingLoader(this::fileFromPath);
public NioFolder(Optional<NioFolder> parent, Path path, NioNodeFactory nodeFactory) {
super(parent, path, nodeFactory);
}
@Override
public Stream<? extends Node> children() throws UncheckedIOException {
try {
return Files.list(path).map(this::childPathToNode);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
private NioNode childPathToNode(Path childPath) {
if (Files.isDirectory(childPath)) {
return folders.get(childPath);
} else {
return files.get(childPath);
}
}
private NioFile fileFromPath(Path path) {
return nodeFactory.file(Optional.of(this), path);
}
private NioFolder folderFromPath(Path path) {
return nodeFactory.folder(Optional.of(this), path);
}
@Override
public File file(String name) throws UncheckedIOException {
return files.get(path.resolve(name));
}
@Override
public Folder folder(String name) throws UncheckedIOException {
return folders.get(path.resolve(name));
}
@Override
public void create(FolderCreateMode mode) throws UncheckedIOException {
NioFolderCreateMode.valueOf(mode).create(path);
}
@Override
public void moveTo(Folder target) {
if (belongsToSameFilesystem(target)) {
internalMoveTo((NioFolder) target);
} else {
throw new IllegalArgumentException("Can only move a Folder to a Folder in the same FileSystem");
}
}
private void internalMoveTo(NioFolder target) {
try {
target.delete();
target.parent().ifPresent(folder -> folder.create(INCLUDING_PARENTS));
Files.move(path, target.path);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

View File

@@ -0,0 +1,41 @@
package org.cryptomator.filesystem.nio;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.cryptomator.filesystem.FolderCreateMode;
enum NioFolderCreateMode {
FAIL_IF_PARENT_IS_MISSING {
@Override
void create(Path folderPath) {
try {
Files.createDirectory(folderPath);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
},
INCLUDING_PARENTS {
@Override
void create(Path folderPath) {
try {
Files.createDirectories(folderPath);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
;
public static NioFolderCreateMode valueOf(FolderCreateMode mode) {
return valueOf(mode.name());
}
abstract void create(Path folderPath);
}

View File

@@ -0,0 +1,54 @@
package org.cryptomator.filesystem.nio;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.Optional;
import org.cryptomator.filesystem.Folder;
import org.cryptomator.filesystem.Node;
class NioNode implements Node {
protected final Optional<NioFolder> parent;
protected final Path path;
protected final NioNodeFactory nodeFactory;
public NioNode(Optional<NioFolder> parent, Path path, NioNodeFactory nodeFactory) {
this.path = path.toAbsolutePath();
this.nodeFactory = nodeFactory;
this.parent = parent;
}
boolean belongsToSameFilesystem(Node other) {
return other instanceof NioNode //
&& ((NioNode) other).nodeFactory == nodeFactory;
}
@Override
public String name() throws UncheckedIOException {
return path.getFileName().toString();
}
@Override
public Optional<? extends Folder> parent() throws UncheckedIOException {
return parent;
}
@Override
public boolean exists() throws UncheckedIOException {
return Files.exists(path);
}
@Override
public Instant lastModified() throws UncheckedIOException {
try {
return Files.getLastModifiedTime(path).toInstant();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}

View File

@@ -0,0 +1,12 @@
package org.cryptomator.filesystem.nio;
import java.nio.file.Path;
import java.util.Optional;
interface NioNodeFactory {
NioFile file(Optional<NioFolder> parent, Path path);
NioFolder folder(Optional<NioFolder> parent, Path path);
}

View File

@@ -0,0 +1,37 @@
package org.cryptomator.filesystem.nio;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
class WeakValuedCache<Key, Value> {
private final LoadingCache<Key, Value> delegate;
private WeakValuedCache(Function<Key, Value> loader) {
delegate = CacheBuilder.newBuilder() //
.weakValues() //
.build(new CacheLoader<Key, Value>() {
@Override
public Value load(Key key) {
return loader.apply(key);
}
});
}
public static <Key, Value> WeakValuedCache<Key, Value> usingLoader(Function<Key, Value> loader) {
return new WeakValuedCache<>(loader);
}
public Value get(Key key) {
try {
return delegate.get(key);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
}

View File

@@ -215,6 +215,7 @@
<modules>
<module>filesystem-api</module>
<module>filesystem-inmemory</module>
<module>filesystem-nio</module>
<module>crypto-layer</module>
<module>crypto-api</module>
<module>crypto-aes</module>

View File

@@ -1,14 +1,11 @@
package org.cryptomator.shortening;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.commons.codec.binary.Base32;
import org.apache.commons.codec.binary.BaseNCodec;
@@ -57,10 +54,8 @@ class FilenameShortener {
final File mappingFile = mappingFile(shortName);
if (!mappingFile.exists()) {
mappingFile.parent().get().create(FolderCreateMode.INCLUDING_PARENTS);
try (WritableFile writable = mappingFile.openWritable(1, TimeUnit.SECONDS)) {
try (WritableFile writable = mappingFile.openWritable()) {
writable.write(ByteBuffer.wrap(longName.getBytes(StandardCharsets.UTF_8)));
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock mapping file in time. " + mappingFile, e));
}
}
}
@@ -75,7 +70,7 @@ class FilenameShortener {
if (!mappingFile.exists()) {
throw new UncheckedIOException(new FileNotFoundException("Mapping file not found " + mappingFile));
} else {
try (ReadableFile readable = mappingFile.openReadable(1, TimeUnit.SECONDS)) {
try (ReadableFile readable = mappingFile.openReadable()) {
// TODO buffer might be to small
final ByteBuffer buf = ByteBuffer.allocate(1024);
readable.read(buf);
@@ -83,8 +78,6 @@ class FilenameShortener {
final byte[] bytes = new byte[buf.remaining()];
buf.get(bytes);
return new String(bytes, StandardCharsets.UTF_8);
} catch (TimeoutException e) {
throw new UncheckedIOException(new IOException("Failed to lock mapping file in time. " + mappingFile, e));
}
}
}

View File

@@ -1,14 +1,12 @@
package org.cryptomator.shortening;
import java.io.UncheckedIOException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.cryptomator.filesystem.File;
import org.cryptomator.filesystem.ReadableFile;
import org.cryptomator.filesystem.WritableFile;
class ShorteningFile extends ShorteningNode<File>implements File {
class ShorteningFile extends ShorteningNode<File> implements File {
private final FilenameShortener shortener;
@@ -18,21 +16,26 @@ class ShorteningFile extends ShorteningNode<File>implements File {
}
@Override
public ReadableFile openReadable(long timeout, TimeUnit unit) throws UncheckedIOException, TimeoutException {
return delegate.openReadable(timeout, unit);
public ReadableFile openReadable() throws UncheckedIOException {
return delegate.openReadable();
}
@Override
public WritableFile openWritable(long timeout, TimeUnit unit) throws UncheckedIOException, TimeoutException {
public WritableFile openWritable() throws UncheckedIOException {
if (shortener.isShortened(shortName())) {
shortener.saveMapping(name(), shortName());
}
return delegate.openWritable(timeout, unit);
return delegate.openWritable();
}
@Override
public String toString() {
return name();
return parent + name();
}
@Override
public int compareTo(File o) {
return toString().compareTo(o.toString());
}
}

View File

@@ -4,9 +4,11 @@ import org.cryptomator.filesystem.FileSystem;
import org.cryptomator.filesystem.Folder;
/**
* Filesystem implementation, that shortens filenames when they reach a certain threshold (inclusive).
* Shortening is done by SHA1-hashing those files, so a threshold below the length of the hashed files makes no sense.
* Hashes are then mapped back to the original filenames by storing metadata files inside the given metadataRoot.
* Filesystem implementation, that shortens filenames when they reach a certain
* threshold (inclusive). Shortening is done by SHA1-hashing those files, so a
* threshold below the length of the hashed files makes no sense. Hashes are
* then mapped back to the original filenames by storing metadata files inside
* the given metadataRoot.
*/
public class ShorteningFileSystem extends ShorteningFolder implements FileSystem {
@@ -24,4 +26,9 @@ public class ShorteningFileSystem extends ShorteningFolder implements FileSystem
// no-op.
}
@Override
public String toString() {
return "/";
}
}

View File

@@ -11,7 +11,7 @@ import org.cryptomator.filesystem.Folder;
import org.cryptomator.filesystem.FolderCreateMode;
import org.cryptomator.filesystem.Node;
class ShorteningFolder extends ShorteningNode<Folder>implements Folder {
class ShorteningFolder extends ShorteningNode<Folder> implements Folder {
private final Folder metadataRoot;
private final FilenameShortener shortener;
@@ -35,7 +35,10 @@ class ShorteningFolder extends ShorteningNode<Folder>implements Folder {
@Override
public File file(String name) {
final File original = delegate.file(shortener.deflate(name));
if (metadataRoot.equals(original)) { // comparing apples and oranges, but we don't know if the underlying fs distinguishes files and folders...
if (metadataRoot.equals(original)) { // comparing apples and oranges,
// but we don't know if the
// underlying fs distinguishes
// files and folders...
throw new UncheckedIOException("'" + name + "' is a reserved name.", new FileAlreadyExistsException(name));
}
return new ShorteningFile(this, original, name, shortener);
@@ -109,7 +112,7 @@ class ShorteningFolder extends ShorteningNode<Folder>implements Folder {
@Override
public String toString() {
return name() + "/";
return parent + name() + "/";
}
}

View File

@@ -9,7 +9,7 @@ import org.cryptomator.filesystem.Node;
class ShorteningNode<E extends Node> implements Node {
protected final E delegate;
private final ShorteningFolder parent;
protected final ShorteningFolder parent;
private final String longName;
private final String shortName;

View File

@@ -70,14 +70,14 @@ public class ShorteningFileSystemTest {
final FileSystem fs = new ShorteningFileSystem(underlyingFs, metadataRoot, 10);
final File shortNamedFolder = fs.file("test");
try (WritableFile file = shortNamedFolder.openWritable(1, TimeUnit.MILLISECONDS)) {
try (WritableFile file = shortNamedFolder.openWritable()) {
file.write(ByteBuffer.wrap("hello world".getBytes()));
}
Assert.assertFalse(metadataRoot.children().findAny().isPresent());
final File longNamedFolder = fs.file("morethantenchars");
try (WritableFile src = shortNamedFolder.openWritable(1, TimeUnit.MILLISECONDS); //
WritableFile dst = longNamedFolder.openWritable(1, TimeUnit.MILLISECONDS)) {
try (WritableFile src = shortNamedFolder.openWritable(); //
WritableFile dst = longNamedFolder.openWritable()) {
src.moveTo(dst);
}
Assert.assertTrue(metadataRoot.children().findAny().isPresent());
@@ -104,13 +104,13 @@ public class ShorteningFileSystemTest {
// write:
final FileSystem fs1 = new ShorteningFileSystem(underlyingFs, metadataRoot, 10);
fs1.folder("morethantenchars").create(FolderCreateMode.INCLUDING_PARENTS);
try (WritableFile file = fs1.folder("morethantenchars").file("morethanelevenchars.txt").openWritable(1, TimeUnit.MILLISECONDS)) {
try (WritableFile file = fs1.folder("morethantenchars").file("morethanelevenchars.txt").openWritable()) {
file.write(ByteBuffer.wrap("hello world".getBytes()));
}
// read
final FileSystem fs2 = new ShorteningFileSystem(underlyingFs, metadataRoot, 10);
try (ReadableFile file = fs2.folder("morethantenchars").file("morethanelevenchars.txt").openReadable(1, TimeUnit.MILLISECONDS)) {
try (ReadableFile file = fs2.folder("morethantenchars").file("morethanelevenchars.txt").openReadable()) {
ByteBuffer buf = ByteBuffer.allocate(11);
file.read(buf);
Assert.assertEquals("hello world", new String(buf.array()));
@@ -132,10 +132,10 @@ public class ShorteningFileSystemTest {
Assert.assertTrue(fs.folder("foo").folder("bar").exists());
// from underlying:
try (WritableFile file = underlyingFs.folder("foo").file("test1.txt").openWritable(1, TimeUnit.MILLISECONDS)) {
try (WritableFile file = underlyingFs.folder("foo").file("test1.txt").openWritable()) {
file.write(ByteBuffer.wrap("hello world".getBytes()));
}
try (ReadableFile file = fs.folder("foo").file("test1.txt").openReadable(1, TimeUnit.MILLISECONDS)) {
try (ReadableFile file = fs.folder("foo").file("test1.txt").openReadable()) {
ByteBuffer buf = ByteBuffer.allocate(11);
file.read(buf);
Assert.assertEquals("hello world", new String(buf.array()));
@@ -143,10 +143,10 @@ public class ShorteningFileSystemTest {
Assert.assertTrue(fs.folder("foo").file("test1.txt").lastModified().isAfter(testStart));
// to underlying:
try (WritableFile file = fs.folder("foo").file("test2.txt").openWritable(1, TimeUnit.MILLISECONDS)) {
try (WritableFile file = fs.folder("foo").file("test2.txt").openWritable()) {
file.write(ByteBuffer.wrap("hello world".getBytes()));
}
try (ReadableFile file = underlyingFs.folder("foo").file("test2.txt").openReadable(1, TimeUnit.MILLISECONDS)) {
try (ReadableFile file = underlyingFs.folder("foo").file("test2.txt").openReadable()) {
ByteBuffer buf = ByteBuffer.allocate(11);
file.read(buf);
Assert.assertEquals("hello world", new String(buf.array()));