1.35

TODO: make this configurable

.travis.yml

@@ -1,18 +1,21 @@
 sudo: false
 language: go
 go:
-- 1.10.x
-- 1.11.x
-- tip
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
+  # - tip
 
 before_install:
-- export PATH=/home/travis/gopath/bin:$PATH
+  - export PATH=/home/travis/gopath/bin:$PATH
 
 install:
-- go get ./weed/...
+  - export CGO_ENABLED="0"
+  - go env
+  - go get -u ./weed/...
 
 script:
-- go test ./weed/...
+  - go test ./weed/...
 
 before_deploy:
   - make release
@@ -22,23 +25,26 @@ deploy:
   api_key:
     secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
   file:
-  - build/linux_arm.tar.gz
-  - build/linux_arm64.tar.gz
-  - build/linux_386.tar.gz
-  - build/linux_amd64.tar.gz
-  - build/darwin_amd64.tar.gz
-  - build/windows_386.zip
-  - build/windows_amd64.zip
-  - build/freebsd_arm.tar.gz
-  - build/freebsd_amd64.tar.gz
-  - build/freebsd_386.tar.gz
-  - build/netbsd_arm.tar.gz
-  - build/netbsd_amd64.tar.gz
-  - build/netbsd_386.tar.gz
-  - build/openbsd_arm.tar.gz
-  - build/openbsd_amd64.tar.gz
-  - build/openbsd_386.tar.gz
+    - build/linux_arm.tar.gz
+    - build/linux_arm64.tar.gz
+    - build/linux_386.tar.gz
+    - build/linux_amd64.tar.gz
+    - build/linux_amd64_large_disk.tar.gz
+    - build/darwin_amd64.tar.gz
+    - build/darwin_amd64_large_disk.tar.gz
+    - build/windows_386.zip
+    - build/windows_amd64.zip
+    - build/windows_amd64_large_disk.zip
+    - build/freebsd_arm.tar.gz
+    - build/freebsd_amd64.tar.gz
+    - build/freebsd_386.tar.gz
+    - build/netbsd_arm.tar.gz
+    - build/netbsd_amd64.tar.gz
+    - build/netbsd_386.tar.gz
+    - build/openbsd_arm.tar.gz
+    - build/openbsd_amd64.tar.gz
+    - build/openbsd_386.tar.gz
   on:
     tags: true
     repo: chrislusf/seaweedfs
-    go: tip
+    go: 1.12.x

Makefile

@@ -12,6 +12,9 @@ build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -stat
 tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
 zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
 
+build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
 
 all: build
 
@@ -32,9 +35,21 @@ linux: deps
 	mkdir -p linux
 	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
 
-release: deps windows_build darwin_build linux_build bsd_build
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
 
 ##### LINUX BUILDS #####
+5_byte_linux_build:
+	$(call build_large,linux,amd64,)
+	$(call tar_large,linux,amd64)
+
+5_byte_darwin_build:
+	$(call build_large,darwin,amd64,)
+	$(call tar_large,darwin,amd64)
+
+5_byte_windows_build:
+	$(call build_large,windows,amd64,.exe)
+	$(call zip_large,windows,amd64,.exe)
+
 linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
 
 build/linux_386.tar.gz: $(sources)

README.md

@@ -10,18 +10,16 @@
 
 <h2 align="center">Supporting SeaweedFS</h2>
 
-SeaweedFS is an independent Apache-licensed open source project with its ongoing development made 
-possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
+SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
+possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md).
 If you'd like to grow SeaweedFS even stronger, please consider joining our
 <a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
 
-Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
-Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
-
 Your support will be really appreciated by me and other supporters!
 
 <h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
 
+<!--
 <h4 align="center">Platinum</h4>
 
 <p align="center">
@@ -45,6 +43,8 @@ Your support will be really appreciated by me and other supporters!
   </tbody>
 </table>
 
+-->
+
 ---
 
 
@@ -52,9 +52,29 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTc4MmVlYmFlNjBmZTgzZmJlYmI1MDE1YzkyNWYyZjkwZDFiM2RlMDdjNjVlNjdjYzc4NGFhZGIyYzEyMzJkYTA)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 
+Table of Contents
+=================
 
-## Introduction
+* [Introduction](#introduction)
+* [Features](#features)
+    * [Additional Features](#additional-features)
+    * [Filer Features](#filer-features)
+* [Example Usage](#example-usage)
+* [Architecture](#architecture)
+* [Compared to Other File Systems](#compared-to-other-file-systems)
+    * [Compared to HDFS](#compared-to-hdfs)
+    * [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
+    * [Compared to GlusterFS](#compared-to-glusterfs)
+    * [Compared to Ceph](#compared-to-ceph)
+* [Dev Plan](#dev-plan)
+* [Installation Guide](#installation-guide)
+* [Disk Related Topics](#disk-related-topics)
+* [Benchmark](#Benchmark)
+* [License](#license)
+
+## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
 
@@ -69,37 +89,53 @@ SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.
 
 SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
 
-## Additional Features
-* Can choose no replication or different replication levels, rack and data center aware
-* Automatic master servers failover - no single point of failure (SPOF)
-* Automatic Gzip compression depending on file mime type
-* Automatic compaction to reclaim disk space after deletion or update
-* Servers in the same cluster can have different disk spaces, file systems, OS etc.
-* Adding/Removing servers does **not** cause any data re-balancing
-* Optionally fix the orientation for jpeg pictures
-* Support Etag, Accept-Range, Last-Modified, etc.
-* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
+[Back to TOC](#table-of-contents)
 
-## Filer Features
+## Features ##
+
+[Back to TOC](#table-of-contents)
+
+## Additional Features ##
+* Can choose no replication or different replication levels, rack and data center aware.
+* Automatic master servers failover - no single point of failure (SPOF).
+* Automatic Gzip compression depending on file mime type.
+* Automatic compaction to reclaim disk space after deletion or update.
+* Servers in the same cluster can have different disk spaces, file systems, OS etc.
+* Adding/Removing servers does **not** cause any data re-balancing.
+* Optionally fix the orientation for jpeg pictures.
+* Support ETag, Accept-Range, Last-Modified, etc.
+* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
+* Support rebalancing the writable and readonly volumes.
+
+[Back to TOC](#table-of-contents)
+
+## Filer Features ##
 * [filer server][Filer] provide "normal" directories and files via http.
 * [mount filer][Mount] to read and write files directly as a local directory via FUSE.
 * [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
+* [Erasure Coding for warm storage][ErasureCoding] to reduce storage cost.
 * [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
 * [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+* [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
 [Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
 [AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
 [BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
 [Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
+[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
+[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
+
+[Back to TOC](#table-of-contents)
+
+## Example Usage ##
 
-## Example Usage
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.
 Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
 
 SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
 
-### Start Master Server
+### Start Master Server ###
 
 ```
 > ./weed master
@@ -125,7 +161,7 @@ Second, to store the file content, send a HTTP multi-part POST request to `url +
 
 ```
 > curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
-{"size": 43234}
+{"name":"myphoto.jpg","size":43234,"eTag":"1cc0118e"}
 ```
 
 To update, send another POST request with updated file content.
@@ -135,6 +171,7 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 ```
 > curl -X DELETE http://127.0.0.1:8080/3,01637037d6
 ```
+
 ### Save File Id ###
 
 Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
@@ -157,7 +194,7 @@ First look up the volume server's URLs by the file's volumeId:
 
 ```
 > curl http://localhost:9333/dir/lookup?volumeId=3
-{"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
+{"volumeId":"3","locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
 ```
 
 Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.
@@ -213,7 +250,7 @@ More details about replication can be found [on the wiki][Replication].
 
 You can also set the default replication strategy when starting the master server.
 
-### Allocate File Key on specific data center ###
+### Allocate File Key on Specific Data Center ###
 
 Volume servers can be started with a specific data center name:
 
@@ -239,6 +276,8 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
 [feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files
 [feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
 
+[Back to TOC](#table-of-contents)
+
 ## Architecture ##
 
 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
@@ -279,12 +318,16 @@ Each individual file size is limited to the volume size.
 
 All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
+[Back to TOC](#table-of-contents)
+
 ## Compared to Other File Systems ##
 
 Most other distributed file systems seem more complicated than necessary.
 
 SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
 
+[Back to TOC](#table-of-contents)
+
 ### Compared to HDFS ###
 
 HDFS uses the chunk approach for each file, and is ideal for storing large files.
@@ -293,6 +336,7 @@ SeaweedFS is ideal for serving relatively smaller files quickly and concurrently
 
 SeaweedFS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it.
 
+[Back to TOC](#table-of-contents)
 
 ### Compared to GlusterFS, Ceph ###
 
@@ -310,17 +354,21 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa
 | GlusterFS      | hashing          |                  | FUSE, NFS          |          |                           |
 | Ceph           | hashing + rules  |                  | FUSE               | Yes      |                           |
 
+[Back to TOC](#table-of-contents)
+
 ### Compared to GlusterFS ###
 
 GlusterFS stores files, both directories and content, in configurable volumes called "bricks".
 
 GlusterFS hashes the path and filename into ids, and assigned to virtual volumes, and then mapped to "bricks".
 
+[Back to TOC](#table-of-contents)
+
 ### Compared to Ceph ###
 
 Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
 
-SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. 
+SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
 
 Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
 
@@ -336,16 +384,26 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Cassa
 | Volume  | OSD | optimized for small files |
 | Filer  | Ceph FS | linearly scalable, Customizable, O(1) or O(logN) |
 
+[Back to TOC](#table-of-contents)
 
-## Dev plan ##
+## Dev Plan ##
 
 More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
 Other key features include: Erasure Encoding, JWT security.
 
 This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
+BTW, We suggest run the code style check script `util/gostd` before you push your branch to remote, it will make SeaweedFS easy to review, maintain and develop:
 
-## Installation guide for users who are not familiar with golang
+```
+$ ./util/gostd
+```
+
+[Back to TOC](#table-of-contents)
+
+## Installation Guide ##
+
+> Installation guide for users who are not familiar with golang
 
 Step 1: install go on your machine and setup the environment by following the instructions at:
 
@@ -366,23 +424,27 @@ go get github.com/chrislusf/seaweedfs/weed
 
 Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-Step 4: after you modify your code locally, you could start a local build by calling `go install` under 
+Step 4: after you modify your code locally, you could start a local build by calling `go install` under
 
 ```
 $GOPATH/src/github.com/chrislusf/seaweedfs/weed
 ```
 
-## Disk Related topics ##
+[Back to TOC](#table-of-contents)
+
+## Disk Related Topics ##
 
 ### Hard Drive Performance ###
 
 When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
 
-### Solid State Disk
+### Solid State Disk ###
 
 To modify or delete small files, SSD must delete a whole block at a time, and move content in existing blocks to a new block. SSD is fast when brand new, but will get fragmented over time and you have to garbage collect, compacting blocks. SeaweedFS is friendly to SSD since it is append-only. Deletion and compaction are done on volume level in the background, not slowing reading and not causing fragmentation.
 
-## Benchmark
+[Back to TOC](#table-of-contents)
+
+## Benchmark ##
 
 My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.6GHz.
 
@@ -435,8 +497,9 @@ Percentage of the requests served within a certain time (ms)
   100%     20.7 ms
 ```
 
+[Back to TOC](#table-of-contents)
 
-## License
+## License ##
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -450,7 +513,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
+[Back to TOC](#table-of-contents)
 
-## Stargazers over time
+## Stargazers over time ##
 
 [![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)

docker/Dockerfile

@@ -1,9 +1,20 @@
 FROM frolvlad/alpine-glibc
 
+# Supercronic install settings
+ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.8/supercronic-linux-amd64 \
+    SUPERCRONIC=supercronic-linux-amd64 \
+    SUPERCRONIC_SHA1SUM=be43e64c45acd6ec4fce5831e03759c89676a0ea
+
+# Install SeaweedFS and Supercronic ( for cron job mode )
 # Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
 RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
     wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o 'chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
     tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+    echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+    chmod +x "$SUPERCRONIC" && \
+    mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+    ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
     apk del build-dependencies && \
     rm -rf /tmp/*
 
@@ -15,11 +26,15 @@ EXPOSE 8080
 EXPOSE 18888
 # filer server http port
 EXPOSE 8888
-# master server shared gprc+http port
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
 
+RUN mkdir -p /data/filerdb
+
 VOLUME /data
 
 COPY filer.toml /etc/seaweedfs/filer.toml

docker/Dockerfile.go_build

@@ -9,11 +9,15 @@ EXPOSE 8080
 EXPOSE 18888
 # filer server http port
 EXPOSE 8888
-# master server shared gprc+http port
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
 EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
 
+RUN mkdir -p /data/filerdb
+
 VOLUME /data
 
 RUN mkdir -p /etc/seaweedfs

docker/dev-compose.yml

@@ -7,6 +7,7 @@ services:
       dockerfile: Dockerfile.go_build
     ports:
     - 9333:9333
+    - 19333:19333
     command: "master"
   volume:
     build:

docker/entrypoint.sh

@@ -29,11 +29,10 @@ case "$1" in
   	;;
 
   'filer')
-  	ARGS="-ip `hostname -i` "
+  	ARGS=""
   	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
 		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
 	fi
-	mkdir -p /data/filerdb
   	exec /usr/bin/weed $@ $ARGS
 	;;
 
@@ -45,6 +44,16 @@ case "$1" in
   	exec /usr/bin/weed $@ $ARGS
 	;;
 
+  'cronjob')
+	MASTER=${WEED_MASTER-localhost:9333}
+	FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
+	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
+	BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
+	echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab
+	echo "Running Crontab:"
+	cat /crontab
+	exec supercronic /crontab
+	;;
   *)
   	exec /usr/bin/weed $@
 	;;

docker/seaweedfs-compose.yml

@@ -5,6 +5,7 @@ services:
     image: chrislusf/seaweedfs # use a remote image
     ports:
     - 9333:9333
+    - 19333:19333
     command: "master"
   volume:
     image: chrislusf/seaweedfs # use a remote image
@@ -25,6 +26,16 @@ services:
     depends_on:
     - master
     - volume
+  cronjob:
+    image: chrislusf/seaweedfs # use a remote image
+    command: 'cronjob'
+    environment:
+      # Run re-replication every 2 minutes
+      CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
+      WEED_MASTER: master:9333 # Default: localhost:9333
+    depends_on:
+    - master
+    - volume
   s3:
     image: chrislusf/seaweedfs # use a remote image
     ports:

other/java/client/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.0.5</version>
+    <version>1.1.0</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -51,12 +51,26 @@ public class FilerClient {
         }
 
         return createEntry(
-            parent,
-            newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
+                parent,
+                newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
         );
 
     }
 
+    public boolean mv(String oldPath, String newPath) {
+
+        Path oldPathObject = Paths.get(oldPath);
+        String oldParent = oldPathObject.getParent().toString();
+        String oldName = oldPathObject.getFileName().toString();
+
+        Path newPathObject = Paths.get(newPath);
+        String newParent = newPathObject.getParent().toString();
+        String newName = newPathObject.getFileName().toString();
+
+        return atomicRenameEntry(oldParent, oldName, newParent, newName);
+
+    }
+
     public boolean rm(String path, boolean isRecursive) {
 
         Path pathObject = Paths.get(path);
@@ -64,10 +78,10 @@ public class FilerClient {
         String name = pathObject.getFileName().toString();
 
         return deleteEntry(
-            parent,
-            name,
-            true,
-            isRecursive);
+                parent,
+                name,
+                true,
+                isRecursive);
     }
 
     public boolean touch(String path, int mode) {
@@ -84,18 +98,18 @@ public class FilerClient {
         FilerProto.Entry entry = lookupEntry(parent, name);
         if (entry == null) {
             return createEntry(
-                parent,
-                newFileEntry(name, mode, uid, gid, userName, groupNames).build()
+                    parent,
+                    newFileEntry(name, mode, uid, gid, userName, groupNames).build()
             );
         }
         long now = System.currentTimeMillis() / 1000L;
         FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
-            .setMtime(now)
-            .setUid(uid)
-            .setGid(gid)
-            .setUserName(userName)
-            .clearGroupName()
-            .addAllGroupName(Arrays.asList(groupNames));
+                .setMtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames));
         return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
     }
 
@@ -105,17 +119,17 @@ public class FilerClient {
         long now = System.currentTimeMillis() / 1000L;
 
         return FilerProto.Entry.newBuilder()
-            .setName(name)
-            .setIsDirectory(true)
-            .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                .setMtime(now)
-                .setCrtime(now)
-                .setUid(uid)
-                .setGid(gid)
-                .setFileMode(mode | 1 << 31)
-                .setUserName(userName)
-                .clearGroupName()
-                .addAllGroupName(Arrays.asList(groupNames)));
+                .setName(name)
+                .setIsDirectory(true)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                        .setMtime(now)
+                        .setCrtime(now)
+                        .setUid(uid)
+                        .setGid(gid)
+                        .setFileMode(mode | 1 << 31)
+                        .setUserName(userName)
+                        .clearGroupName()
+                        .addAllGroupName(Arrays.asList(groupNames)));
     }
 
     public FilerProto.Entry.Builder newFileEntry(String name, int mode,
@@ -124,17 +138,17 @@ public class FilerClient {
         long now = System.currentTimeMillis() / 1000L;
 
         return FilerProto.Entry.newBuilder()
-            .setName(name)
-            .setIsDirectory(false)
-            .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                .setMtime(now)
-                .setCrtime(now)
-                .setUid(uid)
-                .setGid(gid)
-                .setFileMode(mode)
-                .setUserName(userName)
-                .clearGroupName()
-                .addAllGroupName(Arrays.asList(groupNames)));
+                .setName(name)
+                .setIsDirectory(false)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                        .setMtime(now)
+                        .setCrtime(now)
+                        .setUid(uid)
+                        .setGid(gid)
+                        .setFileMode(mode)
+                        .setUserName(userName)
+                        .clearGroupName()
+                        .addAllGroupName(Arrays.asList(groupNames)));
     }
 
     public List<FilerProto.Entry> listEntries(String path) {
@@ -159,21 +173,27 @@ public class FilerClient {
     }
 
     public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
-        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
-            .setDirectory(path)
-            .setPrefix(entryPrefix)
-            .setStartFromFileName(lastEntryName)
-            .setLimit(limit)
-            .build()).getEntriesList();
+        List<FilerProto.Entry> entries = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+                .setDirectory(path)
+                .setPrefix(entryPrefix)
+                .setStartFromFileName(lastEntryName)
+                .setLimit(limit)
+                .build()).getEntriesList();
+        List<FilerProto.Entry> fixedEntries = new ArrayList<>(entries.size());
+        for (FilerProto.Entry entry : entries) {
+            fixedEntries.add(fixEntryAfterReading(entry));
+        }
+        return fixedEntries;
     }
 
     public FilerProto.Entry lookupEntry(String directory, String entryName) {
         try {
-            return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
-                FilerProto.LookupDirectoryEntryRequest.newBuilder()
-                    .setDirectory(directory)
-                    .setName(entryName)
-                    .build()).getEntry();
+            FilerProto.Entry entry = filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+                    FilerProto.LookupDirectoryEntryRequest.newBuilder()
+                            .setDirectory(directory)
+                            .setName(entryName)
+                            .build()).getEntry();
+            return fixEntryAfterReading(entry);
         } catch (Exception e) {
             LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
             return null;
@@ -184,9 +204,9 @@ public class FilerClient {
     public boolean createEntry(String parent, FilerProto.Entry entry) {
         try {
             filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
-                .setDirectory(parent)
-                .setEntry(entry)
-                .build());
+                    .setDirectory(parent)
+                    .setEntry(entry)
+                    .build());
         } catch (Exception e) {
             LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
             return false;
@@ -197,9 +217,9 @@ public class FilerClient {
     public boolean updateEntry(String parent, FilerProto.Entry entry) {
         try {
             filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
-                .setDirectory(parent)
-                .setEntry(entry)
-                .build());
+                    .setDirectory(parent)
+                    .setEntry(entry)
+                    .build());
         } catch (Exception e) {
             LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
             return false;
@@ -210,11 +230,11 @@ public class FilerClient {
     public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive) {
         try {
             filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
-                .setDirectory(parent)
-                .setName(entryName)
-                .setIsDeleteData(isDeleteFileChunk)
-                .setIsRecursive(isRecursive)
-                .build());
+                    .setDirectory(parent)
+                    .setName(entryName)
+                    .setIsDeleteData(isDeleteFileChunk)
+                    .setIsRecursive(isRecursive)
+                    .build());
         } catch (Exception e) {
             LOG.warn("deleteEntry {}/{}: {}", parent, entryName, e);
             return false;
@@ -222,4 +242,39 @@ public class FilerClient {
         return true;
     }
 
+    public boolean atomicRenameEntry(String oldParent, String oldName, String newParent, String newName) {
+        try {
+            filerGrpcClient.getBlockingStub().atomicRenameEntry(FilerProto.AtomicRenameEntryRequest.newBuilder()
+                    .setOldDirectory(oldParent)
+                    .setOldName(oldName)
+                    .setNewDirectory(newParent)
+                    .setNewName(newName)
+                    .build());
+        } catch (Exception e) {
+            LOG.warn("atomicRenameEntry {}/{} => {}/{}: {}", oldParent, oldName, newParent, newName, e);
+            return false;
+        }
+        return true;
+    }
+
+    private FilerProto.Entry fixEntryAfterReading(FilerProto.Entry entry) {
+        if (entry.getChunksList().size() <= 0) {
+            return entry;
+        }
+        String fileId = entry.getChunks(0).getFileId();
+        if (fileId != null && fileId.length() != 0) {
+            return entry;
+        }
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        entryBuilder.clearChunks();
+        for (FilerProto.FileChunk chunk : entry.getChunksList()) {
+            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
+            FilerProto.FileId fid = chunk.getFid();
+            fileId = String.format("%d,%d%x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie());
+            chunkBuilder.setFileId(fileId);
+            entryBuilder.addChunks(chunkBuilder);
+        }
+        return entryBuilder.build();
+    }
+
 }

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -2,7 +2,14 @@ package seaweedfs.client;
 
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
+import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
 
+import javax.net.ssl.SSLException;
+import java.io.File;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
@@ -20,6 +27,16 @@ public class FilerGrpcClient {
         this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
     }
 
+    public FilerGrpcClient(String host, int grpcPort,
+                           String caFilePath,
+                           String clientCertFilePath,
+                           String clientPrivateKeyFilePath) throws SSLException {
+
+        this(NettyChannelBuilder.forAddress(host, grpcPort)
+                .negotiationType(NegotiationType.TLS)
+                .sslContext(buildSslContext(caFilePath,clientCertFilePath,clientPrivateKeyFilePath)));
+    }
+
     public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
         channel = channelBuilder.build();
         blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
@@ -42,4 +59,18 @@ public class FilerGrpcClient {
     public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() {
         return futureStub;
     }
+
+    private static SslContext buildSslContext(String trustCertCollectionFilePath,
+                                              String clientCertChainFilePath,
+                                              String clientPrivateKeyFilePath) throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (trustCertCollectionFilePath != null) {
+            builder.trustManager(new File(trustCertCollectionFilePath));
+        }
+        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
+        }
+        return builder.build();
+    }
+
 }

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -20,25 +20,26 @@ public class SeaweedWrite {
                                  final byte[] bytes,
                                  final long bytesOffset, final long bytesLength) throws IOException {
         FilerProto.AssignVolumeResponse response = filerGrpcClient.getBlockingStub().assignVolume(
-            FilerProto.AssignVolumeRequest.newBuilder()
-                .setCollection("")
-                .setReplication(replication)
-                .setDataCenter("")
-                .setReplication("")
-                .setTtlSec(0)
-                .build());
+                FilerProto.AssignVolumeRequest.newBuilder()
+                        .setCollection("")
+                        .setReplication(replication)
+                        .setDataCenter("")
+                        .setReplication("")
+                        .setTtlSec(0)
+                        .build());
         String fileId = response.getFileId();
         String url = response.getUrl();
+        String auth = response.getAuth();
         String targetUrl = String.format("http://%s/%s", url, fileId);
 
-        String etag = multipartUpload(targetUrl, bytes, bytesOffset, bytesLength);
+        String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength);
 
         entry.addChunks(FilerProto.FileChunk.newBuilder()
-            .setFileId(fileId)
-            .setOffset(offset)
-            .setSize(bytesLength)
-            .setMtime(System.currentTimeMillis() / 10000L)
-            .setETag(etag)
+                .setFileId(fileId)
+                .setOffset(offset)
+                .setSize(bytesLength)
+                .setMtime(System.currentTimeMillis() / 10000L)
+                .setETag(etag)
         );
 
     }
@@ -46,14 +47,15 @@ public class SeaweedWrite {
     public static void writeMeta(final FilerGrpcClient filerGrpcClient,
                                  final String parentDirectory, final FilerProto.Entry.Builder entry) {
         filerGrpcClient.getBlockingStub().createEntry(
-            FilerProto.CreateEntryRequest.newBuilder()
-                .setDirectory(parentDirectory)
-                .setEntry(entry)
-                .build()
+                FilerProto.CreateEntryRequest.newBuilder()
+                        .setDirectory(parentDirectory)
+                        .setEntry(entry)
+                        .build()
         );
     }
 
     private static String multipartUpload(String targetUrl,
+                                          String auth,
                                           final byte[] bytes,
                                           final long bytesOffset, final long bytesLength) throws IOException {
 
@@ -62,11 +64,14 @@ public class SeaweedWrite {
         InputStream inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
 
         HttpPost post = new HttpPost(targetUrl);
+        if (auth != null && auth.length() != 0) {
+            post.addHeader("Authorization", "BEARER " + auth);
+        }
 
         post.setEntity(MultipartEntityBuilder.create()
-            .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
-            .addBinaryBody("upload", inputStream)
-            .build());
+                .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
+                .addBinaryBody("upload", inputStream)
+                .build());
 
         try {
             HttpResponse response = client.execute(post);

other/java/client/src/main/proto/filer.proto

@@ -24,6 +24,9 @@ service SeaweedFiler {
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
+    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
+    }
+
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
 
@@ -69,19 +72,33 @@ message Entry {
     map<string, bytes> extended = 5;
 }
 
+message FullEntry {
+    string dir = 1;
+    Entry entry = 2;
+}
+
 message EventNotification {
     Entry old_entry = 1;
     Entry new_entry = 2;
     bool delete_chunks = 3;
+    string new_parent_path = 4;
 }
 
 message FileChunk {
-    string file_id = 1;
+    string file_id = 1; // to be deprecated
     int64 offset = 2;
     uint64 size = 3;
     int64 mtime = 4;
     string e_tag = 5;
-    string source_file_id = 6;
+    string source_file_id = 6; // to be deprecated
+    FileId fid = 7;
+    FileId source_fid = 8;
+}
+
+message FileId {
+    uint32 volume_id = 1;
+    uint64 file_key = 2;
+    fixed32 cookie = 3;
 }
 
 message FuseAttributes {
@@ -126,6 +143,16 @@ message DeleteEntryRequest {
 message DeleteEntryResponse {
 }
 
+message AtomicRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+}
+
+message AtomicRenameEntryResponse {
+}
+
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;
@@ -139,6 +166,7 @@ message AssignVolumeResponse {
     string url = 2;
     string public_url = 3;
     int32 count = 4;
+    string auth = 5;
 }
 
 message LookupVolumeRequest {

other/java/hdfs/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.0.5</seaweedfs.client.version>
+        <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -34,6 +34,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
+    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
+    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
     private static int BUFFER_SIZE = 16 * 1024 * 1024;
@@ -72,7 +75,17 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
+        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
+            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
+                    conf.get(FS_SEAWEED_GRPC_CA),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
+        } else {
+            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
+        }
+
     }
 
     @Override
@@ -206,8 +219,8 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
             UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
             return seaweedFileSystemStore.createDirectory(path, currentUser,
-                fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
-                FsPermission.getUMask(getConf()));
+                    fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
+                    FsPermission.getUMask(getConf()));
 
         }
 
@@ -238,7 +251,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      */
     @Override
     public void setOwner(Path path, final String owner, final String group)
-        throws IOException {
+            throws IOException {
         LOG.debug("setOwner path: {}", path);
         path = qualify(path);
 
@@ -271,54 +284,55 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     /**
      * Concat existing files together.
-     * @param trg the path to the target destination.
+     *
+     * @param trg   the path to the target destination.
      * @param psrcs the paths to the sources to use for the concatenation.
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default).
+     *                                       (default).
      */
     @Override
-    public void concat(final Path trg, final Path [] psrcs) throws IOException {
+    public void concat(final Path trg, final Path[] psrcs) throws IOException {
         throw new UnsupportedOperationException("Not implemented by the " +
-            getClass().getSimpleName() + " FileSystem implementation");
+                getClass().getSimpleName() + " FileSystem implementation");
     }
 
     /**
      * Truncate the file in the indicated path to the indicated size.
      * <ul>
-     *   <li>Fails if path is a directory.</li>
-     *   <li>Fails if path does not exist.</li>
-     *   <li>Fails if path is not closed.</li>
-     *   <li>Fails if new size is greater than current size.</li>
+     * <li>Fails if path is a directory.</li>
+     * <li>Fails if path does not exist.</li>
+     * <li>Fails if path is not closed.</li>
+     * <li>Fails if new size is greater than current size.</li>
      * </ul>
-     * @param f The path to the file to be truncated
-     * @param newLength The size the file is to be truncated to
      *
+     * @param f         The path to the file to be truncated
+     * @param newLength The size the file is to be truncated to
      * @return <code>true</code> if the file has been truncated to the desired
      * <code>newLength</code> and is immediately available to be reused for
      * write operations such as <code>append</code>, or
      * <code>false</code> if a background process of adjusting the length of
      * the last block has been started, and clients should wait for it to
      * complete before proceeding with further file updates.
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default).
+     *                                       (default).
      */
     @Override
     public boolean truncate(Path f, long newLength) throws IOException {
         throw new UnsupportedOperationException("Not implemented by the " +
-            getClass().getSimpleName() + " FileSystem implementation");
+                getClass().getSimpleName() + " FileSystem implementation");
     }
 
     @Override
     public void createSymlink(final Path target, final Path link,
                               final boolean createParent) throws AccessControlException,
-        FileAlreadyExistsException, FileNotFoundException,
-        ParentNotDirectoryException, UnsupportedFileSystemException,
-        IOException {
+            FileAlreadyExistsException, FileNotFoundException,
+            ParentNotDirectoryException, UnsupportedFileSystemException,
+            IOException {
         // Supporting filesystems should override this method
         throw new UnsupportedOperationException(
-            "Filesystem does not support symlinks!");
+                "Filesystem does not support symlinks!");
     }
 
     public boolean supportsSymlinks() {
@@ -327,48 +341,51 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     /**
      * Create a snapshot.
-     * @param path The directory where snapshots will be taken.
+     *
+     * @param path         The directory where snapshots will be taken.
      * @param snapshotName The name of the snapshot
      * @return the snapshot path.
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
      */
     @Override
     public Path createSnapshot(Path path, String snapshotName)
-        throws IOException {
+            throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support createSnapshot");
+                + " doesn't support createSnapshot");
     }
 
     /**
      * Rename a snapshot.
-     * @param path The directory path where the snapshot was taken
+     *
+     * @param path            The directory path where the snapshot was taken
      * @param snapshotOldName Old name of the snapshot
      * @param snapshotNewName New name of the snapshot
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void renameSnapshot(Path path, String snapshotOldName,
                                String snapshotNewName) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support renameSnapshot");
+                + " doesn't support renameSnapshot");
     }
 
     /**
      * Delete a snapshot of a directory.
-     * @param path  The directory that the to-be-deleted snapshot belongs to
+     *
+     * @param path         The directory that the to-be-deleted snapshot belongs to
      * @param snapshotName The name of the snapshot
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void deleteSnapshot(Path path, String snapshotName)
-        throws IOException {
+            throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support deleteSnapshot");
+                + " doesn't support deleteSnapshot");
     }
 
     /**
@@ -377,49 +394,49 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      * ACL entries that are not specified in this call are retained without
      * changes.  (Modifications are merged into the current ACL.)
      *
-     * @param path Path to modify
+     * @param path    Path to modify
      * @param aclSpec List&lt;AclEntry&gt; describing modifications
-     * @throws IOException if an ACL could not be modified
+     * @throws IOException                   if an ACL could not be modified
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
-        throws IOException {
+            throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support modifyAclEntries");
+                + " doesn't support modifyAclEntries");
     }
 
     /**
      * Removes ACL entries from files and directories.  Other ACL entries are
      * retained.
      *
-     * @param path Path to modify
+     * @param path    Path to modify
      * @param aclSpec List describing entries to remove
-     * @throws IOException if an ACL could not be modified
+     * @throws IOException                   if an ACL could not be modified
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void removeAclEntries(Path path, List<AclEntry> aclSpec)
-        throws IOException {
+            throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support removeAclEntries");
+                + " doesn't support removeAclEntries");
     }
 
     /**
      * Removes all default ACL entries from files and directories.
      *
      * @param path Path to modify
-     * @throws IOException if an ACL could not be modified
+     * @throws IOException                   if an ACL could not be modified
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void removeDefaultAcl(Path path)
-        throws IOException {
+            throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support removeDefaultAcl");
+                + " doesn't support removeDefaultAcl");
     }
 
     /**
@@ -428,32 +445,32 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      * bits.
      *
      * @param path Path to modify
-     * @throws IOException if an ACL could not be removed
+     * @throws IOException                   if an ACL could not be removed
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void removeAcl(Path path)
-        throws IOException {
+            throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support removeAcl");
+                + " doesn't support removeAcl");
     }
 
     /**
      * Fully replaces ACL of files and directories, discarding all existing
      * entries.
      *
-     * @param path Path to modify
+     * @param path    Path to modify
      * @param aclSpec List describing modifications, which must include entries
-     *   for user, group, and others for compatibility with permission bits.
-     * @throws IOException if an ACL could not be modified
+     *                for user, group, and others for compatibility with permission bits.
+     * @throws IOException                   if an ACL could not be modified
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support setAcl");
+                + " doesn't support setAcl");
     }
 
     /**
@@ -461,14 +478,14 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      *
      * @param path Path to get
      * @return AclStatus describing the ACL of the file or directory
-     * @throws IOException if an ACL could not be read
+     * @throws IOException                   if an ACL could not be read
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public AclStatus getAclStatus(Path path) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support getAclStatus");
+                + " doesn't support getAclStatus");
     }
 
     /**
@@ -478,19 +495,19 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      * <p>
      * Refer to the HDFS extended attributes user documentation for details.
      *
-     * @param path Path to modify
-     * @param name xattr name.
+     * @param path  Path to modify
+     * @param name  xattr name.
      * @param value xattr value.
-     * @param flag xattr set flag
-     * @throws IOException IO failure
+     * @param flag  xattr set flag
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void setXAttr(Path path, String name, byte[] value,
                          EnumSet<XAttrSetFlag> flag) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support setXAttr");
+                + " doesn't support setXAttr");
     }
 
     /**
@@ -503,14 +520,14 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      * @param path Path to get extended attribute
      * @param name xattr name.
      * @return byte[] xattr value.
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public byte[] getXAttr(Path path, String name) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support getXAttr");
+                + " doesn't support getXAttr");
     }
 
     /**
@@ -522,14 +539,14 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      *
      * @param path Path to get extended attributes
      * @return Map describing the XAttrs of the file or directory
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public Map<String, byte[]> getXAttrs(Path path) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support getXAttrs");
+                + " doesn't support getXAttrs");
     }
 
     /**
@@ -539,18 +556,18 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      * <p>
      * Refer to the HDFS extended attributes user documentation for details.
      *
-     * @param path Path to get extended attributes
+     * @param path  Path to get extended attributes
      * @param names XAttr names.
      * @return Map describing the XAttrs of the file or directory
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public Map<String, byte[]> getXAttrs(Path path, List<String> names)
-        throws IOException {
+            throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support getXAttrs");
+                + " doesn't support getXAttrs");
     }
 
     /**
@@ -562,14 +579,14 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      *
      * @param path Path to get extended attributes
      * @return List{@literal <String>} of the XAttr names of the file or directory
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public List<String> listXAttrs(Path path) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support listXAttrs");
+                + " doesn't support listXAttrs");
     }
 
     /**
@@ -581,14 +598,14 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      *
      * @param path Path to remove extended attribute
      * @param name xattr name
-     * @throws IOException IO failure
+     * @throws IOException                   IO failure
      * @throws UnsupportedOperationException if the operation is unsupported
-     *         (default outcome).
+     *                                       (default outcome).
      */
     @Override
     public void removeXAttr(Path path, String name) throws IOException {
         throw new UnsupportedOperationException(getClass().getSimpleName()
-            + " doesn't support removeXAttr");
+                + " doesn't support removeXAttr");
     }
 
 }

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -12,6 +12,7 @@ import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
 import seaweedfs.client.SeaweedRead;
 
+import javax.net.ssl.SSLException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -33,6 +34,13 @@ public class SeaweedFileSystemStore {
         filerClient = new FilerClient(filerGrpcClient);
     }
 
+    public SeaweedFileSystemStore(String host, int port,
+                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
+        int grpcPort = 10000 + port;
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
+        filerClient = new FilerClient(filerGrpcClient);
+    }
+
     public static String getParentDirectory(Path path) {
         return path.isRoot() ? "/" : path.getParent().toUri().getPath();
     }
@@ -143,35 +151,7 @@ public class SeaweedFileSystemStore {
             LOG.warn("rename non-existing source: {}", source);
             return;
         }
-        LOG.warn("rename moveEntry source: {}", source);
-        moveEntry(source.getParent(), entry, destination);
-    }
-
-    private boolean moveEntry(Path oldParent, FilerProto.Entry entry, Path destination) {
-
-        LOG.debug("moveEntry: {}/{}  => {}", oldParent, entry.getName(), destination);
-
-        FilerProto.Entry.Builder newEntry = entry.toBuilder().setName(destination.getName());
-        boolean isDirectoryCreated = filerClient.createEntry(getParentDirectory(destination), newEntry.build());
-
-        if (!isDirectoryCreated) {
-            return false;
-        }
-
-        if (entry.getIsDirectory()) {
-            Path entryPath = new Path(oldParent, entry.getName());
-            List<FilerProto.Entry> entries = filerClient.listEntries(entryPath.toUri().getPath());
-            for (FilerProto.Entry ent : entries) {
-                boolean isSucess = moveEntry(entryPath, ent, new Path(destination, ent.getName()));
-                if (!isSucess) {
-                    return false;
-                }
-            }
-        }
-
-        return filerClient.deleteEntry(
-            oldParent.toUri().getPath(), entry.getName(), false, false);
-
+        filerClient.mv(source.toUri().getPath(), destination.toUri().getPath());
     }
 
     public OutputStream createFile(final Path path,

unmaintained/change_superblock/change_superblock.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 var (
@@ -73,7 +74,7 @@ func main() {
 	}
 
 	if *targetTTL != "" {
-		ttl, err := storage.ReadTTL(*targetTTL)
+		ttl, err := needle.ReadTTL(*targetTTL)
 
 		if err != nil {
 			glog.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)

unmaintained/compact_leveldb/compact_leveldb.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	dir = flag.String("dir", ".", "data directory to store leveldb files")
+)
+
+func main() {
+
+	flag.Parse()
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+		OpenFilesCacheCapacity:        -1,
+	}
+
+	db, err := leveldb.OpenFile(*dir, opts)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+	if err := db.CompactRange(util.Range{}); err != nil {
+		log.Fatal(err)
+	}
+}

unmaintained/fix_dat/fix_dat.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -61,7 +62,7 @@ func main() {
 	}
 	newDatFile.Write(superBlock.Bytes())
 
-	iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) {
+	iterateEntries(datFile, indexFile, func(n *needle.Needle, offset int64) {
 		fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
 		_, s, _, e := n.Append(newDatFile, superBlock.Version())
 		fmt.Printf("size %d error %v\n", s, e)
@@ -69,7 +70,7 @@ func main() {
 
 }
 
-func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needle, offset int64)) {
+func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) {
 	// start to read index file
 	var readerOffset int64
 	bytes := make([]byte, 16)
@@ -84,7 +85,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 	}
 	offset := int64(superBlock.BlockSize())
 	version := superBlock.Version()
-	n, rest, err := storage.ReadNeedleHeader(datFile, version, offset)
+	n, _, rest, err := needle.ReadNeedleHeader(datFile, version, offset)
 	if err != nil {
 		fmt.Printf("cannot read needle header: %v", err)
 		return
@@ -106,7 +107,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 
 		fmt.Printf("key: %d offsetFromIndex %d n.Size %d sizeFromIndex:%d\n", key, offsetFromIndex, n.Size, sizeFromIndex)
 
-		rest = storage.NeedleBodyLength(sizeFromIndex, version)
+		rest = needle.NeedleBodyLength(sizeFromIndex, version)
 
 		func() {
 			defer func() {
@@ -114,7 +115,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 					fmt.Println("Recovered in f", r)
 				}
 			}()
-			if err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleEntrySize), rest); err != nil {
+			if _, err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleHeaderSize), rest); err != nil {
 				fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err)
 			}
 		}()
@@ -124,9 +125,9 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 		}
 		visitNeedle(n, offset)
 
-		offset += types.NeedleEntrySize + rest
+		offset += types.NeedleHeaderSize + rest
 		//fmt.Printf("==> new entry offset %d\n", offset)
-		if n, rest, err = storage.ReadNeedleHeader(datFile, version, offset); err != nil {
+		if n, _, rest, err = needle.ReadNeedleHeader(datFile, version, offset); err != nil {
 			if err == io.EOF {
 				return
 			}

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -7,6 +7,9 @@ import (
 	"log"
 	"math/rand"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -19,8 +22,11 @@ var (
 func main() {
 	flag.Parse()
 
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	for i := 0; i < *repeat; i++ {
-		assignResult, err := operation.Assign(*master, &operation.VolumeAssignRequest{Count: 1})
+		assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{Count: 1})
 		if err != nil {
 			log.Fatalf("assign: %v", err)
 		}
@@ -31,12 +37,12 @@ func main() {
 
 		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
 
-		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, "")
+		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth)
 		if err != nil {
 			log.Fatalf("upload: %v", err)
 		}
 
-		util.Delete(targetUrl, "")
+		util.Delete(targetUrl, string(assignResult.Auth))
 
 		util.Get(fmt.Sprintf("http://%s/vol/vacuum", *master))
 

unmaintained/see_dat/see_dat.go

@@ -2,8 +2,12 @@ package main
 
 import (
 	"flag"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+
+	"time"
 )
 
 var (
@@ -13,7 +17,7 @@ var (
 )
 
 type VolumeFileScanner4SeeDat struct {
-	version storage.Version
+	version needle.Version
 }
 
 func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
@@ -22,18 +26,19 @@ func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.Supe
 
 }
 func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
-	return false
+	return true
 }
 
-func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *storage.Needle, offset int64) error {
-	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
+func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64) error {
+	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
+	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x appendedAt %v", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie, t)
 	return nil
 }
 
 func main() {
 	flag.Parse()
 
-	vid := storage.VolumeId(*volumeId)
+	vid := needle.VolumeId(*volumeId)
 
 	scanner := &VolumeFileScanner4SeeDat{}
 	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)

unmaintained/see_idx/see_idx.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/idx"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
@@ -35,7 +35,7 @@ func main() {
 	}
 	defer indexFile.Close()
 
-	storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
+	idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
 		fmt.Printf("key:%v offset:%v size:%v\n", key, offset, size)
 		return nil
 	})

unmaintained/see_meta/see_meta.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+)
+
+var (
+	metaFile = flag.String("meta", "", "meta file generated via fs.meta.save")
+)
+
+func main() {
+	flag.Parse()
+
+	dst, err := os.OpenFile(*metaFile, os.O_RDONLY, 0644)
+	if err != nil {
+		log.Fatalf("failed to open %s: %v", *metaFile, err)
+	}
+	defer dst.Close()
+
+	err = walkMetaFile(dst)
+	if err != nil {
+		log.Fatalf("failed to visit %s: %v", *metaFile, err)
+	}
+
+}
+
+func walkMetaFile(dst *os.File) error {
+
+	sizeBuf := make([]byte, 4)
+
+	for {
+		if n, err := dst.Read(sizeBuf); n != 4 {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+
+		size := util.BytesToUint32(sizeBuf)
+
+		data := make([]byte, int(size))
+
+		if n, err := dst.Read(data); n != len(data) {
+			return err
+		}
+
+		fullEntry := &filer_pb.FullEntry{}
+		if err := proto.Unmarshal(data, fullEntry); err != nil {
+			return err
+		}
+
+		fmt.Fprintf(os.Stdout, "file %s %v\n", filer2.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
+		for i, chunk := range fullEntry.Entry.Chunks {
+			fmt.Fprintf(os.Stdout, "  chunk %d %v\n", i+1, chunk.String())
+		}
+
+	}
+
+}

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	size        = flag.Int("size", 1024, "file size")
+	concurrency = flag.Int("c", 4, "concurrent number of uploads")
+	times       = flag.Int("n", 1024, "repeated number of times")
+	fileCount   = flag.Int("fileCount", 1, "number of files to write")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+
+	flag.Parse()
+
+	data := make([]byte, *size)
+	println("data len", len(data))
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		go func(x int) {
+			defer wg.Done()
+
+			client := &http.Client{Transport: &http.Transport{
+				MaxConnsPerHost:     1024,
+				MaxIdleConnsPerHost: 1024,
+			}}
+			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
+
+			for t := 0; t < *times; t++ {
+				for f := 0; f < *fileCount; f++ {
+					fn := r.Intn(*fileCount)
+					if size, err := uploadFileToFiler(client, data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					} else {
+						log.Fatalf("client %d upload %d times: %v", x, t, err)
+					}
+				}
+			}
+		}(x)
+	}
+
+	go func() {
+		ticker := time.NewTicker(1000 * time.Millisecond)
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(client *http.Client, data []byte, filename, destination string) (size int64, err error) {
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", filename)
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", filename, err)
+	}
+
+	part.Write(data)
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", filename, err)
+	}
+
+	uri := destination + filename
+
+	request, err := http.NewRequest("POST", uri, body)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	// request.Close = true  // can not use this, which do not reuse http connection, impacting filer->volume also.
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return int64(len(data)), nil
+}

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	dir         = flag.String("dir", ".", "upload files under this directory")
+	concurrency = flag.Int("c", 1, "concurrent number of uploads")
+	times       = flag.Int("n", 1, "repeated number of times")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+
+	flag.Parse()
+
+	var fileNames []string
+
+	files, err := ioutil.ReadDir(*dir)
+	if err != nil {
+		log.Fatalf("fail to read dir %v: %v", *dir, err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		fileNames = append(fileNames, filepath.Join(*dir, file.Name()))
+	}
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		client := &http.Client{}
+
+		go func() {
+			defer wg.Done()
+			rand.Shuffle(len(fileNames), func(i, j int) {
+				fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
+			})
+			for t := 0; t < *times; t++ {
+				for _, filename := range fileNames {
+					if size, err := uploadFileToFiler(client, filename, *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					}
+				}
+			}
+		}()
+	}
+
+	go func() {
+		ticker := time.NewTicker(500 * time.Millisecond)
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(client *http.Client, filename, destination string) (size int64, err error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", file.Name())
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", file.Name(), err)
+	}
+	_, err = io.Copy(part, file)
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	uri := destination + file.Name()
+
+	request, err := http.NewRequest("POST", uri, body)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return fi.Size(), nil
+}

unmaintained/volume_tailer/volume_tailer.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	util2 "github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+	"golang.org/x/tools/godoc/util"
+)
+
+var (
+	master         = flag.String("master", "localhost:9333", "master server host and port")
+	volumeId       = flag.Int("volumeId", -1, "a volume id")
+	rewindDuration = flag.Duration("rewind", -1, "rewind back in time. -1 means from the first entry. 0 means from now.")
+	timeoutSeconds = flag.Int("timeoutSeconds", 0, "disconnect if no activity after these seconds")
+	showTextFile   = flag.Bool("showTextFile", false, "display textual file content")
+)
+
+func main() {
+	flag.Parse()
+
+	util2.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	vid := needle.VolumeId(*volumeId)
+
+	var sinceTimeNs int64
+	if *rewindDuration == 0 {
+		sinceTimeNs = time.Now().UnixNano()
+	} else if *rewindDuration == -1 {
+		sinceTimeNs = 0
+	} else if *rewindDuration > 0 {
+		sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
+	}
+
+	err := operation.TailVolume(*master, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
+		if n.Size == 0 {
+			println("-", n.String())
+			return nil
+		} else {
+			println("+", n.String())
+		}
+
+		if *showTextFile {
+
+			data := n.Data
+			if n.IsGzipped() {
+				if data, err = util2.UnGzipData(data); err != nil {
+					return err
+				}
+			}
+			if util.IsText(data) {
+				println(string(data))
+			}
+
+			println("-", n.String(), "compressed", n.IsGzipped(), "original size", len(data))
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("Error VolumeTailSender volume %d: %v", vid, err)
+	}
+
+}

util/gostd

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+############################  GLOBAL VARIABLES
+regex=' '
+branch="master"
+max_length=150
+
+REGEX_SUFFIX_GO=".+\.go$"
+
+############################  FUNCTIONS
+msg() {
+    printf '%b' "$1" >&2
+}
+
+die() {
+    msg "\33[31m[✘]\33[0m ${1}${2}"
+    exit 1
+}
+
+succ() {
+    msg "\33[34m[√]\33[0m ${1}${2}"
+}
+
+gostd() {
+    local branch=$1
+    local reg4exclude=$2
+    local max_length=$3
+
+    for file in `git diff $branch --name-only`
+    do
+        if ! [[ $file =~ $REGEX_SUFFIX_GO ]] || [[ $file =~ $reg4exclude ]]; then
+            continue
+        fi
+
+        error=`go fmt $file 2>&1`
+        if ! [ $? -eq 0 ]; then
+            die "go fmt $file:" "$error"
+        fi
+
+        succ "$file\n"
+
+        grep -n -E --color=always ".{$max_length}" $file | awk '{ printf ("%4s %s\n", "", $0) }'
+    done
+}
+
+get_options() {
+    while getopts "b:e:hl:" opts
+    do
+        case $opts in
+            b)
+                branch=$OPTARG
+                ;;
+            e)
+                regex=$OPTARG
+                ;;
+            h)
+                usage
+                exit 0
+                ;;
+            l)
+                max_length=$OPTARG
+                ;;
+            \?)
+                usage
+                exit 1
+                ;;
+        esac
+    done
+}
+
+usage () {
+    cat << _EOC_
+Usage:
+    gostd [options]
+
+Options:
+    -b <branch/commit>        Specify the git diff branch or commit.
+                              (default: master)
+    -e <regex>                Regex for excluding file or directory.
+    -h                        Print this usage.
+    -l <length>               Show files that exceed the limit line length.
+                              (default: 150)
+
+Examples:
+    gostd
+    gostd -b master -l 100
+    gostd -b 59d532a -e weed/pb -l 100
+_EOC_
+}
+
+main() {
+    get_options "$@"
+
+    gostd "$branch" "$regex" "$max_length"
+}
+
+############################  MAIN()
+main "$@"

weed/command/backup.go

@@ -3,6 +3,11 @@ package command
 import (
 	"fmt"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 )
@@ -30,26 +35,30 @@ var cmdBackup = &Command{
 	UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
 	Short:     "incrementally backup a volume to local folder",
 	Long: `Incrementally backup volume data.
-	
+
 	It is expected that you use this inside a script, to loop through
 	all possible volume ids that needs to be backup to local folder.
-	
+
 	The volume id does not need to exist locally or even remotely.
 	This will help to backup future new volumes.
-	
+
 	Usually backing up is just copying the .dat (and .idx) files.
 	But it's tricky to incrementally copy the differences.
-	
+
 	The complexity comes when there are multiple addition, deletion and compaction.
-	This tool will handle them correctly and efficiently, avoiding unnecessary data transporation.
+	This tool will handle them correctly and efficiently, avoiding unnecessary data transportation.
   `,
 }
 
 func runBackup(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	if *s.volumeId == -1 {
 		return false
 	}
-	vid := storage.VolumeId(*s.volumeId)
+	vid := needle.VolumeId(*s.volumeId)
 
 	// find volume location, replication, ttl info
 	lookup, err := operation.Lookup(*s.master, vid.String())
@@ -59,12 +68,12 @@ func runBackup(cmd *Command, args []string) bool {
 	}
 	volumeServer := lookup.Locations[0].Url
 
-	stats, err := operation.GetVolumeSyncStatus(volumeServer, uint32(vid))
+	stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
 	if err != nil {
 		fmt.Printf("Error get volume %d status: %v\n", vid, err)
 		return true
 	}
-	ttl, err := storage.ReadTTL(stats.Ttl)
+	ttl, err := needle.ReadTTL(stats.Ttl)
 	if err != nil {
 		fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
 		return true
@@ -81,7 +90,34 @@ func runBackup(cmd *Command, args []string) bool {
 		return true
 	}
 
-	if err := v.Synchronize(volumeServer); err != nil {
+	if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
+		if err = v.Compact(0, 0); err != nil {
+			fmt.Printf("Compact Volume before synchronizing %v\n", err)
+			return true
+		}
+		if err = v.CommitCompact(); err != nil {
+			fmt.Printf("Commit Compact before synchronizing %v\n", err)
+			return true
+		}
+		v.SuperBlock.CompactionRevision = uint16(stats.CompactRevision)
+		v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
+	}
+
+	datSize, _, _ := v.FileStat()
+
+	if datSize > stats.TailOffset {
+		// remove the old data
+		v.Destroy()
+		// recreate an empty volume
+		v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0)
+		if err != nil {
+			fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err)
+			return true
+		}
+	}
+	defer v.Close()
+
+	if err := v.IncrementalBackup(volumeServer, grpcDialOption); err != nil {
 		fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
 		return true
 	}

weed/command/benchmark.go

@@ -15,6 +15,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/security"
@@ -33,15 +36,17 @@ type BenchmarkOptions struct {
 	read             *bool
 	sequentialRead   *bool
 	collection       *string
+	replication      *string
 	cpuprofile       *string
 	maxCpu           *int
-	secretKey        *string
+	grpcDialOption   grpc.DialOption
+	masterClient     *wdclient.MasterClient
 }
 
 var (
-	b            BenchmarkOptions
-	sharedBytes  []byte
-	masterClient *wdclient.MasterClient
+	b           BenchmarkOptions
+	sharedBytes []byte
+	isSecure    bool
 )
 
 func init() {
@@ -57,9 +62,9 @@ func init() {
 	b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
 	b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
 	b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
+	b.replication = cmdBenchmark.Flag.String("replication", "000", "replication type")
 	b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
 	b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	b.secretKey = cmdBenchmark.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	sharedBytes = make([]byte, 1024)
 }
 
@@ -102,6 +107,10 @@ var (
 )
 
 func runBenchmark(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+	b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
 	if *b.maxCpu < 1 {
 		*b.maxCpu = runtime.NumCPU()
@@ -116,9 +125,9 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	masterClient = wdclient.NewMasterClient(context.Background(), "benchmark", strings.Split(*b.masters, ","))
-	go masterClient.KeepConnectedToMaster()
-	masterClient.WaitUntilConnected()
+	b.masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
+	go b.masterClient.KeepConnectedToMaster()
+	b.masterClient.WaitUntilConnected()
 
 	if *b.write {
 		benchWrite()
@@ -188,7 +197,6 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 	defer wait.Done()
 	delayedDeleteChan := make(chan *delayedFile, 100)
 	var waitForDeletions sync.WaitGroup
-	secret := security.Secret(*b.secretKey)
 
 	for i := 0; i < 7; i++ {
 		waitForDeletions.Add(1)
@@ -198,8 +206,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 				if df.enterTime.After(time.Now()) {
 					time.Sleep(df.enterTime.Sub(time.Now()))
 				}
-				if e := util.Delete("http://"+df.fp.Server+"/"+df.fp.Fid,
-					security.GenJwt(secret, df.fp.Fid)); e == nil {
+				var jwtAuthorization security.EncodedJwt
+				if isSecure {
+					jwtAuthorization = operation.LookupJwt(b.masterClient.GetMaster(), df.fp.Fid)
+				}
+				if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
 					s.completed++
 				} else {
 					s.failed++
@@ -219,12 +230,16 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 			MimeType: "image/bench", // prevent gzip benchmark content
 		}
 		ar := &operation.VolumeAssignRequest{
-			Count:      1,
-			Collection: *b.collection,
+			Count:       1,
+			Collection:  *b.collection,
+			Replication: *b.replication,
 		}
-		if assignResult, err := operation.Assign(masterClient.GetMaster(), ar); err == nil {
+		if assignResult, err := operation.Assign(b.masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
 			fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
-			if _, err := fp.Upload(0, masterClient.GetMaster(), secret); err == nil {
+			if !isSecure && assignResult.Auth != "" {
+				isSecure = true
+			}
+			if _, err := fp.Upload(0, b.masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
 				if random.Intn(100) < *b.deletePercentage {
 					s.total++
 					delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
@@ -264,7 +279,7 @@ func readFiles(fileIdLineChan chan string, s *stat) {
 			fmt.Printf("reading file %s\n", fid)
 		}
 		start := time.Now()
-		url, err := masterClient.LookupFileId(fid)
+		url, err := b.masterClient.LookupFileId(fid)
 		if err != nil {
 			s.failed++
 			println("!!!! ", fid, " location not found!!!!!")

weed/command/command.go

@@ -13,7 +13,6 @@ var Commands = []*Command{
 	cmdCompact,
 	cmdCopy,
 	cmdFix,
-	cmdFilerExport,
 	cmdFilerReplicate,
 	cmdServer,
 	cmdMaster,
@@ -27,6 +26,7 @@ var Commands = []*Command{
 	cmdVolume,
 	cmdExport,
 	cmdMount,
+	cmdWebDav,
 }
 
 type Command struct {

weed/command/compact.go

@@ -3,6 +3,7 @@ package command
 import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 func init() {
@@ -35,14 +36,14 @@ func runCompact(cmd *Command, args []string) bool {
 
 	preallocate := *compactVolumePreallocate * (1 << 20)
 
-	vid := storage.VolumeId(*compactVolumeId)
+	vid := needle.VolumeId(*compactVolumeId)
 	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
 		storage.NeedleMapInMemory, nil, nil, preallocate)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}
 	if *compactMethod == 0 {
-		if err = v.Compact(preallocate); err != nil {
+		if err = v.Compact(preallocate, 0); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {

weed/command/export.go

@@ -12,10 +12,12 @@ import (
 	"text/template"
 	"time"
 
+	"io"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
-	"io"
 )
 
 const (
@@ -66,10 +68,10 @@ var (
 	localLocation, _             = time.LoadLocation("Local")
 )
 
-func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Version, deleted bool) {
-	key := storage.NewFileIdFromNeedle(vid, n).String()
+func printNeedle(vid needle.VolumeId, n *needle.Needle, version needle.Version, deleted bool) {
+	key := needle.NewFileIdFromNeedle(vid, n).String()
 	size := n.DataSize
-	if version == storage.Version1 {
+	if version == needle.Version1 {
 		size = n.Size
 	}
 	fmt.Printf("%s\t%s\t%d\t%t\t%s\t%s\t%s\t%t\n",
@@ -85,10 +87,10 @@ func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Versio
 }
 
 type VolumeFileScanner4Export struct {
-	version   storage.Version
+	version   needle.Version
 	counter   int
 	needleMap *storage.NeedleMap
-	vid       storage.VolumeId
+	vid       needle.VolumeId
 }
 
 func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
@@ -100,14 +102,14 @@ func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
 	return true
 }
 
-func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset int64) error {
+func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset int64) error {
 	needleMap := scanner.needleMap
 	vid := scanner.vid
 
 	nv, ok := needleMap.Get(n.Id)
 	glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
 		n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
-	if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
+	if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && nv.Offset.ToAcutalOffset() == offset {
 		if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
 			glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
 				n.LastModified, newerThanUnix)
@@ -189,7 +191,7 @@ func runExport(cmd *Command, args []string) bool {
 	if *export.collection != "" {
 		fileName = *export.collection + "_" + fileName
 	}
-	vid := storage.VolumeId(*export.volumeId)
+	vid := needle.VolumeId(*export.volumeId)
 	indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
 	if err != nil {
 		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
@@ -225,8 +227,8 @@ type nameParams struct {
 	Ext  string
 }
 
-func writeFile(vid storage.VolumeId, n *storage.Needle) (err error) {
-	key := storage.NewFileIdFromNeedle(vid, n).String()
+func writeFile(vid needle.VolumeId, n *needle.Needle) (err error) {
+	key := needle.NewFileIdFromNeedle(vid, n).String()
 	fileNameTemplateBuffer.Reset()
 	if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
 		nameParams{

weed/command/filer.go

@@ -6,6 +6,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
@@ -21,17 +24,16 @@ type FilerOptions struct {
 	masters                 *string
 	ip                      *string
 	port                    *int
-	grpcPort                *int
 	publicPort              *int
 	collection              *string
 	defaultReplicaPlacement *string
 	redirectOnRead          *bool
 	disableDirListing       *bool
 	maxMB                   *int
-	secretKey               *string
 	dirListingLimit         *int
 	dataCenter              *string
 	enableNotification      *bool
+	disableHttp             *bool
 
 	// default leveldb directory, used in "weed server" mode
 	defaultLevelDbDirectory *string
@@ -43,15 +45,14 @@ func init() {
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
 	f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
-	f.grpcPort = cmdFiler.Flag.Int("port.grpc", 0, "filer grpc server listen port, default to http port + 10000")
-	f.publicPort = cmdFiler.Flag.Int("port.public", 0, "port opened to public")
+	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
 	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
 	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
-	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
+	f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
 }
 
 var cmdFiler = &Command{
@@ -70,13 +71,15 @@ var cmdFiler = &Command{
 
 	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
 
-	The example filer.toml configuration file can be generated by "weed scaffold filer"
+	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
 
 `,
 }
 
 func runFiler(cmd *Command, args []string) bool {
 
+	util.LoadConfiguration("security", false)
+
 	f.startFiler()
 
 	return true
@@ -97,16 +100,16 @@ func (fo *FilerOptions) startFiler() {
 	}
 
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
-		Masters:            strings.Split(*f.masters, ","),
+		Masters:            strings.Split(*fo.masters, ","),
 		Collection:         *fo.collection,
 		DefaultReplication: *fo.defaultReplicaPlacement,
 		RedirectOnRead:     *fo.redirectOnRead,
 		DisableDirListing:  *fo.disableDirListing,
 		MaxMB:              *fo.maxMB,
-		SecretKey:          *fo.secretKey,
 		DirListingLimit:    *fo.dirListingLimit,
 		DataCenter:         *fo.dataCenter,
 		DefaultLevelDbDir:  defaultLevelDbDirectory,
+		DisableHttp:        *fo.disableHttp,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -128,7 +131,7 @@ func (fo *FilerOptions) startFiler() {
 
 	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.VERSION, *fo.ip, *fo.port)
 	filerListener, e := util.NewListener(
-		":"+strconv.Itoa(*fo.port),
+		*fo.ip+":"+strconv.Itoa(*fo.port),
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
@@ -136,15 +139,12 @@ func (fo *FilerOptions) startFiler() {
 	}
 
 	// starting grpc server
-	grpcPort := *fo.grpcPort
-	if grpcPort == 0 {
-		grpcPort = *fo.port + 10000
-	}
+	grpcPort := *fo.port + 10000
 	grpcL, err := util.NewListener(":"+strconv.Itoa(grpcPort), 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	grpcS := util.NewGrpcServer()
+	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "filer"))
 	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
 	reflection.Register(grpcS)
 	go grpcS.Serve(grpcL)

weed/command/filer_copy.go

@@ -1,39 +1,45 @@
 package command
 
 import (
+	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"sync"
+	"time"
 
-	"context"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 )
 
 var (
-	copy CopyOptions
+	copy      CopyOptions
+	waitGroup sync.WaitGroup
 )
 
 type CopyOptions struct {
-	filerGrpcPort *int
-	master        *string
-	include       *string
-	replication   *string
-	collection    *string
-	ttl           *string
-	maxMB         *int
-	secretKey     *string
-
-	secret security.Secret
+	filerGrpcPort    *int
+	master           *string
+	include          *string
+	replication      *string
+	collection       *string
+	ttl              *string
+	maxMB            *int
+	grpcDialOption   grpc.DialOption
+	masterClient     *wdclient.MasterClient
+	concurrency      *int
+	compressionLevel *int
 }
 
 func init() {
@@ -44,9 +50,10 @@ func init() {
 	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
 	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
+	copy.maxMB = cmdCopy.Flag.Int("maxMB", 32, "split files larger than the limit")
 	copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
-	copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
+	copy.compressionLevel = cmdCopy.Flag.Int("compressionLevel", 9, "local file compression level 1 ~ 9")
 }
 
 var cmdCopy = &Command{
@@ -66,7 +73,9 @@ var cmdCopy = &Command{
 }
 
 func runCopy(cmd *Command, args []string) bool {
-	copy.secret = security.Secret(*copy.secretKey)
+
+	util.LoadConfiguration("security", false)
+
 	if len(args) <= 1 {
 		return false
 	}
@@ -101,62 +110,141 @@ func runCopy(cmd *Command, args []string) bool {
 	}
 
 	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
+	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 
-	for _, fileOrDir := range fileOrDirs {
-		if !doEachCopy(fileOrDir, filerUrl.Host, filerGrpcAddress, urlPath) {
-			return false
-		}
+	copy.masterClient = wdclient.NewMasterClient(context.Background(), copy.grpcDialOption, "client", strings.Split(*copy.master, ","))
+	go copy.masterClient.KeepConnectedToMaster()
+	copy.masterClient.WaitUntilConnected()
+
+	if *cmdCopy.IsDebug {
+		util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
 	}
+
+	fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrency)
+
+	ctx := context.Background()
+
+	go func() {
+		defer close(fileCopyTaskChan)
+		for _, fileOrDir := range fileOrDirs {
+			if err := genFileCopyTask(fileOrDir, urlPath, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "gen file list error: %v\n", err)
+				break
+			}
+		}
+	}()
+	for i := 0; i < *copy.concurrency; i++ {
+		waitGroup.Add(1)
+		go func() {
+			defer waitGroup.Done()
+			worker := FileCopyWorker{
+				options:          &copy,
+				filerHost:        filerUrl.Host,
+				filerGrpcAddress: filerGrpcAddress,
+			}
+			if err := worker.copyFiles(ctx, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
+				return
+			}
+		}()
+	}
+	waitGroup.Wait()
+
 	return true
 }
 
-func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path string) bool {
-	f, err := os.Open(fileOrDir)
-	if err != nil {
-		fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
-		return false
-	}
-	defer f.Close()
+func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan FileCopyTask) error {
 
-	fi, err := f.Stat()
+	fi, err := os.Stat(fileOrDir)
 	if err != nil {
-		fmt.Printf("Failed to get stat for file %s: %v\n", fileOrDir, err)
-		return false
+		fmt.Fprintf(os.Stderr, "Failed to get stat for file %s: %v\n", fileOrDir, err)
+		return nil
 	}
 
 	mode := fi.Mode()
 	if mode.IsDir() {
 		files, _ := ioutil.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
-			if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, path+fi.Name()+"/") {
-				return false
+			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
+				return err
 			}
 		}
-		return true
+		return nil
 	}
 
+	uid, gid := util.GetFileUidGid(fi)
+
+	fileCopyTaskChan <- FileCopyTask{
+		sourceLocation:     fileOrDir,
+		destinationUrlPath: destPath,
+		fileSize:           fi.Size(),
+		fileMode:           fi.Mode(),
+		uid:                uid,
+		gid:                gid,
+	}
+
+	return nil
+}
+
+type FileCopyWorker struct {
+	options          *CopyOptions
+	filerHost        string
+	filerGrpcAddress string
+}
+
+func (worker *FileCopyWorker) copyFiles(ctx context.Context, fileCopyTaskChan chan FileCopyTask) error {
+	for task := range fileCopyTaskChan {
+		if err := worker.doEachCopy(ctx, task); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type FileCopyTask struct {
+	sourceLocation     string
+	destinationUrlPath string
+	fileSize           int64
+	fileMode           os.FileMode
+	uid                uint32
+	gid                uint32
+}
+
+func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask) error {
+
+	f, err := os.Open(task.sourceLocation)
+	if err != nil {
+		fmt.Printf("Failed to open file %s: %v\n", task.sourceLocation, err)
+		if _, ok := err.(*os.PathError); ok {
+			fmt.Printf("skipping %s\n", task.sourceLocation)
+			return nil
+		}
+		return err
+	}
+	defer f.Close()
+
 	// this is a regular file
-	if *copy.include != "" {
-		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
-			return true
+	if *worker.options.include != "" {
+		if ok, _ := filepath.Match(*worker.options.include, filepath.Base(task.sourceLocation)); !ok {
+			return nil
 		}
 	}
 
 	// find the chunk count
-	chunkSize := int64(*copy.maxMB * 1024 * 1024)
+	chunkSize := int64(*worker.options.maxMB * 1024 * 1024)
 	chunkCount := 1
-	if chunkSize > 0 && fi.Size() > chunkSize {
-		chunkCount = int(fi.Size()/chunkSize) + 1
+	if chunkSize > 0 && task.fileSize > chunkSize {
+		chunkCount = int(task.fileSize/chunkSize) + 1
 	}
 
 	if chunkCount == 1 {
-		return uploadFileAsOne(filerAddress, filerGrpcAddress, path, f, fi)
+		return worker.uploadFileAsOne(ctx, task, f)
 	}
 
-	return uploadFileInChunks(filerAddress, filerGrpcAddress, path, f, fi, chunkCount, chunkSize)
+	return worker.uploadFileInChunks(ctx, task, f, chunkCount, chunkSize)
 }
 
-func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo) bool {
+func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopyTask, f *os.File) error {
 
 	// upload the file content
 	fileName := filepath.Base(f.Name())
@@ -164,29 +252,27 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 
 	var chunks []*filer_pb.FileChunk
 
-	if fi.Size() > 0 {
+	if task.fileSize > 0 {
 
 		// assign a volume
-		assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
-			Replication: *copy.replication,
-			Collection:  *copy.collection,
-			Ttl:         *copy.ttl,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
 		})
 		if err != nil {
-			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
+			fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
 		}
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
 
-		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, "")
+		uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth, *worker.options.compressionLevel)
 		if err != nil {
-			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-			return false
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 		}
 		if uploadResult.Error != "" {
-			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-			return false
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
 		}
 		fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
 
@@ -198,43 +284,42 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 			ETag:   uploadResult.ETag,
 		})
 
-		fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
+		fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
 	}
 
-	if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
-			Directory: urlFolder,
+			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
 				Name: fileName,
 				Attributes: &filer_pb.FuseAttributes{
 					Crtime:      time.Now().Unix(),
 					Mtime:       time.Now().Unix(),
-					Gid:         uint32(os.Getgid()),
-					Uid:         uint32(os.Getuid()),
-					FileSize:    uint64(fi.Size()),
-					FileMode:    uint32(fi.Mode()),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
 					Mime:        mimeType,
-					Replication: *copy.replication,
-					Collection:  *copy.collection,
-					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
 				},
 				Chunks: chunks,
 			},
 		}
 
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil
 	}); err != nil {
-		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
-		return false
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
 	}
 
-	return true
+	return nil
 }
 
-func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
+func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
 
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
@@ -244,14 +329,14 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 	for i := int64(0); i < int64(chunkCount); i++ {
 
 		// assign a volume
-		assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
-			Replication: *copy.replication,
-			Collection:  *copy.collection,
-			Ttl:         *copy.ttl,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
 		})
 		if err != nil {
-			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
+			fmt.Printf("Failed to assign from %s: %v\n", *worker.options.master, err)
 		}
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
@@ -259,14 +344,12 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 		uploadResult, err := operation.Upload(targetUrl,
 			fileName+"-"+strconv.FormatInt(i+1, 10),
 			io.LimitReader(f, chunkSize),
-			false, "application/octet-stream", nil, "")
+			false, "application/octet-stream", nil, assignResult.Auth)
 		if err != nil {
-			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-			return false
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 		}
 		if uploadResult.Error != "" {
-			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-			return false
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
 		}
 		chunks = append(chunks, &filer_pb.FileChunk{
 			FileId: assignResult.Fid,
@@ -278,39 +361,38 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 	}
 
-	if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
-			Directory: urlFolder,
+			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
 				Name: fileName,
 				Attributes: &filer_pb.FuseAttributes{
 					Crtime:      time.Now().Unix(),
 					Mtime:       time.Now().Unix(),
-					Gid:         uint32(os.Getgid()),
-					Uid:         uint32(os.Getuid()),
-					FileSize:    uint64(fi.Size()),
-					FileMode:    uint32(fi.Mode()),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
 					Mime:        mimeType,
-					Replication: *copy.replication,
-					Collection:  *copy.collection,
-					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
 				},
 				Chunks: chunks,
 			},
 		}
 
-		if _, err := client.CreateEntry(context.Background(), request); err != nil {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 		return nil
 	}); err != nil {
-		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
-		return false
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
 	}
 
-	fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
+	fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
 
-	return true
+	return nil
 }
 
 func detectMimeType(f *os.File) string {
@@ -329,15 +411,11 @@ func detectMimeType(f *os.File) string {
 	return mimeType
 }
 
-func withFilerClient(filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
+func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(filerAddress)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", filerAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(clientConn)
+		return fn(client)
+	}, filerAddress, grpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }

weed/command/filer_export.go

@@ -1,187 +0,0 @@
-package command
-
-import (
-	"github.com/chrislusf/seaweedfs/weed/filer2"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/notification"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
-)
-
-func init() {
-	cmdFilerExport.Run = runFilerExport // break init cycle
-}
-
-var cmdFilerExport = &Command{
-	UsageLine: "filer.export -sourceStore=mysql -targetStore=cassandra",
-	Short:     "export meta data in filer store",
-	Long: `Iterate the file tree and export all metadata out
-
-	Both source and target store:
-        * should be a store name already specified in filer.toml
-        * do not need to be enabled state
-
-	If target store is empty, only the directory tree will be listed.
-
-	If target store is "notification", the list of entries will be sent to notification.
-	This is usually used to bootstrap filer replication to a remote system.
-
-  `,
-}
-
-var (
-	// filerExportOutputFile  = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
-	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
-	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
-	dir                    = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
-	dirListLimit           = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
-	dryRun                 = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
-	verboseFilerExport     = cmdFilerExport.Flag.Bool("v", false, "verbose entry details")
-)
-
-type statistics struct {
-	directoryCount int
-	fileCount      int
-}
-
-func runFilerExport(cmd *Command, args []string) bool {
-
-	weed_server.LoadConfiguration("filer", true)
-	config := viper.GetViper()
-
-	var sourceStore, targetStore filer2.FilerStore
-
-	for _, store := range filer2.Stores {
-		if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
-			viperSub := config.Sub(store.GetName())
-			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize source store for %s: %+v",
-					store.GetName(), err)
-			} else {
-				sourceStore = store
-			}
-			break
-		}
-	}
-
-	for _, store := range filer2.Stores {
-		if store.GetName() == *filerExportTargetStore {
-			viperSub := config.Sub(store.GetName())
-			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize target store for %s: %+v",
-					store.GetName(), err)
-			} else {
-				targetStore = store
-			}
-			break
-		}
-	}
-
-	if sourceStore == nil {
-		glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
-		println("existing data sources are:")
-		for _, store := range filer2.Stores {
-			println("    " + store.GetName())
-		}
-		return false
-	}
-
-	if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
-		glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
-		println("existing data sources are:")
-		for _, store := range filer2.Stores {
-			println("    " + store.GetName())
-		}
-		return false
-	}
-
-	stat := statistics{}
-
-	var fn func(level int, entry *filer2.Entry) error
-
-	if *filerExportTargetStore == "notification" {
-		weed_server.LoadConfiguration("notification", false)
-		v := viper.GetViper()
-		notification.LoadConfiguration(v.Sub("notification"))
-
-		fn = func(level int, entry *filer2.Entry) error {
-			printout(level, entry)
-			if *dryRun {
-				return nil
-			}
-			return notification.Queue.SendMessage(
-				string(entry.FullPath),
-				&filer_pb.EventNotification{
-					NewEntry: entry.ToProtoEntry(),
-				},
-			)
-		}
-	} else if targetStore == nil {
-		fn = printout
-	} else {
-		fn = func(level int, entry *filer2.Entry) error {
-			printout(level, entry)
-			if *dryRun {
-				return nil
-			}
-			return targetStore.InsertEntry(entry)
-		}
-	}
-
-	doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)
-
-	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
-
-	return true
-}
-
-func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
-
-	limit := *dirListLimit
-	lastEntryName := ""
-	for {
-		entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
-		if err != nil {
-			break
-		}
-		for _, entry := range entries {
-			if fnErr := fn(level, entry); fnErr != nil {
-				glog.Errorf("failed to process entry: %s", entry.FullPath)
-			}
-			if entry.IsDirectory() {
-				stat.directoryCount++
-				doTraverse(stat, filerStore, entry.FullPath, level+1, fn)
-			} else {
-				stat.fileCount++
-			}
-		}
-		if len(entries) < limit {
-			break
-		}
-	}
-}
-
-func printout(level int, entry *filer2.Entry) error {
-	for i := 0; i < level; i++ {
-		if i == level-1 {
-			print("+-")
-		} else {
-			print("| ")
-		}
-	}
-	print(entry.FullPath.Name())
-	if *verboseFilerExport {
-		for _, chunk := range entry.Chunks {
-			print("[")
-			print(chunk.FileId)
-			print(",")
-			print(chunk.Offset)
-			print(",")
-			print(chunk.Size)
-			print(")")
-		}
-	}
-	println()
-	return nil
-}

weed/command/filer_replication.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"context"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -12,7 +13,7 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
 	"github.com/chrislusf/seaweedfs/weed/replication/sub"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/spf13/viper"
 )
 
@@ -28,15 +29,16 @@ var cmdFilerReplicate = &Command{
 	filer.replicate listens on filer notifications. If any file is updated, it will fetch the updated content,
 	and write to the other destination.
 
-	Run "weed scaffold -config replication" to generate a replication.toml file and customize the parameters.
+	Run "weed scaffold -config=replication" to generate a replication.toml file and customize the parameters.
 
   `,
 }
 
 func runFilerReplicate(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("replication", true)
-	weed_server.LoadConfiguration("notification", true)
+	util.LoadConfiguration("security", false)
+	util.LoadConfiguration("replication", true)
+	util.LoadConfiguration("notification", true)
 	config := viper.GetViper()
 
 	var notificationInput sub.NotificationInput
@@ -115,7 +117,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 		} else {
 			glog.V(1).Infof("modify: %s", key)
 		}
-		if err = replicator.Replicate(key, m); err != nil {
+		if err = replicator.Replicate(context.Background(), key, m); err != nil {
 			glog.Errorf("replicate %s: %+v", key, err)
 		} else {
 			glog.V(1).Infof("replicated %s", key)

weed/command/fix.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
@@ -29,7 +30,7 @@ var (
 )
 
 type VolumeFileScanner4Fix struct {
-	version storage.Version
+	version needle.Version
 	nm      *storage.NeedleMap
 }
 
@@ -42,14 +43,14 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
 	return false
 }
 
-func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
+func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64) error {
 	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
-	if n.Size > 0 {
-		pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
+	if n.Size > 0 && n.Size != types.TombstoneFileSize {
+		pe := scanner.nm.Put(n.Id, types.ToOffset(offset), n.Size)
 		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
 	} else {
 		glog.V(2).Infof("skipping deleted file ...")
-		return scanner.nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
+		return scanner.nm.Delete(n.Id, types.ToOffset(offset))
 	}
 	return nil
 }
@@ -74,7 +75,7 @@ func runFix(cmd *Command, args []string) bool {
 	nm := storage.NewBtreeNeedleMap(indexFile)
 	defer nm.Close()
 
-	vid := storage.VolumeId(*fixVolumeId)
+	vid := needle.VolumeId(*fixVolumeId)
 	scanner := &VolumeFileScanner4Fix{
 		nm: nm,
 	}

weed/command/master.go

@@ -6,14 +6,15 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 
+	"github.com/chrislusf/raft/protobuf"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
+	"github.com/spf13/viper"
 	"google.golang.org/grpc/reflection"
 )
 
@@ -24,8 +25,11 @@ func init() {
 var cmdMaster = &Command{
 	UsageLine: "master -port=9333",
 	Short:     "start a master server",
-	Long: `start a master server to provide volume=>location mapping service
-  and sequence number of file ids
+	Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
+
+	The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
+
+	The example security.toml configuration file can be generated by "weed scaffold -config=security"
 
   `,
 }
@@ -44,7 +48,7 @@ var (
 	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	garbageThreshold      = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	masterSecureKey       = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	disableHttp           = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
 	masterMemProfile      = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
 
@@ -52,6 +56,10 @@ var (
 )
 
 func runMaster(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+	util.LoadConfiguration("master", false)
+
 	if *mMaxCpu < 1 {
 		*mMaxCpu = runtime.NumCPU()
 	}
@@ -64,7 +72,7 @@ func runMaster(cmd *Command, args []string) bool {
 	if *masterWhiteListOption != "" {
 		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
 	}
-	if *volumeSizeLimitMB > 30*1000 {
+	if *volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
 		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
 	}
 
@@ -72,42 +80,49 @@ func runMaster(cmd *Command, args []string) bool {
 	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
 		*volumeSizeLimitMB, *volumePreallocate,
 		*mpulse, *defaultReplicaPlacement, *garbageThreshold,
-		masterWhiteList, *masterSecureKey,
+		masterWhiteList,
+		*disableHttp,
 	)
 
 	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
 
 	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
 
-	listener, e := util.NewListener(listeningAddress, 0)
+	masterListener, e := util.NewListener(listeningAddress, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
 
 	go func() {
-		time.Sleep(100 * time.Millisecond)
+		// start raftServer
 		myMasterAddress, peers := checkPeers(*masterIp, *mport, *masterPeers)
-		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
+		raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
+			peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
+		if raftServer == nil {
+			glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717", *metaFolder)
+		}
 		ms.SetRaftServer(raftServer)
+		r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
+
+		// starting grpc server
+		grpcPort := *mport + 10000
+		grpcL, err := util.NewListener(*masterBindIp+":"+strconv.Itoa(grpcPort), 0)
+		if err != nil {
+			glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+		}
+		// Create your protocol servers.
+		grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
+		master_pb.RegisterSeaweedServer(grpcS, ms)
+		protobuf.RegisterRaftServer(grpcS, raftServer)
+		reflection.Register(grpcS)
+
+		glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *masterBindIp, grpcPort)
+		grpcS.Serve(grpcL)
 	}()
 
-	// start grpc and http server
-	m := cmux.New(listener)
-
-	grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-	httpL := m.Match(cmux.Any())
-
-	// Create your protocol servers.
-	grpcS := util.NewGrpcServer()
-	master_pb.RegisterSeaweedServer(grpcS, ms)
-	reflection.Register(grpcS)
-
+	// start http server
 	httpS := &http.Server{Handler: r}
-
-	go grpcS.Serve(grpcL)
-	go httpS.Serve(httpL)
-
-	if err := m.Serve(); err != nil {
+	if err := httpS.Serve(masterListener); err != nil {
 		glog.Fatalf("master server failed to serve: %v", err)
 	}
 

weed/command/mount.go

@@ -8,7 +8,6 @@ import (
 
 type MountOptions struct {
 	filer              *string
-	filerGrpcPort      *int
 	filerMountRootPath *string
 	dir                *string
 	dirListingLimit    *int
@@ -17,6 +16,7 @@ type MountOptions struct {
 	ttlSec             *int
 	chunkSizeLimitMB   *int
 	dataCenter         *string
+	allowOthers        *bool
 }
 
 var (
@@ -28,7 +28,6 @@ var (
 func init() {
 	cmdMount.Run = runMount // break init cycle
 	mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location")
-	mountOptions.filerGrpcPort = cmdMount.Flag.Int("filer.grpc.port", 0, "filer grpc server listen port, default to http port + 10000")
 	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
 	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
 	mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 100000, "limit directory listing size")
@@ -37,6 +36,7 @@ func init() {
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
+	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
 }
@@ -59,7 +59,7 @@ var cmdMount = &Command{
   `,
 }
 
-func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress string, err error) {
+func parseFilerGrpcAddress(filer string) (filerGrpcAddress string, err error) {
 	hostnameAndPort := strings.Split(filer, ":")
 	if len(hostnameAndPort) != 2 {
 		return "", fmt.Errorf("The filer should have hostname:port format: %v", hostnameAndPort)
@@ -71,9 +71,6 @@ func parseFilerGrpcAddress(filer string, optionalGrpcPort int) (filerGrpcAddress
 	}
 
 	filerGrpcPort := int(filerPort) + 10000
-	if optionalGrpcPort != 0 {
-		filerGrpcPort = optionalGrpcPort
-	}
 
 	return fmt.Sprintf("%s:%d", hostnameAndPort[0], filerGrpcPort), nil
 }

weed/command/mount_std.go

@@ -6,11 +6,16 @@ import (
 	"fmt"
 	"os"
 	"os/user"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/jacobsa/daemonize"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -19,40 +24,67 @@ import (
 )
 
 func runMount(cmd *Command, args []string) bool {
+
+	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
+
+	return RunMount(
+		*mountOptions.filer,
+		*mountOptions.filerMountRootPath,
+		*mountOptions.dir,
+		*mountOptions.collection,
+		*mountOptions.replication,
+		*mountOptions.dataCenter,
+		*mountOptions.chunkSizeLimitMB,
+		*mountOptions.allowOthers,
+		*mountOptions.ttlSec,
+		*mountOptions.dirListingLimit,
+	)
+}
+
+func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCenter string, chunkSizeLimitMB int,
+	allowOthers bool, ttlSec int, dirListingLimit int) bool {
+
+	util.LoadConfiguration("security", false)
+
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
-	if *mountOptions.dir == "" {
+	if dir == "" {
 		fmt.Printf("Please specify the mount directory via \"-dir\"")
 		return false
 	}
-	if *mountOptions.chunkSizeLimitMB <= 0 {
+	if chunkSizeLimitMB <= 0 {
 		fmt.Printf("Please specify a reasonable buffer size.")
 		return false
 	}
 
-	fuse.Unmount(*mountOptions.dir)
+	fuse.Unmount(dir)
+
+	uid, gid := uint32(0), uint32(0)
 
 	// detect mount folder mode
 	mountMode := os.ModeDir | 0755
-	if fileInfo, err := os.Stat(*mountOptions.dir); err == nil {
+	fileInfo, err := os.Stat(dir)
+	if err == nil {
 		mountMode = os.ModeDir | fileInfo.Mode()
+		uid, gid = util.GetFileUidGid(fileInfo)
+		fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, fileInfo.Mode())
 	}
 
-	// detect current user
-	uid, gid := uint32(0), uint32(0)
-	if u, err := user.Current(); err == nil {
-		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
-			uid = uint32(parsedId)
-		}
-		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
-			gid = uint32(parsedId)
+	if uid == 0 {
+		if u, err := user.Current(); err == nil {
+			if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+				uid = uint32(parsedId)
+			}
+			if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+				gid = uint32(parsedId)
+			}
+			fmt.Printf("current uid=%d gid=%d\n", uid, gid)
 		}
 	}
 
-	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
+	mountName := path.Base(dir)
 
-	c, err := fuse.Mount(
-		*mountOptions.dir,
-		fuse.VolumeName("SeaweedFS"),
+	options := []fuse.MountOption{
+		fuse.VolumeName(mountName),
 		fuse.FSName("SeaweedFS"),
 		fuse.Subtype("SeaweedFS"),
 		fuse.NoAppleDouble(),
@@ -61,56 +93,69 @@ func runMount(cmd *Command, args []string) bool {
 		fuse.AutoXattr(),
 		fuse.ExclCreate(),
 		fuse.DaemonTimeout("3600"),
-		fuse.AllowOther(),
 		fuse.AllowSUID(),
 		fuse.DefaultPermissions(),
-		fuse.MaxReadahead(1024*128),
+		fuse.MaxReadahead(1024 * 128),
 		fuse.AsyncRead(),
 		fuse.WritebackCache(),
-	)
+		fuse.AllowNonEmptyMount(),
+	}
+	if allowOthers {
+		options = append(options, fuse.AllowOther())
+	}
+
+	c, err := fuse.Mount(dir, options...)
 	if err != nil {
 		glog.Fatal(err)
+		daemonize.SignalOutcome(err)
 		return false
 	}
 
 	util.OnInterrupt(func() {
-		fuse.Unmount(*mountOptions.dir)
+		fuse.Unmount(dir)
 		c.Close()
 	})
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer, *mountOptions.filerGrpcPort)
+	filerGrpcAddress, err := parseFilerGrpcAddress(filer)
 	if err != nil {
 		glog.Fatal(err)
+		daemonize.SignalOutcome(err)
 		return false
 	}
 
-	mountRoot := *mountOptions.filerMountRootPath
+	mountRoot := filerMountRootPath
 	if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
 		mountRoot = mountRoot[0 : len(mountRoot)-1]
 	}
 
+	daemonize.SignalOutcome(nil)
+
 	err = fs.Serve(c, filesys.NewSeaweedFileSystem(&filesys.Option{
 		FilerGrpcAddress:   filerGrpcAddress,
+		GrpcDialOption:     security.LoadClientTLS(viper.Sub("grpc"), "client"),
 		FilerMountRootPath: mountRoot,
-		Collection:         *mountOptions.collection,
-		Replication:        *mountOptions.replication,
-		TtlSec:             int32(*mountOptions.ttlSec),
-		ChunkSizeLimit:     int64(*mountOptions.chunkSizeLimitMB) * 1024 * 1024,
-		DataCenter:         *mountOptions.dataCenter,
-		DirListingLimit:    *mountOptions.dirListingLimit,
+		Collection:         collection,
+		Replication:        replication,
+		TtlSec:             int32(ttlSec),
+		ChunkSizeLimit:     int64(chunkSizeLimitMB) * 1024 * 1024,
+		DataCenter:         dataCenter,
+		DirListingLimit:    dirListingLimit,
 		EntryCacheTtl:      3 * time.Second,
 		MountUid:           uid,
 		MountGid:           gid,
 		MountMode:          mountMode,
+		MountCtime:         fileInfo.ModTime(),
+		MountMtime:         time.Now(),
 	}))
 	if err != nil {
-		fuse.Unmount(*mountOptions.dir)
+		fuse.Unmount(dir)
 	}
 
 	// check if the mount process has an error to report
 	<-c.Ready
 	if err := c.MountError; err != nil {
 		glog.Fatal(err)
+		daemonize.SignalOutcome(err)
 	}
 
 	return true

weed/command/s3.go

@@ -4,7 +4,11 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"fmt"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -12,12 +16,11 @@ import (
 )
 
 var (
-	s3options S3Options
+	s3StandaloneOptions S3Options
 )
 
 type S3Options struct {
 	filer            *string
-	filerGrpcPort    *int
 	filerBucketsPath *string
 	port             *int
 	domainName       *string
@@ -27,13 +30,12 @@ type S3Options struct {
 
 func init() {
 	cmdS3.Run = runS3 // break init cycle
-	s3options.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
-	s3options.filerGrpcPort = cmdS3.Flag.Int("filer.grpcPort", 0, "filer server grpc port, default to filer http port plus 10000")
-	s3options.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
-	s3options.port = cmdS3.Flag.Int("port", 8333, "s3options server http listen port")
-	s3options.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
-	s3options.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
-	s3options.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
+	s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
+	s3StandaloneOptions.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
+	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
+	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
+	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
+	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 }
 
 var cmdS3 = &Command{
@@ -46,7 +48,15 @@ var cmdS3 = &Command{
 
 func runS3(cmd *Command, args []string) bool {
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
+	util.LoadConfiguration("security", false)
+
+	return s3StandaloneOptions.startS3Server()
+
+}
+
+func (s3opt *S3Options) startS3Server() bool {
+
+	filerGrpcAddress, err := parseFilerGrpcAddress(*s3opt.filer)
 	if err != nil {
 		glog.Fatal(err)
 		return false
@@ -55,10 +65,11 @@ func runS3(cmd *Command, args []string) bool {
 	router := mux.NewRouter().SkipClean(true)
 
 	_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
-		Filer:            *s3options.filer,
+		Filer:            *s3opt.filer,
 		FilerGrpcAddress: filerGrpcAddress,
-		DomainName:       *s3options.domainName,
-		BucketsPath:      *s3options.filerBucketsPath,
+		DomainName:       *s3opt.domainName,
+		BucketsPath:      *s3opt.filerBucketsPath,
+		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
@@ -66,22 +77,22 @@ func runS3(cmd *Command, args []string) bool {
 
 	httpS := &http.Server{Handler: router}
 
-	listenAddress := fmt.Sprintf(":%d", *s3options.port)
+	listenAddress := fmt.Sprintf(":%d", *s3opt.port)
 	s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}
 
-	if *s3options.tlsPrivateKey != "" {
-		if err = httpS.ServeTLS(s3ApiListener, *s3options.tlsCertificate, *s3options.tlsPrivateKey); err != nil {
+	if *s3opt.tlsPrivateKey != "" {
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.VERSION, *s3opt.port)
+		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
-		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.VERSION, *s3options.port)
 	} else {
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.VERSION, *s3opt.port)
 		if err = httpS.Serve(s3ApiListener); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
-		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.VERSION, *s3options.port)
 	}
 
 	return true

weed/command/scaffold.go

@@ -10,7 +10,7 @@ func init() {
 }
 
 var cmdScaffold = &Command{
-	UsageLine: "scaffold [filer]",
+	UsageLine: "scaffold -config=[filer|notification|replication|security|master]",
 	Short:     "generate basic configuration files",
 	Long: `Generate filer.toml with all possible configurations for you to customize.
 
@@ -19,7 +19,7 @@ var cmdScaffold = &Command{
 
 var (
 	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
-	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication] the configuration file to generate")
+	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master] the configuration file to generate")
 )
 
 func runScaffold(cmd *Command, args []string) bool {
@@ -32,6 +32,10 @@ func runScaffold(cmd *Command, args []string) bool {
 		content = NOTIFICATION_TOML_EXAMPLE
 	case "replication":
 		content = REPLICATION_TOML_EXAMPLE
+	case "security":
+		content = SECURITY_TOML_EXAMPLE
+	case "master":
+		content = MASTER_TOML_EXAMPLE
 	}
 	if content == "" {
 		println("need a valid -config option")
@@ -61,6 +65,12 @@ enabled = false
 
 [leveldb]
 # local on disk, mostly for simple single-machine setup, fairly scalable
+enabled = false
+dir = "."					# directory to store level db files
+
+[leveldb2]
+# local on disk, mostly for simple single-machine setup, fairly scalable
+# faster than previous leveldb, recommended.
 enabled = true
 dir = "."					# directory to store level db files
 
@@ -70,12 +80,13 @@ dir = "."					# directory to store level db files
 
 [mysql]
 # CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT        COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   name        VARCHAR(1000) COMMENT 'directory or file name',
-#   directory   VARCHAR(4096) COMMENT 'full path to parent directory',
-#   meta        BLOB,
+#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000)  COMMENT 'directory or file name',
+#   directory   TEXT           COMMENT 'full path to parent directory',
+#   meta        LONGBLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
+
 enabled = false
 hostname = "localhost"
 port = 3306
@@ -88,8 +99,8 @@ connection_max_open = 100
 [postgres]
 # CREATE TABLE IF NOT EXISTS filemeta (
 #   dirhash     BIGINT,
-#   name        VARCHAR(1000),
-#   directory   VARCHAR(4096),
+#   name        VARCHAR(65535),
+#   directory   VARCHAR(65535),
 #   meta        bytea,
 #   PRIMARY KEY (dirhash, name)
 # );
@@ -132,6 +143,7 @@ addresses = [
     "localhost:30005",
     "localhost:30006",
 ]
+password = ""
 
 `
 
@@ -178,6 +190,17 @@ google_application_credentials = "/path/to/x.json" # path to json credential fil
 project_id = ""                       # an existing project id
 topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists
 
+[notification.gocdk_pub_sub]
+# The Go Cloud Development Kit (https://gocloud.dev).
+# PubSub API (https://godoc.org/gocloud.dev/pubsub).
+# Supports AWS SNS/SQS, Azure Service Bus, Google PubSub, NATS and RabbitMQ.
+enabled = false
+# This URL will Dial the RabbitMQ server at the URL in the environment
+# variable RABBIT_SERVER_URL and open the exchange "myexchange".
+# The exchange must have already been created by some other means, like
+# the RabbitMQ management plugin.
+topic_url = "rabbit://myexchange"
+sub_url = "rabbit://myqueue"
 `
 
 	REPLICATION_TOML_EXAMPLE = `
@@ -239,5 +262,79 @@ b2_master_application_key  = ""
 bucket = "mybucket"            # an existing bucket
 directory = "/"                # destination directory
 
+`
+
+	SECURITY_TOML_EXAMPLE = `
+# Put this file to one of the location, with descending priority
+#    ./security.toml
+#    $HOME/.seaweedfs/security.toml
+#    /etc/seaweedfs/security.toml
+# this file is read by master, volume server, and filer
+
+# the jwt signing key is read by master and volume server.
+# a jwt defaults to expire after 10 seconds.
+[jwt.signing]
+key = ""
+expires_after_seconds = 10           # seconds
+
+# jwt for read is only supported with master+volume setup. Filer does not support this mode.
+[jwt.signing.read]
+key = ""
+expires_after_seconds = 10           # seconds
+
+# all grpc tls authentications are mutual
+# the values for the following ca, cert, and key are paths to the PERM files.
+# the host name is not checked, so the PERM files can be shared.
+[grpc]
+ca = ""
+
+[grpc.volume]
+cert = ""
+key  = ""
+
+[grpc.master]
+cert = ""
+key  = ""
+
+[grpc.filer]
+cert = ""
+key  = ""
+
+# use this for any place needs a grpc client
+# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
+[grpc.client]
+cert = ""
+key  = ""
+
+
+# volume server https options
+# Note: work in progress!
+#     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
+[https.client]
+enabled = true
+[https.volume]
+cert = ""
+key  = ""
+
+
+`
+
+	MASTER_TOML_EXAMPLE = `
+# Put this file to one of the location, with descending priority
+#    ./master.toml
+#    $HOME/.seaweedfs/master.toml
+#    /etc/seaweedfs/master.toml
+# this file is read by master
+
+[master.maintenance]
+# periodically run these scripts are the same as running them from 'weed shell'
+scripts = """
+  ec.encode -fullPercent=95 -quietFor=1h
+  ec.rebuild -force
+  ec.balance -force
+  volume.balance -force
+"""
+sleep_minutes = 17          # sleep minutes between each script execution
+
 `
 )

weed/command/server.go

@@ -1,6 +1,7 @@
 package command
 
 import (
+	"fmt"
 	"net/http"
 	"os"
 	"runtime"
@@ -10,12 +11,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/chrislusf/raft/protobuf"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
 	"google.golang.org/grpc/reflection"
 )
 
@@ -27,6 +31,7 @@ type ServerOptions struct {
 var (
 	serverOptions ServerOptions
 	filerOptions  FilerOptions
+	s3Options     S3Options
 )
 
 func init() {
@@ -35,17 +40,16 @@ func init() {
 
 var cmdServer = &Command{
 	UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
-	Short:     "start a server, including volume server, and automatically elect a master server",
+	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
 	Long: `start both a volume server to provide storage spaces
   and a master server to provide volume=>location mapping service and sequence number of file ids
 
   This is provided as a convenient way to start both volume server and master server.
-  The servers are exactly the same as starting them separately.
+  The servers acts exactly the same as starting them separately.
+  So other volume servers can connect to this master server also.
 
-  So other volume servers can use this embedded master server also.
-
-  Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
-  They run with meta data on disk, not shared. So each filer server is different.
+  Optionally, a filer server can be started.
+  Also optionally, a S3 gateway can be started.
 
   `,
 }
@@ -58,8 +62,8 @@ var (
 	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
 	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
+	serverDisableHttp             = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	serverPeers                   = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
-	serverSecureKey               = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	serverGarbageThreshold        = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
@@ -70,6 +74,7 @@ var (
 	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
 	pulseSeconds                  = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")
+	isStartingS3                  = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
 
 	serverWhiteList []string
 )
@@ -78,7 +83,6 @@ func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
-	filerOptions.grpcPort = cmdServer.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to http port + 10000")
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
 	filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
@@ -88,15 +92,25 @@ func init() {
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
-	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
+	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 
+	s3Options.filerBucketsPath = cmdServer.Flag.String("s3.filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
+	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
+	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
+	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
+	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
+
 }
 
 func runServer(cmd *Command, args []string) bool {
-	filerOptions.secretKey = serverSecureKey
+
+	util.LoadConfiguration("security", false)
+	util.LoadConfiguration("master", false)
+
 	if *serverOptions.cpuprofile != "" {
 		f, err := os.Create(*serverOptions.cpuprofile)
 		if err != nil {
@@ -110,8 +124,13 @@ func runServer(cmd *Command, args []string) bool {
 		*isStartingFiler = true
 	}
 
+	if *isStartingS3 {
+		*isStartingFiler = true
+	}
+
 	master := *serverIp + ":" + strconv.Itoa(*masterPort)
-	filerOptions.ip = serverIp
+	filerOptions.masters = &master
+	filerOptions.ip = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
 	serverOptions.v.masters = &master
@@ -122,6 +141,10 @@ func runServer(cmd *Command, args []string) bool {
 	serverOptions.v.pulseSeconds = pulseSeconds
 
 	filerOptions.dataCenter = serverDataCenter
+	filerOptions.disableHttp = serverDisableHttp
+
+	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
+	s3Options.filer = &filerAddress
 
 	if *filerOptions.defaultReplicaPlacement == "" {
 		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
@@ -134,7 +157,7 @@ func runServer(cmd *Command, args []string) bool {
 
 	folders := strings.Split(*volumeDataFolders, ",")
 
-	if *masterVolumeSizeLimitMB > 30*1000 {
+	if *masterVolumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
 		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
 	}
 
@@ -159,10 +182,17 @@ func runServer(cmd *Command, args []string) bool {
 		}()
 	}
 
-	var raftWaitForMaster sync.WaitGroup
+	if *isStartingS3 {
+		go func() {
+			time.Sleep(2 * time.Second)
+
+			s3Options.startS3Server()
+
+		}()
+	}
+
 	var volumeWait sync.WaitGroup
 
-	raftWaitForMaster.Add(1)
 	volumeWait.Add(1)
 
 	go func() {
@@ -170,7 +200,7 @@ func runServer(cmd *Command, args []string) bool {
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
 			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
 			*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
-			serverWhiteList, *serverSecureKey,
+			serverWhiteList, *serverDisableHttp,
 		)
 
 		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
@@ -180,33 +210,35 @@ func runServer(cmd *Command, args []string) bool {
 		}
 
 		go func() {
-			raftWaitForMaster.Wait()
-			time.Sleep(100 * time.Millisecond)
-			myAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
-			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
+			// start raftServer
+			myMasterAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
+			raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
+				peers, myMasterAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
 			ms.SetRaftServer(raftServer)
-			volumeWait.Done()
+			r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
+
+			// starting grpc server
+			grpcPort := *masterPort + 10000
+			grpcL, err := util.NewListener(*serverBindIp+":"+strconv.Itoa(grpcPort), 0)
+			if err != nil {
+				glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+			}
+			// Create your protocol servers.
+			glog.V(1).Infof("grpc config %+v", viper.Sub("grpc"))
+			grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
+			master_pb.RegisterSeaweedServer(grpcS, ms)
+			protobuf.RegisterRaftServer(grpcS, raftServer)
+			reflection.Register(grpcS)
+
+			glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort)
+			grpcS.Serve(grpcL)
 		}()
 
-		raftWaitForMaster.Done()
-
-		// start grpc and http server
-		m := cmux.New(masterListener)
-
-		grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-		httpL := m.Match(cmux.Any())
-
-		// Create your protocol servers.
-		grpcS := util.NewGrpcServer()
-		master_pb.RegisterSeaweedServer(grpcS, ms)
-		reflection.Register(grpcS)
+		volumeWait.Done()
 
+		// start http server
 		httpS := &http.Server{Handler: r}
-
-		go grpcS.Serve(grpcL)
-		go httpS.Serve(httpL)
-
-		if err := m.Serve(); err != nil {
+		if err := httpS.Serve(masterListener); err != nil {
 			glog.Fatalf("master server failed to serve: %v", err)
 		}
 

weed/command/shell.go

@@ -1,21 +1,25 @@
 package command
 
 import (
-	"bufio"
-	"fmt"
-	"os"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/shell"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+)
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
+var (
+	shellOptions shell.ShellOptions
 )
 
 func init() {
 	cmdShell.Run = runShell // break init cycle
+	shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers")
 }
 
 var cmdShell = &Command{
 	UsageLine: "shell",
-	Short:     "run interactive commands, now just echo",
-	Long: `run interactive commands.
+	Short:     "run interactive administrative commands",
+	Long: `run interactive administrative commands.
 
   `,
 }
@@ -23,39 +27,16 @@ var cmdShell = &Command{
 var ()
 
 func runShell(command *Command, args []string) bool {
-	r := bufio.NewReader(os.Stdin)
-	o := bufio.NewWriter(os.Stdout)
-	e := bufio.NewWriter(os.Stderr)
-	prompt := func() {
-		var err error
-		if _, err = o.WriteString("> "); err != nil {
-			glog.V(0).Infoln("error writing to stdout:", err)
-		}
-		if err = o.Flush(); err != nil {
-			glog.V(0).Infoln("error flushing stdout:", err)
-		}
-	}
-	readLine := func() string {
-		ret, err := r.ReadString('\n')
-		if err != nil {
-			fmt.Fprint(e, err)
-			os.Exit(1)
-		}
-		return ret
-	}
-	execCmd := func(cmd string) int {
-		if cmd != "" {
-			if _, err := o.WriteString(cmd); err != nil {
-				glog.V(0).Infoln("error writing to stdout:", err)
-			}
-		}
-		return 0
-	}
 
-	cmd := ""
-	for {
-		prompt()
-		cmd = readLine()
-		execCmd(cmd)
-	}
+	util.LoadConfiguration("security", false)
+	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	shellOptions.FilerHost = "localhost"
+	shellOptions.FilerPort = 8888
+	shellOptions.Directory = "/"
+
+	shell.RunShell(shellOptions)
+
+	return true
+
 }

weed/command/upload.go

@@ -6,8 +6,11 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
 )
 
 var (
@@ -23,7 +26,6 @@ type UploadOptions struct {
 	dataCenter  *string
 	ttl         *string
 	maxMB       *int
-	secretKey   *string
 }
 
 func init() {
@@ -36,8 +38,7 @@ func init() {
 	upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name")
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
-	upload.secretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
 }
 
 var cmdUpload = &Command{
@@ -53,14 +54,17 @@ var cmdUpload = &Command{
   All files under the folder and subfolders will be uploaded, each with its own file key.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separately.
   The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
 
   `,
 }
 
 func runUpload(cmd *Command, args []string) bool {
-	secret := security.Secret(*upload.secretKey)
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	if len(args) == 0 {
 		if *upload.dir == "" {
 			return false
@@ -77,9 +81,9 @@ func runUpload(cmd *Command, args []string) bool {
 					if e != nil {
 						return e
 					}
-					results, e := operation.SubmitFiles(*upload.master, parts,
+					results, e := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
 						*upload.replication, *upload.collection, *upload.dataCenter,
-						*upload.ttl, *upload.maxMB, secret)
+						*upload.ttl, *upload.maxMB)
 					bytes, _ := json.Marshal(results)
 					fmt.Println(string(bytes))
 					if e != nil {
@@ -96,9 +100,9 @@ func runUpload(cmd *Command, args []string) bool {
 		if e != nil {
 			fmt.Println(e.Error())
 		}
-		results, _ := operation.SubmitFiles(*upload.master, parts,
+		results, _ := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
 			*upload.replication, *upload.collection, *upload.dataCenter,
-			*upload.ttl, *upload.maxMB, secret)
+			*upload.ttl, *upload.maxMB)
 		bytes, _ := json.Marshal(results)
 		fmt.Println(string(bytes))
 	}

weed/command/volume.go

@@ -9,6 +9,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
@@ -41,6 +44,7 @@ type VolumeServerOptions struct {
 	readRedirect          *bool
 	cpuProfile            *string
 	memProfile            *string
+	compactionMBPerSecond *int
 }
 
 func init() {
@@ -56,11 +60,12 @@ func init() {
 	v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
-	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
+	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
 }
 
 var cmdVolume = &Command{
@@ -78,6 +83,9 @@ var (
 )
 
 func runVolume(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+
 	if *v.maxCpu < 1 {
 		*v.maxCpu = runtime.NumCPU()
 	}
@@ -137,10 +145,10 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	switch *v.indexType {
 	case "leveldb":
 		volumeNeedleMapKind = storage.NeedleMapLevelDb
-	case "boltdb":
-		volumeNeedleMapKind = storage.NeedleMapBoltDb
-	case "btree":
-		volumeNeedleMapKind = storage.NeedleMapBtree
+	case "leveldbMedium":
+		volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
+	case "leveldbLarge":
+		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
 	}
 
 	masters := *v.masters
@@ -152,6 +160,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		strings.Split(masters, ","), *v.pulseSeconds, *v.dataCenter, *v.rack,
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readRedirect,
+		*v.compactionMBPerSecond,
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
@@ -185,13 +194,20 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	grpcS := util.NewGrpcServer()
+	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "volume"))
 	volume_server_pb.RegisterVolumeServerServer(grpcS, volumeServer)
 	reflection.Register(grpcS)
 	go grpcS.Serve(grpcL)
 
-	if e := http.Serve(listener, volumeMux); e != nil {
-		glog.Fatalf("Volume server fail to serve: %v", e)
+	if viper.GetString("https.volume.key") != "" {
+		if e := http.ServeTLS(listener, volumeMux,
+			viper.GetString("https.volume.cert"), viper.GetString("https.volume.key")); e != nil {
+			glog.Fatalf("Volume server fail to serve: %v", e)
+		}
+	} else {
+		if e := http.Serve(listener, volumeMux); e != nil {
+			glog.Fatalf("Volume server fail to serve: %v", e)
+		}
 	}
 
 }

weed/command/webdav.go

@@ -0,0 +1,109 @@
+package command
+
+import (
+	"fmt"
+	"net/http"
+	"os/user"
+	"strconv"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+)
+
+var (
+	webDavStandaloneOptions WebDavOption
+)
+
+type WebDavOption struct {
+	filer          *string
+	port           *int
+	collection     *string
+	tlsPrivateKey  *string
+	tlsCertificate *string
+}
+
+func init() {
+	cmdWebDav.Run = runWebDav // break init cycle
+	webDavStandaloneOptions.filer = cmdWebDav.Flag.String("filer", "localhost:8888", "filer server address")
+	webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port")
+	webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files")
+	webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file")
+	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
+}
+
+var cmdWebDav = &Command{
+	UsageLine: "webdav -port=7333 -filer=<ip:port>",
+	Short:     "<unstable> start a webdav server that is backed by a filer",
+	Long: `start a webdav server that is backed by a filer.
+
+`,
+}
+
+func runWebDav(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+
+	glog.V(0).Infof("Starting Seaweed WebDav Server %s at https port %d", util.VERSION, *webDavStandaloneOptions.port)
+
+	return webDavStandaloneOptions.startWebDav()
+
+}
+
+func (wo *WebDavOption) startWebDav() bool {
+
+	filerGrpcAddress, err := parseFilerGrpcAddress(*wo.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	// detect current user
+	uid, gid := uint32(0), uint32(0)
+	if u, err := user.Current(); err == nil {
+		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+			uid = uint32(parsedId)
+		}
+		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+			gid = uint32(parsedId)
+		}
+	}
+
+	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
+		Filer:            *wo.filer,
+		FilerGrpcAddress: filerGrpcAddress,
+		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
+		Collection:       *wo.collection,
+		Uid:              uid,
+		Gid:              gid,
+	})
+	if webdavServer_err != nil {
+		glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)
+	}
+
+	httpS := &http.Server{Handler: ws.Handler}
+
+	listenAddress := fmt.Sprintf(":%d", *wo.port)
+	webDavListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	if err != nil {
+		glog.Fatalf("WebDav Server listener on %s error: %v", listenAddress, err)
+	}
+
+	if *wo.tlsPrivateKey != "" {
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at https port %d", util.VERSION, *wo.port)
+		if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
+			glog.Fatalf("WebDav Server Fail to serve: %v", err)
+		}
+	} else {
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at http port %d", util.VERSION, *wo.port)
+		if err = httpS.Serve(webDavListener); err != nil {
+			glog.Fatalf("WebDav Server Fail to serve: %v", err)
+		}
+	}
+
+	return true
+
+}

weed/command/weedfuse/README.md

@@ -0,0 +1,84 @@
+Mount the SeaweedFS via FUSE
+
+# Mount by fstab
+
+
+```
+$ # on linux
+$ sudo apt-get install fuse
+$ sudo echo 'user_allow_other' >> /etc/fuse.conf
+$ sudo mv weedfuse /sbin/mount.weedfuse
+
+$ # on Mac
+$ sudo mv weedfuse /sbin/mount_weedfuse
+
+```
+
+On both OS X and Linux, you can add one of the entries to your /etc/fstab file like the following:
+
+```
+# mount the whole SeaweedFS
+localhost:8888/    /home/some/mount/folder  weedfuse
+
+# mount the SeaweedFS sub folder
+localhost:8888/sub/dir    /home/some/mount/folder  weedfuse
+
+# mount the SeaweedFS sub folder with some options
+localhost:8888/sub/dir    /home/some/mount/folder  weedfuse  user
+
+```
+
+To verify it can work, try this command
+```
+$ sudo mount -av
+
+...
+
+/home/some/mount/folder           : successfully mounted
+
+```
+
+If you see `successfully mounted`, try to access the mounted folder and verify everything works.
+
+
+To debug, run these:
+```
+
+$ weedfuse -foreground localhost:8888/ /home/some/mount/folder
+
+```
+
+
+To unmount the folder:
+```
+
+$ sudo umount /home/some/mount/folder
+
+```
+
+<!-- not working yet!
+
+# Mount by autofs
+
+AutoFS can mount a folder if accessed.
+
+```
+# install autofs
+$ sudo apt-get install autofs
+```
+
+Here is an example on how to mount a folder for all users under `/home` directory.
+Assuming there exists corresponding folders under `/home` on both local and SeaweedFS.
+
+Edit `/etc/auto.master` and `/etc/auto.weedfuse` file with these content
+```
+$ cat /etc/auto.master
+/home    /etc/auto.weedfuse
+
+$ cat /etc/auto.weedfuse
+# map /home/<user> to localhost:8888/home/<user>
+* -fstype=weedfuse,rw,allow_other,foreground :localhost\:8888/home/&
+
+```
+
+-->

weed/command/weedfuse/weedfuse.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/command"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/jacobsa/daemonize"
+	"github.com/kardianos/osext"
+)
+
+var (
+	fuseCommand  = flag.NewFlagSet("weedfuse", flag.ContinueOnError)
+	options      = fuseCommand.String("o", "", "comma separated options rw,uid=xxx,gid=xxx")
+	isForeground = fuseCommand.Bool("foreground", false, "starts as a daemon")
+)
+
+func main() {
+
+	err := fuseCommand.Parse(os.Args[1:])
+	if err != nil {
+		glog.Fatal(err)
+	}
+	fmt.Printf("options: %v\n", *options)
+
+	// seems this value is always empty, need to parse it differently
+	optionsString := *options
+	prev := ""
+	for i, arg := range os.Args {
+		fmt.Printf("args[%d]: %v\n", i, arg)
+		if prev == "-o" {
+			optionsString = arg
+		}
+		prev = arg
+	}
+
+	device := fuseCommand.Arg(0)
+	mountPoint := fuseCommand.Arg(1)
+
+	fmt.Printf("source: %v\n", device)
+	fmt.Printf("target: %v\n", mountPoint)
+
+	nouser := true
+	for _, option := range strings.Split(optionsString, ",") {
+		fmt.Printf("option: %v\n", option)
+		switch option {
+		case "user":
+			nouser = false
+		}
+	}
+
+	maybeSetupPath()
+
+	if !*isForeground {
+		startAsDaemon()
+		return
+	}
+
+	parts := strings.SplitN(device, "/", 2)
+	filer, filerPath := parts[0], parts[1]
+
+	command.RunMount(
+		filer, "/"+filerPath, mountPoint, "", "000", "",
+		4, !nouser, 0, 1000000)
+
+}
+
+func maybeSetupPath() {
+	// sudo mount -av may not include PATH in some linux, e.g., Ubuntu
+	hasPathEnv := false
+	for _, e := range os.Environ() {
+		if strings.HasPrefix(e, "PATH=") {
+			hasPathEnv = true
+		}
+		fmt.Println(e)
+	}
+	if !hasPathEnv {
+		os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+	}
+}
+
+func startAsDaemon() {
+
+	// adapted from gcsfuse
+
+	// Find the executable.
+	var path string
+	path, err := osext.Executable()
+	if err != nil {
+		glog.Fatalf("osext.Executable: %v", err)
+	}
+
+	// Set up arguments. Be sure to use foreground mode.
+	args := append([]string{"-foreground"}, os.Args[1:]...)
+
+	// Pass along PATH so that the daemon can find fusermount on Linux.
+	env := []string{
+		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
+	}
+
+	err = daemonize.Run(path, args, env, os.Stdout)
+	if err != nil {
+		glog.Fatalf("daemonize.Run: %v", err)
+	}
+
+}

weed/filer2/abstract_sql/abstract_sql_store.go

@@ -1,6 +1,7 @@
 package abstract_sql
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 
@@ -18,7 +19,44 @@ type AbstractSqlStore struct {
 	SqlListInclusive string
 }
 
-func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
+type TxOrDB interface {
+	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
+	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+}
+
+func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
+		Isolation: sql.LevelReadCommitted,
+		ReadOnly:  false,
+	})
+	if err != nil {
+		return ctx, err
+	}
+
+	return context.WithValue(ctx, "tx", tx), nil
+}
+func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Commit()
+	}
+	return nil
+}
+func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Rollback()
+	}
+	return nil
+}
+
+func (store *AbstractSqlStore) getTxOrDB(ctx context.Context) TxOrDB {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx
+	}
+	return store.DB
+}
+
+func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
@@ -26,7 +64,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	res, err := store.DB.Exec(store.SqlInsert, hashToLong(dir), name, dir, meta)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlInsert, hashToLong(dir), name, dir, meta)
 	if err != nil {
 		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
 	}
@@ -38,7 +76,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
@@ -46,7 +84,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
 		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
 	}
 
-	res, err := store.DB.Exec(store.SqlUpdate, meta, hashToLong(dir), name, dir)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlUpdate, meta, hashToLong(dir), name, dir)
 	if err != nil {
 		return fmt.Errorf("update %s: %s", entry.FullPath, err)
 	}
@@ -58,10 +96,10 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) {
+func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (*filer2.Entry, error) {
 
 	dir, name := fullpath.DirAndName()
-	row := store.DB.QueryRow(store.SqlFind, hashToLong(dir), name, dir)
+	row := store.getTxOrDB(ctx).QueryRowContext(ctx, store.SqlFind, hashToLong(dir), name, dir)
 	var data []byte
 	if err := row.Scan(&data); err != nil {
 		return nil, filer2.ErrNotFound
@@ -77,11 +115,11 @@ func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entr
 	return entry, nil
 }
 
-func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) error {
+func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
 
 	dir, name := fullpath.DirAndName()
 
-	res, err := store.DB.Exec(store.SqlDelete, hashToLong(dir), name, dir)
+	res, err := store.getTxOrDB(ctx).ExecContext(ctx, store.SqlDelete, hashToLong(dir), name, dir)
 	if err != nil {
 		return fmt.Errorf("delete %s: %s", fullpath, err)
 	}
@@ -94,14 +132,14 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) error {
 	return nil
 }
 
-func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
+func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
 
 	sqlText := store.SqlListExclusive
 	if inclusive {
 		sqlText = store.SqlListInclusive
 	}
 
-	rows, err := store.DB.Query(sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
+	rows, err := store.getTxOrDB(ctx).QueryContext(ctx, sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
 	if err != nil {
 		return nil, fmt.Errorf("list %s : %v", fullpath, err)
 	}

weed/filer2/cassandra/cassandra_store.go

@@ -1,6 +1,7 @@
 package cassandra
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -39,7 +40,17 @@ func (store *CassandraStore) initialize(keyspace string, hosts []string) (err er
 	return
 }
 
-func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
+func (store *CassandraStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *CassandraStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *CassandraStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	dir, name := entry.FullPath.DirAndName()
 	meta, err := entry.EncodeAttributesAndChunks()
@@ -56,12 +67,12 @@ func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *CassandraStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *CassandraStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
-	return store.InsertEntry(entry)
+	return store.InsertEntry(ctx, entry)
 }
 
-func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *CassandraStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 
 	dir, name := fullpath.DirAndName()
 	var data []byte
@@ -88,7 +99,7 @@ func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.
 	return entry, nil
 }
 
-func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) error {
+func (store *CassandraStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) error {
 
 	dir, name := fullpath.DirAndName()
 
@@ -101,7 +112,7 @@ func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) error {
 	return nil
 }
 
-func (store *CassandraStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
+func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer2.Entry, err error) {
 
 	cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?"

weed/filer2/entry.go

@@ -52,9 +52,20 @@ func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
 		return nil
 	}
 	return &filer_pb.Entry{
-		Name:        string(entry.FullPath),
+		Name:        entry.FullPath.Name(),
 		IsDirectory: entry.IsDirectory(),
 		Attributes:  EntryAttributeToPb(entry),
 		Chunks:      entry.Chunks,
 	}
 }
+
+func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
+	if entry == nil {
+		return nil
+	}
+	dir, _ := entry.FullPath.DirAndName()
+	return &filer_pb.FullEntry{
+		Dir:   dir,
+		Entry: entry.ToProtoEntry(),
+	}
+}

weed/filer2/entry_codec.go

@@ -6,7 +6,7 @@ import (
 
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/gogo/protobuf/proto"
+	"github.com/golang/protobuf/proto"
 )
 
 func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {

weed/filer2/filer.go

@@ -3,6 +3,7 @@ package filer2
 import (
 	"context"
 	"fmt"
+	"google.golang.org/grpc"
 	"math"
 	"os"
 	"path/filepath"
@@ -20,17 +21,19 @@ var (
 )
 
 type Filer struct {
-	store              FilerStore
+	store              *FilerStoreWrapper
 	directoryCache     *ccache.Cache
 	MasterClient       *wdclient.MasterClient
 	fileIdDeletionChan chan string
+	GrpcDialOption     grpc.DialOption
 }
 
-func NewFiler(masters []string) *Filer {
+func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
 	f := &Filer{
 		directoryCache:     ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
-		MasterClient:       wdclient.NewMasterClient(context.Background(), "filer", masters),
+		MasterClient:       wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
 		fileIdDeletionChan: make(chan string, 4096),
+		GrpcDialOption:     grpcDialOption,
 	}
 
 	go f.loopProcessingDeletion()
@@ -39,7 +42,7 @@ func NewFiler(masters []string) *Filer {
 }
 
 func (f *Filer) SetStore(store FilerStore) {
-	f.store = store
+	f.store = NewFilerStoreWrapper(store)
 }
 
 func (f *Filer) DisableDirectoryCache() {
@@ -54,7 +57,19 @@ func (fs *Filer) KeepConnectedToMaster() {
 	fs.MasterClient.KeepConnectedToMaster()
 }
 
-func (f *Filer) CreateEntry(entry *Entry) error {
+func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return f.store.BeginTransaction(ctx)
+}
+
+func (f *Filer) CommitTransaction(ctx context.Context) error {
+	return f.store.CommitTransaction(ctx)
+}
+
+func (f *Filer) RollbackTransaction(ctx context.Context) error {
+	return f.store.RollbackTransaction(ctx)
+}
+
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 
 	if string(entry.FullPath) == "/" {
 		return nil
@@ -67,7 +82,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 	var lastDirectoryEntry *Entry
 
 	for i := 1; i < len(dirParts); i++ {
-		dirPath := "/" + filepath.Join(dirParts[:i]...)
+		dirPath := "/" + filepath.ToSlash(filepath.Join(dirParts[:i]...))
 		// fmt.Printf("%d directory: %+v\n", i, dirPath)
 
 		// first check local cache
@@ -76,7 +91,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 		// not found, check the store directly
 		if dirEntry == nil {
 			glog.V(4).Infof("find uncached directory: %s", dirPath)
-			dirEntry, _ = f.FindEntry(FullPath(dirPath))
+			dirEntry, _ = f.FindEntry(ctx, FullPath(dirPath))
 		} else {
 			glog.V(4).Infof("found cached directory: %s", dirPath)
 		}
@@ -99,9 +114,9 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 			}
 
 			glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
-			mkdirErr := f.store.InsertEntry(dirEntry)
+			mkdirErr := f.store.InsertEntry(ctx, dirEntry)
 			if mkdirErr != nil {
-				if _, err := f.FindEntry(FullPath(dirPath)); err == ErrNotFound {
+				if _, err := f.FindEntry(ctx, FullPath(dirPath)); err == ErrNotFound {
 					return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
 				}
 			} else {
@@ -134,14 +149,14 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 		}
 	*/
 
-	oldEntry, _ := f.FindEntry(entry.FullPath)
+	oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
 
 	if oldEntry == nil {
-		if err := f.store.InsertEntry(entry); err != nil {
+		if err := f.store.InsertEntry(ctx, entry); err != nil {
 			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
 		}
 	} else {
-		if err := f.UpdateEntry(oldEntry, entry); err != nil {
+		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
 			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
 		}
 	}
@@ -153,7 +168,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 	return nil
 }
 
-func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
+func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
 	if oldEntry != nil {
 		if oldEntry.IsDirectory() && !entry.IsDirectory() {
 			return fmt.Errorf("existing %s is a directory", entry.FullPath)
@@ -162,10 +177,10 @@ func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
 			return fmt.Errorf("existing %s is a file", entry.FullPath)
 		}
 	}
-	return f.store.UpdateEntry(entry)
+	return f.store.UpdateEntry(ctx, entry)
 }
 
-func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
+func (f *Filer) FindEntry(ctx context.Context, p FullPath) (entry *Entry, err error) {
 
 	now := time.Now()
 
@@ -181,11 +196,11 @@ func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
 			},
 		}, nil
 	}
-	return f.store.FindEntry(p)
+	return f.store.FindEntry(ctx, p)
 }
 
-func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
-	entry, err := f.FindEntry(p)
+func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecursive bool, shouldDeleteChunks bool) (err error) {
+	entry, err := f.FindEntry(ctx, p)
 	if err != nil {
 		return err
 	}
@@ -198,37 +213,40 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 		lastFileName := ""
 		includeLastFile := false
 		for limit > 0 {
-			entries, err := f.ListDirectoryEntries(p, lastFileName, includeLastFile, 1024)
+			entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
 			if err != nil {
 				return fmt.Errorf("list folder %s: %v", p, err)
 			}
+
 			if len(entries) == 0 {
 				break
-			} else {
-				if isRecursive {
-					for _, sub := range entries {
-						lastFileName = sub.Name()
-						f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
-						limit--
-						if limit <= 0 {
-							break
-						}
+			}
+
+			if isRecursive {
+				for _, sub := range entries {
+					lastFileName = sub.Name()
+					err = f.DeleteEntryMetaAndData(ctx, sub.FullPath, isRecursive, shouldDeleteChunks)
+					if err != nil {
+						return err
 					}
-				} else {
-					if len(entries) > 0 {
-						return fmt.Errorf("folder %s is not empty", p)
+					limit--
+					if limit <= 0 {
+						break
 					}
 				}
-				f.cacheDelDirectory(string(p))
-				if len(entries) < 1024 {
-					break
-				}
+			}
+
+			if len(entries) < 1024 {
+				break
 			}
 		}
+
+		f.cacheDelDirectory(string(p))
+
 	}
 
 	if shouldDeleteChunks {
-		f.DeleteChunks(entry.Chunks)
+		f.DeleteChunks(p, entry.Chunks)
 	}
 
 	if p == "/" {
@@ -238,17 +256,22 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 
 	f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
 
-	return f.store.DeleteEntry(p)
+	return f.store.DeleteEntry(ctx, p)
 }
 
-func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
+func (f *Filer) ListDirectoryEntries(ctx context.Context, p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) {
 	if strings.HasSuffix(string(p), "/") && len(p) > 1 {
 		p = p[0 : len(p)-1]
 	}
-	return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit)
+	return f.store.ListDirectoryEntries(ctx, p, startFileName, inclusive, limit)
 }
 
 func (f *Filer) cacheDelDirectory(dirpath string) {
+
+	if dirpath == "/" {
+		return
+	}
+
 	if f.directoryCache == nil {
 		return
 	}
@@ -257,6 +280,7 @@ func (f *Filer) cacheDelDirectory(dirpath string) {
 }
 
 func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
+
 	if f.directoryCache == nil {
 		return nil
 	}

weed/filer2/filer_client_util.go

@@ -0,0 +1,163 @@
+package filer2
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func VolumeId(fileId string) string {
+	lastCommaIndex := strings.LastIndex(fileId, ",")
+	if lastCommaIndex > 0 {
+		return fileId[:lastCommaIndex]
+	}
+	return fileId
+}
+
+type FilerClient interface {
+	WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error
+}
+
+func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath string, buff []byte, chunkViews []*ChunkView, baseOffset int64) (totalRead int64, err error) {
+	var vids []string
+	for _, chunkView := range chunkViews {
+		vids = append(vids, VolumeId(chunkView.FileId))
+	}
+
+	vid2Locations := make(map[string]*filer_pb.Locations)
+
+	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+
+		glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+			VolumeIds: vids,
+		})
+		if err != nil {
+			return err
+		}
+
+		vid2Locations = resp.LocationsMap
+
+		return nil
+	})
+
+	if err != nil {
+		return 0, fmt.Errorf("failed to lookup volume ids %v: %v", vids, err)
+	}
+
+	var wg sync.WaitGroup
+	for _, chunkView := range chunkViews {
+		wg.Add(1)
+		go func(chunkView *ChunkView) {
+			defer wg.Done()
+
+			glog.V(4).Infof("read fh reading chunk: %+v", chunkView)
+
+			locations := vid2Locations[VolumeId(chunkView.FileId)]
+			if locations == nil || len(locations.Locations) == 0 {
+				glog.V(0).Infof("failed to locate %s", chunkView.FileId)
+				err = fmt.Errorf("failed to locate %s", chunkView.FileId)
+				return
+			}
+
+			var n int64
+			n, err = util.ReadUrl(
+				fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
+				chunkView.Offset,
+				int(chunkView.Size),
+				buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)],
+				!chunkView.IsFullChunk)
+
+			if err != nil {
+
+				glog.V(0).Infof("%v read http://%s/%v %v bytes: %v", fullFilePath, locations.Locations[0].Url, chunkView.FileId, n, err)
+
+				err = fmt.Errorf("failed to read http://%s/%s: %v",
+					locations.Locations[0].Url, chunkView.FileId, err)
+				return
+			}
+
+			glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView)
+			totalRead += n
+
+		}(chunkView)
+	}
+	wg.Wait()
+	return
+}
+
+func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath string) (entry *filer_pb.Entry, err error) {
+
+	dir, name := FullPath(fullFilePath).DirAndName()
+
+	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		}
+
+		glog.V(3).Infof("read %s request: %v", fullFilePath, request)
+		resp, err := client.LookupDirectoryEntry(ctx, request)
+		if err != nil {
+			if err == ErrNotFound || strings.Contains(err.Error(), ErrNotFound.Error()) {
+				return nil
+			}
+			glog.V(3).Infof("read %s attr %v: %v", fullFilePath, request, err)
+			return err
+		}
+
+		if resp.Entry != nil {
+			entry = resp.Entry
+		}
+
+		return nil
+	})
+
+	return
+}
+
+func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath string, fn func(entry *filer_pb.Entry)) (err error) {
+
+	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+
+		paginationLimit := 1024
+
+		lastEntryName := ""
+
+		for {
+
+			request := &filer_pb.ListEntriesRequest{
+				Directory:         fullDirPath,
+				StartFromFileName: lastEntryName,
+				Limit:             uint32(paginationLimit),
+			}
+
+			glog.V(3).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
+			if err != nil {
+				return fmt.Errorf("list %s: %v", fullDirPath, err)
+			}
+
+			for _, entry := range resp.Entries {
+				fn(entry)
+				lastEntryName = entry.Name
+			}
+
+			if len(resp.Entries) < paginationLimit {
+				break
+			}
+
+		}
+
+		return nil
+
+	})
+
+	return
+}

weed/filer2/filer_deletion.go

@@ -38,25 +38,28 @@ func (f *Filer) loopProcessingDeletion() {
 			fileIds = append(fileIds, fid)
 			if len(fileIds) >= 4096 {
 				glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
-				operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+				operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
 				fileIds = fileIds[:0]
 			}
 		case <-ticker.C:
 			if len(fileIds) > 0 {
 				glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
-				operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+				operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
 				fileIds = fileIds[:0]
 			}
 		}
 	}
 }
 
-func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
+func (f *Filer) DeleteChunks(fullpath FullPath, chunks []*filer_pb.FileChunk) {
 	for _, chunk := range chunks {
+		glog.V(3).Infof("deleting %s chunk %s", fullpath, chunk.String())
 		f.fileIdDeletionChan <- chunk.FileId
 	}
 }
 
+// DeleteFileByFileId direct delete by file id.
+// Only used when the fileId is not being managed by snapshots.
 func (f *Filer) DeleteFileByFileId(fileId string) {
 	f.fileIdDeletionChan <- fileId
 }
@@ -67,7 +70,7 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
 		return
 	}
 	if newEntry == nil {
-		f.DeleteChunks(oldEntry.Chunks)
+		f.DeleteChunks(oldEntry.FullPath, oldEntry.Chunks)
 	}
 
 	var toDelete []*filer_pb.FileChunk
@@ -84,5 +87,5 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
 			toDelete = append(toDelete, oldChunk)
 		}
 	}
-	f.DeleteChunks(toDelete)
+	f.DeleteChunks(oldEntry.FullPath, toDelete)
 }

weed/filer2/filer_notify.go

@@ -20,12 +20,18 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool)
 
 		glog.V(3).Infof("notifying entry update %v", key)
 
+		newParentPath := ""
+		if newEntry != nil {
+			newParentPath, _ = newEntry.FullPath.DirAndName()
+		}
+
 		notification.Queue.SendMessage(
 			key,
 			&filer_pb.EventNotification{
-				OldEntry:     oldEntry.ToProtoEntry(),
-				NewEntry:     newEntry.ToProtoEntry(),
-				DeleteChunks: deleteChunks,
+				OldEntry:      oldEntry.ToProtoEntry(),
+				NewEntry:      newEntry.ToProtoEntry(),
+				DeleteChunks:  deleteChunks,
+				NewParentPath: newParentPath,
 			},
 		)
 

weed/filer2/filerstore.go

@@ -1,7 +1,9 @@
 package filer2
 
 import (
+	"context"
 	"errors"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -10,12 +12,80 @@ type FilerStore interface {
 	GetName() string
 	// Initialize initializes the file store
 	Initialize(configuration util.Configuration) error
-	InsertEntry(*Entry) error
-	UpdateEntry(*Entry) (err error)
+	InsertEntry(context.Context, *Entry) error
+	UpdateEntry(context.Context, *Entry) (err error)
 	// err == filer2.ErrNotFound if not found
-	FindEntry(FullPath) (entry *Entry, err error)
-	DeleteEntry(FullPath) (err error)
-	ListDirectoryEntries(dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error)
+	FindEntry(context.Context, FullPath) (entry *Entry, err error)
+	DeleteEntry(context.Context, FullPath) (err error)
+	ListDirectoryEntries(ctx context.Context, dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error)
+
+	BeginTransaction(ctx context.Context) (context.Context, error)
+	CommitTransaction(ctx context.Context) error
+	RollbackTransaction(ctx context.Context) error
 }
 
 var ErrNotFound = errors.New("filer: no entry is found in filer store")
+
+type FilerStoreWrapper struct {
+	actualStore FilerStore
+}
+
+func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
+	return &FilerStoreWrapper{
+		actualStore: store,
+	}
+}
+
+func (fsw *FilerStoreWrapper) GetName() string {
+	return fsw.actualStore.GetName()
+}
+
+func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration) error {
+	return fsw.actualStore.Initialize(configuration)
+}
+
+func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {
+	filer_pb.BeforeEntrySerialization(entry.Chunks)
+	return fsw.actualStore.InsertEntry(ctx, entry)
+}
+
+func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
+	filer_pb.BeforeEntrySerialization(entry.Chunks)
+	return fsw.actualStore.UpdateEntry(ctx, entry)
+}
+
+func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp FullPath) (entry *Entry, err error) {
+	entry, err = fsw.actualStore.FindEntry(ctx, fp)
+	if err != nil {
+		return nil, err
+	}
+	filer_pb.AfterEntryDeserialization(entry.Chunks)
+	return
+}
+
+func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp FullPath) (err error) {
+	return fsw.actualStore.DeleteEntry(ctx, fp)
+}
+
+func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) {
+	entries, err := fsw.actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range entries {
+		filer_pb.AfterEntryDeserialization(entry.Chunks)
+	}
+	return entries, err
+}
+
+func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return fsw.actualStore.BeginTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) CommitTransaction(ctx context.Context) error {
+	return fsw.actualStore.CommitTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) RollbackTransaction(ctx context.Context) error {
+	return fsw.actualStore.RollbackTransaction(ctx)
+}

weed/filer2/fullpath.go

@@ -8,10 +8,7 @@ import (
 type FullPath string
 
 func NewFullPath(dir, name string) FullPath {
-	if strings.HasSuffix(dir, "/") {
-		return FullPath(dir + name)
-	}
-	return FullPath(dir + "/" + name)
+	return FullPath(dir).Child(name)
 }
 
 func (fp FullPath) DirAndName() (string, string) {
@@ -29,3 +26,11 @@ func (fp FullPath) Name() string {
 	_, name := filepath.Split(string(fp))
 	return name
 }
+
+func (fp FullPath) Child(name string) FullPath {
+	dir := string(fp)
+	if strings.HasSuffix(dir, "/") {
+		return FullPath(dir + name)
+	}
+	return FullPath(dir + "/" + name)
+}

weed/filer2/leveldb/leveldb_store.go

@@ -2,12 +2,14 @@ package leveldb
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	weed_util "github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -38,14 +40,30 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}
 
-	if store.db, err = leveldb.OpenFile(dir, nil); err != nil {
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+	}
+
+	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
 		glog.Infof("filer store open dir %s: %v", dir, err)
 		return
 	}
 	return
 }
 
-func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) {
+func (store *LevelDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *LevelDBStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 	key := genKey(entry.DirAndName())
 
 	value, err := entry.EncodeAttributesAndChunks()
@@ -64,12 +82,12 @@ func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *LevelDBStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *LevelDBStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
-	return store.InsertEntry(entry)
+	return store.InsertEntry(ctx, entry)
 }
 
-func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 	key := genKey(fullpath.DirAndName())
 
 	data, err := store.db.Get(key, nil)
@@ -94,7 +112,7 @@ func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.En
 	return entry, nil
 }
 
-func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
+func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
 	key := genKey(fullpath.DirAndName())
 
 	err = store.db.Delete(key, nil)
@@ -105,7 +123,7 @@ func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
 	return nil
 }
 
-func (store *LevelDBStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
+func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer2.Entry, err error) {
 
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")

weed/filer2/leveldb/leveldb_store_test.go

@@ -1,6 +1,7 @@
 package leveldb
 
 import (
+	"context"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"io/ioutil"
 	"os"
@@ -8,7 +9,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -18,6 +19,8 @@ func TestCreateAndFind(t *testing.T) {
 
 	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
 
+	ctx := context.Background()
+
 	entry1 := &filer2.Entry{
 		FullPath: fullpath,
 		Attr: filer2.Attr{
@@ -27,12 +30,12 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := filer.CreateEntry(entry1); err != nil {
+	if err := filer.CreateEntry(ctx, entry1); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
 
-	entry, err := filer.FindEntry(fullpath)
+	entry, err := filer.FindEntry(ctx, fullpath)
 
 	if err != nil {
 		t.Errorf("find entry: %v", err)
@@ -45,14 +48,14 @@ func TestCreateAndFind(t *testing.T) {
 	}
 
 	// checking one upper directory
-	entries, _ := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
+	entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking one upper directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
@@ -61,7 +64,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -69,8 +72,10 @@ func TestEmptyRoot(t *testing.T) {
 	filer.SetStore(store)
 	filer.DisableDirectoryCache()
 
+	ctx := context.Background()
+
 	// checking one upper directory
-	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
 	if err != nil {
 		t.Errorf("list entries: %v", err)
 		return

weed/filer2/leveldb2/leveldb2_store.go

@@ -0,0 +1,190 @@
+package leveldb
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	weed_util "github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func init() {
+	filer2.Stores = append(filer2.Stores, &LevelDB2Store{})
+}
+
+// known theoretically 128 bit MD5 collision of 2 directories.
+// (but really? please show some real examples)
+type LevelDB2Store struct {
+	db *leveldb.DB
+}
+
+func (store *LevelDB2Store) GetName() string {
+	return "leveldb2"
+}
+
+func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration) (err error) {
+	dir := configuration.GetString("dir")
+	return store.initialize(dir)
+}
+
+func (store *LevelDB2Store) initialize(dir string) (err error) {
+	glog.Infof("filer store leveldb2 dir: %s", dir)
+	if err := weed_util.TestFolderWritable(dir); err != nil {
+		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
+	}
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+	}
+
+	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
+		glog.Infof("filer store open dir %s: %v", dir, err)
+		return
+	}
+	return
+}
+
+func (store *LevelDB2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *LevelDB2Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+	key := genKey(entry.DirAndName())
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	err = store.db.Put(key, value, nil)
+
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	// println("saved", entry.FullPath, "chunks", len(entry.Chunks))
+
+	return nil
+}
+
+func (store *LevelDB2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+	key := genKey(fullpath.DirAndName())
+
+	data, err := store.db.Get(key, nil)
+
+	if err == leveldb.ErrNotFound {
+		return nil, filer2.ErrNotFound
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)
+	}
+
+	entry = &filer2.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(data)
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))
+
+	return entry, nil
+}
+
+func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
+	key := genKey(fullpath.DirAndName())
+
+	err = store.db.Delete(key, nil)
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
+	limit int) (entries []*filer2.Entry, err error) {
+
+	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
+
+	iter := store.db.NewIterator(&leveldb_util.Range{Start: genDirectoryKeyPrefix(fullpath, startFileName)}, nil)
+	for iter.Next() {
+		key := iter.Key()
+		if !bytes.HasPrefix(key, directoryPrefix) {
+			break
+		}
+		fileName := getNameFromKey(key)
+		if fileName == "" {
+			continue
+		}
+		if fileName == startFileName && !inclusive {
+			continue
+		}
+		limit--
+		if limit < 0 {
+			break
+		}
+		entry := &filer2.Entry{
+			FullPath: filer2.NewFullPath(string(fullpath), fileName),
+		}
+		if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+			break
+		}
+		entries = append(entries, entry)
+	}
+	iter.Release()
+
+	return entries, err
+}
+
+func genKey(dirPath, fileName string) (key []byte) {
+	key = hashToBytes(dirPath)
+	key = append(key, []byte(fileName)...)
+	return key
+}
+
+func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyPrefix []byte) {
+	keyPrefix = hashToBytes(string(fullpath))
+	if len(startFileName) > 0 {
+		keyPrefix = append(keyPrefix, []byte(startFileName)...)
+	}
+	return keyPrefix
+}
+
+func getNameFromKey(key []byte) string {
+
+	return string(key[8:])
+
+}
+
+func hashToBytes(dir string) []byte {
+	h := md5.New()
+	io.WriteString(h, dir)
+
+	b := h.Sum(nil)
+
+	return b
+}

weed/filer2/leveldb2/leveldb2_store_test.go

@@ -0,0 +1,88 @@
+package leveldb
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestCreateAndFind(t *testing.T) {
+	filer := filer2.NewFiler(nil, nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	defer os.RemoveAll(dir)
+	store := &LevelDB2Store{}
+	store.initialize(dir)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
+
+	ctx := context.Background()
+
+	entry1 := &filer2.Entry{
+		FullPath: fullpath,
+		Attr: filer2.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+
+	if err := filer.CreateEntry(ctx, entry1); err != nil {
+		t.Errorf("create entry %v: %v", entry1.FullPath, err)
+		return
+	}
+
+	entry, err := filer.FindEntry(ctx, fullpath)
+
+	if err != nil {
+		t.Errorf("find entry: %v", err)
+		return
+	}
+
+	if entry.FullPath != entry1.FullPath {
+		t.Errorf("find wrong entry: %v", entry.FullPath)
+		return
+	}
+
+	// checking one upper directory
+	entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+	// checking one upper directory
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}
+
+func TestEmptyRoot(t *testing.T) {
+	filer := filer2.NewFiler(nil, nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	defer os.RemoveAll(dir)
+	store := &LevelDB2Store{}
+	store.initialize(dir)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	ctx := context.Background()
+
+	// checking one upper directory
+	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
+	if err != nil {
+		t.Errorf("list entries: %v", err)
+		return
+	}
+	if len(entries) != 0 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}

weed/filer2/memdb/memdb_store.go

@@ -1,11 +1,13 @@
 package memdb
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/google/btree"
 	"strings"
+	"sync"
 )
 
 func init() {
@@ -13,7 +15,8 @@ func init() {
 }
 
 type MemDbStore struct {
-	tree *btree.BTree
+	tree     *btree.BTree
+	treeLock sync.Mutex
 }
 
 type entryItem struct {
@@ -33,21 +36,35 @@ func (store *MemDbStore) Initialize(configuration util.Configuration) (err error
 	return nil
 }
 
-func (store *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
-	// println("inserting", entry.FullPath)
-	store.tree.ReplaceOrInsert(entryItem{entry})
+func (store *MemDbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *MemDbStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *MemDbStore) RollbackTransaction(ctx context.Context) error {
 	return nil
 }
 
-func (store *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) {
-	if _, err = store.FindEntry(entry.FullPath); err != nil {
+func (store *MemDbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+	// println("inserting", entry.FullPath)
+	store.treeLock.Lock()
+	store.tree.ReplaceOrInsert(entryItem{entry})
+	store.treeLock.Unlock()
+	return nil
+}
+
+func (store *MemDbStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+	if _, err = store.FindEntry(ctx, entry.FullPath); err != nil {
 		return fmt.Errorf("no such file %s : %v", entry.FullPath, err)
 	}
+	store.treeLock.Lock()
 	store.tree.ReplaceOrInsert(entryItem{entry})
+	store.treeLock.Unlock()
 	return nil
 }
 
-func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *MemDbStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 	item := store.tree.Get(entryItem{&filer2.Entry{FullPath: fullpath}})
 	if item == nil {
 		return nil, filer2.ErrNotFound
@@ -56,12 +73,14 @@ func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entr
 	return entry, nil
 }
 
-func (store *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
+func (store *MemDbStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
+	store.treeLock.Lock()
 	store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}})
+	store.treeLock.Unlock()
 	return nil
 }
 
-func (store *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
+func (store *MemDbStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
 
 	startFrom := string(fullpath)
 	if startFileName != "" {

weed/filer2/memdb/memdb_store_test.go

@@ -1,17 +1,20 @@
 package memdb
 
 import (
+	"context"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"testing"
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	store := &MemDbStore{}
 	store.Initialize(nil)
 	filer.SetStore(store)
 	filer.DisableDirectoryCache()
 
+	ctx := context.Background()
+
 	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
 
 	entry1 := &filer2.Entry{
@@ -23,12 +26,12 @@ func TestCreateAndFind(t *testing.T) {
 		},
 	}
 
-	if err := filer.CreateEntry(entry1); err != nil {
+	if err := filer.CreateEntry(ctx, entry1); err != nil {
 		t.Errorf("create entry %v: %v", entry1.FullPath, err)
 		return
 	}
 
-	entry, err := filer.FindEntry(fullpath)
+	entry, err := filer.FindEntry(ctx, fullpath)
 
 	if err != nil {
 		t.Errorf("find entry: %v", err)
@@ -43,12 +46,14 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestCreateFileAndList(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	store := &MemDbStore{}
 	store.Initialize(nil)
 	filer.SetStore(store)
 	filer.DisableDirectoryCache()
 
+	ctx := context.Background()
+
 	entry1 := &filer2.Entry{
 		FullPath: filer2.FullPath("/home/chris/this/is/one/file1.jpg"),
 		Attr: filer2.Attr{
@@ -67,11 +72,11 @@ func TestCreateFileAndList(t *testing.T) {
 		},
 	}
 
-	filer.CreateEntry(entry1)
-	filer.CreateEntry(entry2)
+	filer.CreateEntry(ctx, entry1)
+	filer.CreateEntry(ctx, entry2)
 
 	// checking the 2 files
-	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)
+	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "", false, 100)
 
 	if err != nil {
 		t.Errorf("list entries: %v", err)
@@ -94,21 +99,21 @@ func TestCreateFileAndList(t *testing.T) {
 	}
 
 	// checking the offset
-	entries, err = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
+	entries, err = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking one upper directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// checking root directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
@@ -124,18 +129,18 @@ func TestCreateFileAndList(t *testing.T) {
 			Gid:  5678,
 		},
 	}
-	filer.CreateEntry(entry3)
+	filer.CreateEntry(ctx, entry3)
 
 	// checking one upper directory
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
 	if len(entries) != 2 {
 		t.Errorf("list entries count: %v", len(entries))
 		return
 	}
 
 	// delete file and count
-	filer.DeleteEntryMetaAndData(file3Path, false, false)
-	entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100)
+	filer.DeleteEntryMetaAndData(ctx, file3Path, false, false)
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is"), "", false, 100)
 	if len(entries) != 1 {
 		t.Errorf("list entries count: %v", len(entries))
 		return

weed/filer2/postgres/README.txt

@@ -9,8 +9,8 @@ $PGHOME/bin/psql --username=postgres --password seaweedfs
 
 CREATE TABLE IF NOT EXISTS filemeta (
   dirhash     BIGINT,
-  name        VARCHAR(1000),
-  directory   VARCHAR(4096),
+  name        VARCHAR(65535),
+  directory   VARCHAR(65535),
   meta        bytea,
   PRIMARY KEY (dirhash, name)
 );

weed/filer2/redis/redis_cluster_store.go

@@ -21,12 +21,14 @@ func (store *RedisClusterStore) GetName() string {
 func (store *RedisClusterStore) Initialize(configuration util.Configuration) (err error) {
 	return store.initialize(
 		configuration.GetStringSlice("addresses"),
+		configuration.GetString("password"),
 	)
 }
 
-func (store *RedisClusterStore) initialize(addresses []string) (err error) {
+func (store *RedisClusterStore) initialize(addresses []string, password string) (err error) {
 	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
-		Addrs: addresses,
+		Addrs:    addresses,
+		Password: password,
 	})
 	return
 }

weed/filer2/redis/universal_redis_store.go

@@ -1,6 +1,7 @@
 package redis
 
 import (
+	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -18,7 +19,17 @@ type UniversalRedisStore struct {
 	Client redis.UniversalClient
 }
 
-func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
+func (store *UniversalRedisStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedisStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedisStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
 	value, err := entry.EncodeAttributesAndChunks()
 	if err != nil {
@@ -42,12 +53,12 @@ func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (store *UniversalRedisStore) UpdateEntry(entry *filer2.Entry) (err error) {
+func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 
-	return store.InsertEntry(entry)
+	return store.InsertEntry(ctx, entry)
 }
 
-func (store *UniversalRedisStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
 
 	data, err := store.Client.Get(string(fullpath)).Result()
 	if err == redis.Nil {
@@ -69,7 +80,7 @@ func (store *UniversalRedisStore) FindEntry(fullpath filer2.FullPath) (entry *fi
 	return entry, nil
 }
 
-func (store *UniversalRedisStore) DeleteEntry(fullpath filer2.FullPath) (err error) {
+func (store *UniversalRedisStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
 
 	_, err = store.Client.Del(string(fullpath)).Result()
 
@@ -88,7 +99,7 @@ func (store *UniversalRedisStore) DeleteEntry(fullpath filer2.FullPath) (err err
 	return nil
 }
 
-func (store *UniversalRedisStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
+func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
 	limit int) (entries []*filer2.Entry, err error) {
 
 	members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result()
@@ -126,7 +137,7 @@ func (store *UniversalRedisStore) ListDirectoryEntries(fullpath filer2.FullPath,
 	// fetch entry meta
 	for _, fileName := range members {
 		path := filer2.NewFullPath(string(fullpath), fileName)
-		entry, err := store.FindEntry(path)
+		entry, err := store.FindEntry(ctx, path)
 		if err != nil {
 			glog.V(0).Infof("list %s : %v", path, err)
 		} else {

weed/filer2/stream.go

@@ -0,0 +1,41 @@
+package filer2
+
+import (
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+)
+
+func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int) error {
+
+	chunkViews := ViewFromChunks(chunks, offset, size)
+
+	fileId2Url := make(map[string]string)
+
+	for _, chunkView := range chunkViews {
+
+		urlString, err := masterClient.LookupFileId(chunkView.FileId)
+		if err != nil {
+			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
+			return err
+		}
+		fileId2Url[chunkView.FileId] = urlString
+	}
+
+	for _, chunkView := range chunkViews {
+		urlString := fileId2Url[chunkView.FileId]
+		_, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) {
+			w.Write(data)
+		})
+		if err != nil {
+			glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
+			return err
+		}
+	}
+
+	return nil
+
+}

weed/filesys/dir.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"path"
-	"path/filepath"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
@@ -29,15 +28,13 @@ var _ = fs.NodeRemover(&Dir{})
 var _ = fs.NodeRenamer(&Dir{})
 var _ = fs.NodeSetattrer(&Dir{})
 
-func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
+func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 
 	// https://github.com/bazil/fuse/issues/196
 	attr.Valid = time.Second
 
 	if dir.Path == dir.wfs.option.FilerMountRootPath {
-		attr.Uid = dir.wfs.option.MountUid
-		attr.Gid = dir.wfs.option.MountGid
-		attr.Mode = dir.wfs.option.MountMode
+		dir.setRootDirAttributes(attr)
 		return nil
 	}
 
@@ -54,40 +51,14 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 		return nil
 	}
 
-	parent, name := filepath.Split(dir.Path)
-
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: parent,
-			Name:      name,
-		}
-
-		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
-		resp, err := client.LookupDirectoryEntry(context, request)
-		if err != nil {
-			if err == filer2.ErrNotFound {
-				return nil
-			}
-			glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
-			return err
-		}
-
-		if resp.Entry != nil {
-			dir.attributes = resp.Entry.Attributes
-		}
-
-		// dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
-
-		return nil
-	})
-
+	entry, err := filer2.GetEntry(ctx, dir.wfs, dir.Path)
 	if err != nil {
+		glog.V(2).Infof("read dir %s attr: %v, error: %v", dir.Path, dir.attributes, err)
 		return err
 	}
+	dir.attributes = entry.Attributes
 
-	// glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
-	// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))
+	glog.V(2).Infof("dir %s: %v perm: %v", dir.Path, dir.attributes, os.FileMode(dir.attributes.FileMode))
 
 	attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir
 
@@ -99,6 +70,16 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 	return nil
 }
 
+func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
+	attr.Uid = dir.wfs.option.MountUid
+	attr.Gid = dir.wfs.option.MountGid
+	attr.Mode = dir.wfs.option.MountMode
+	attr.Crtime = dir.wfs.option.MountCtime
+	attr.Ctime = dir.wfs.option.MountCtime
+	attr.Mtime = dir.wfs.option.MountMtime
+	attr.Atime = dir.wfs.option.MountMtime
+}
+
 func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
 	return &File{
 		Name:           name,
@@ -132,7 +113,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	glog.V(1).Infof("create: %v", request)
 
 	if request.Entry.IsDirectory {
-		if err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		if err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 			if _, err := client.CreateEntry(ctx, request); err != nil {
 				glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
 				return fuse.EIO
@@ -155,7 +136,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 
 func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
 
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.CreateEntryRequest{
 			Directory: dir.Path,
@@ -192,33 +173,18 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err
 func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
 
 	var entry *filer_pb.Entry
+	fullFilePath := path.Join(dir.Path, req.Name)
 
-	item := dir.wfs.listDirectoryEntriesCache.Get(path.Join(dir.Path, req.Name))
+	item := dir.wfs.listDirectoryEntriesCache.Get(fullFilePath)
 	if item != nil && !item.Expired() {
 		entry = item.Value().(*filer_pb.Entry)
 	}
 
 	if entry == nil {
-		err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-			request := &filer_pb.LookupDirectoryEntryRequest{
-				Directory: dir.Path,
-				Name:      req.Name,
-			}
-
-			glog.V(4).Infof("lookup directory entry: %v", request)
-			resp, err := client.LookupDirectoryEntry(ctx, request)
-			if err != nil {
-				// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
-				return fuse.ENOENT
-			}
-
-			entry = resp.Entry
-
-			// dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
-
-			return nil
-		})
+		entry, err = filer2.GetEntry(ctx, dir.wfs, fullFilePath)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if entry != nil {
@@ -243,7 +209,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 
 func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
-	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		paginationLimit := 1024
 		remaining := dir.wfs.option.DirListingLimit
@@ -305,33 +271,14 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 
 func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
 
-	var entry *filer_pb.Entry
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: dir.Path,
-			Name:      req.Name,
-		}
-
-		glog.V(4).Infof("lookup to-be-removed entry: %v", request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
-		if err != nil {
-			// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
-			return fuse.ENOENT
-		}
-
-		entry = resp.Entry
-
-		return nil
-	})
-
+	entry, err := filer2.GetEntry(ctx, dir.wfs, path.Join(dir.Path, req.Name))
 	if err != nil {
 		return err
 	}
 
-	dir.wfs.deleteFileChunks(entry.Chunks)
+	dir.wfs.deleteFileChunks(ctx, entry.Chunks)
 
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -355,7 +302,7 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
 
 func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
 
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -379,6 +326,10 @@ func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error
 
 func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
 
+	if dir.attributes == nil {
+		return nil
+	}
+
 	glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
 	if req.Valid.Mode() {
 		dir.attributes.FileMode = uint32(req.Mode)
@@ -397,7 +348,7 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
 	}
 
 	parentDir, name := filer2.FullPath(dir.Path).DirAndName()
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: parentDir,

weed/filesys/dir_link.go

@@ -35,7 +35,7 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node,
 		},
 	}
 
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			glog.V(0).Infof("symlink %s/%s: %v", dir.Path, req.NewName, err)
 			return fuse.EIO

weed/filesys/dir_rename.go

@@ -2,118 +2,32 @@ package filesys
 
 import (
 	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
-	"math"
-	"path/filepath"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
 
 	newDir := newDirectory.(*Dir)
 
-	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
-		// find existing entry
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: dir.Path,
-			Name:      req.OldName,
+		request := &filer_pb.AtomicRenameEntryRequest{
+			OldDirectory: dir.Path,
+			OldName:      req.OldName,
+			NewDirectory: newDir.Path,
+			NewName:      req.NewName,
 		}
 
-		glog.V(4).Infof("find existing directory entry: %v", request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
+		_, err := client.AtomicRenameEntry(ctx, request)
 		if err != nil {
-			glog.V(3).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
-			return fuse.ENOENT
+			return fmt.Errorf("renaming %s/%s => %s/%s: %v", dir.Path, req.OldName, newDir.Path, req.NewName, err)
 		}
 
-		entry := resp.Entry
+		return nil
 
-		glog.V(4).Infof("found existing directory entry resp: %+v", resp)
-
-		return moveEntry(ctx, client, dir.Path, entry, newDir.Path, req.NewName)
 	})
 
 }
-
-func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParent string, entry *filer_pb.Entry, newParent, newName string) error {
-	if entry.IsDirectory {
-		currentDirPath := filepath.Join(oldParent, entry.Name)
-
-		lastFileName := ""
-		includeLastFile := false
-		limit := math.MaxInt32
-		for limit > 0 {
-			request := &filer_pb.ListEntriesRequest{
-				Directory:          currentDirPath,
-				StartFromFileName:  lastFileName,
-				InclusiveStartFrom: includeLastFile,
-				Limit:              1024,
-			}
-			glog.V(4).Infof("read directory: %v", request)
-			resp, err := client.ListEntries(ctx, request)
-			if err != nil {
-				glog.V(0).Infof("list %s: %v", oldParent, err)
-				return fuse.EIO
-			}
-			if len(resp.Entries) == 0 {
-				break
-			}
-
-			for _, item := range resp.Entries {
-				lastFileName = item.Name
-				err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
-				if err != nil {
-					return err
-				}
-				limit--
-			}
-			if len(resp.Entries) < 1024 {
-				break
-			}
-		}
-
-	}
-
-	// add to new directory
-	{
-		request := &filer_pb.CreateEntryRequest{
-			Directory: newParent,
-			Entry: &filer_pb.Entry{
-				Name:        newName,
-				IsDirectory: entry.IsDirectory,
-				Attributes:  entry.Attributes,
-				Chunks:      entry.Chunks,
-			},
-		}
-
-		glog.V(1).Infof("create new entry: %v", request)
-		if _, err := client.CreateEntry(ctx, request); err != nil {
-			glog.V(0).Infof("renaming create %s/%s: %v", newParent, newName, err)
-			return fuse.EIO
-		}
-	}
-
-	// delete old entry
-	{
-		request := &filer_pb.DeleteEntryRequest{
-			Directory:    oldParent,
-			Name:         entry.Name,
-			IsDeleteData: false,
-		}
-
-		glog.V(1).Infof("remove old entry: %v", request)
-		_, err := client.DeleteEntry(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("renaming delete %s/%s: %v", oldParent, entry.Name, err)
-			return fuse.EIO
-		}
-
-	}
-
-	return nil
-
-}

weed/filesys/dirty_page.go

@@ -4,13 +4,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"sync"
+	"github.com/chrislusf/seaweedfs/weed/security"
 )
 
 type ContinuousDirtyPages struct {
@@ -164,8 +165,9 @@ func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Contex
 func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {
 
 	var fileId, host string
+	var auth security.EncodedJwt
 
-	if err := pages.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	if err := pages.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AssignVolumeRequest{
 			Count:       1,
@@ -181,7 +183,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 			return err
 		}
 
-		fileId, host = resp.FileId, resp.Url
+		fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
 
 		return nil
 	}); err != nil {
@@ -190,7 +192,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 
 	fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
 	bufReader := bytes.NewReader(buf)
-	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, "")
+	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, auth)
 	if err != nil {
 		glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
 		return nil, fmt.Errorf("upload data: %v", err)

weed/filesys/file.go

@@ -74,10 +74,6 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		return err
 	}
 
-	if file.isOpen {
-		return nil
-	}
-
 	glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
 	if req.Valid.Size() {
 
@@ -109,7 +105,11 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		file.entry.Attributes.Mtime = req.Mtime.Unix()
 	}
 
-	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	if file.isOpen {
+		return nil
+	}
+
+	return file.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: file.dir.Path,
@@ -144,7 +144,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 			file.setEntry(entry)
 			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
 		} else {
-			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+			err := file.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 				request := &filer_pb.LookupDirectoryEntryRequest{
 					Name:      file.Name,

weed/filesys/filehandle.go

@@ -3,17 +3,18 @@ package filesys
 import (
 	"context"
 	"fmt"
+	"mime"
+	"path"
+	"time"
+
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
+	"google.golang.org/grpc"
 )
 
 type FileHandle struct {
@@ -65,72 +66,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 
 	chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)
 
-	var vids []string
-	for _, chunkView := range chunkViews {
-		vids = append(vids, volumeId(chunkView.FileId))
-	}
-
-	vid2Locations := make(map[string]*filer_pb.Locations)
-
-	err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
-		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
-			VolumeIds: vids,
-		})
-		if err != nil {
-			return err
-		}
-
-		vid2Locations = resp.LocationsMap
-
-		return nil
-	})
-
-	if err != nil {
-		glog.V(4).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err)
-		return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err)
-	}
-
-	var totalRead int64
-	var wg sync.WaitGroup
-	for _, chunkView := range chunkViews {
-		wg.Add(1)
-		go func(chunkView *filer2.ChunkView) {
-			defer wg.Done()
-
-			glog.V(4).Infof("read fh reading chunk: %+v", chunkView)
-
-			locations := vid2Locations[volumeId(chunkView.FileId)]
-			if locations == nil || len(locations.Locations) == 0 {
-				glog.V(0).Infof("failed to locate %s", chunkView.FileId)
-				err = fmt.Errorf("failed to locate %s", chunkView.FileId)
-				return
-			}
-
-			var n int64
-			n, err = util.ReadUrl(
-				fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
-				chunkView.Offset,
-				int(chunkView.Size),
-				buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)],
-				!chunkView.IsFullChunk)
-
-			if err != nil {
-
-				glog.V(0).Infof("%v/%v read http://%s/%v %v bytes: %v", fh.f.dir.Path, fh.f.Name, locations.Locations[0].Url, chunkView.FileId, n, err)
-
-				err = fmt.Errorf("failed to read http://%s/%s: %v",
-					locations.Locations[0].Url, chunkView.FileId, err)
-				return
-			}
-
-			glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView)
-			totalRead += n
-
-		}(chunkView)
-	}
-	wg.Wait()
+	totalRead, err := filer2.ReadIntoBuffer(ctx, fh.f.wfs, fh.f.fullpath(), buff, chunkViews, req.Offset)
 
 	resp.Data = buff[:totalRead]
 
@@ -153,7 +89,13 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 	resp.Size = len(req.Data)
 
 	if req.Offset == 0 {
-		fh.contentType = http.DetectContentType(req.Data)
+		// detect mime type
+		var possibleExt string
+		fh.contentType, possibleExt = mimetype.Detect(req.Data)
+		if ext := path.Ext(fh.f.Name); ext != possibleExt {
+			fh.contentType = mime.TypeByExtension(ext)
+		}
+
 		fh.dirtyMetadata = true
 	}
 
@@ -196,7 +138,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		return nil
 	}
 
-	return fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	return fh.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		if fh.f.entry.Attributes != nil {
 			fh.f.entry.Attributes.Mime = fh.contentType
@@ -220,7 +162,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
 		fh.f.entry.Chunks = chunks
 		// fh.f.entryViewCache = nil
-		fh.f.wfs.deleteFileChunks(garbages)
+		fh.f.wfs.deleteFileChunks(ctx, garbages)
 
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
@@ -230,11 +172,11 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	})
 }
 
-func deleteFileIds(ctx context.Context, client filer_pb.SeaweedFilerClient, fileIds []string) error {
+func deleteFileIds(ctx context.Context, grpcDialOption grpc.DialOption, client filer_pb.SeaweedFilerClient, fileIds []string) error {
 
 	var vids []string
 	for _, fileId := range fileIds {
-		vids = append(vids, volumeId(fileId))
+		vids = append(vids, filer2.VolumeId(fileId))
 	}
 
 	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
@@ -267,15 +209,7 @@ func deleteFileIds(ctx context.Context, client filer_pb.SeaweedFilerClient, file
 		return m, err
 	}
 
-	_, err := operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+	_, err := operation.DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
 
 	return err
 }
-
-func volumeId(fileId string) string {
-	lastCommaIndex := strings.LastIndex(fileId, ",")
-	if lastCommaIndex > 0 {
-		return fileId[:lastCommaIndex]
-	}
-	return fileId
-}

weed/filesys/wfs.go

@@ -19,6 +19,7 @@ import (
 
 type Option struct {
 	FilerGrpcAddress   string
+	GrpcDialOption     grpc.DialOption
 	FilerMountRootPath string
 	Collection         string
 	Replication        string
@@ -28,9 +29,11 @@ type Option struct {
 	DirListingLimit    int
 	EntryCacheTtl      time.Duration
 
-	MountUid  uint32
-	MountGid  uint32
-	MountMode os.FileMode
+	MountUid   uint32
+	MountGid   uint32
+	MountMode  os.FileMode
+	MountCtime time.Time
+	MountMtime time.Time
 }
 
 var _ = fs.FS(&WFS{})
@@ -46,8 +49,6 @@ type WFS struct {
 	pathToHandleLock  sync.Mutex
 	bufPool           sync.Pool
 
-	fileIdsDeletionChan chan []string
-
 	stats statsCache
 }
 type statsCache struct {
@@ -65,11 +66,8 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 				return make([]byte, option.ChunkSizeLimit)
 			},
 		},
-		fileIdsDeletionChan: make(chan []string, 32),
 	}
 
-	go wfs.loopProcessingDeletion()
-
 	return wfs
 }
 
@@ -77,12 +75,12 @@ func (wfs *WFS) Root() (fs.Node, error) {
 	return &Dir{Path: wfs.option.FilerMountRootPath, wfs: wfs}, nil
 }
 
-func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+func (wfs *WFS) WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
-	}, wfs.option.FilerGrpcAddress)
+	}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
 
 }
 
@@ -137,7 +135,7 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
 
 	if wfs.stats.lastChecked < time.Now().Unix()-20 {
 
-		err := wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		err := wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 			request := &filer_pb.StatisticsRequest{
 				Collection:  wfs.option.Collection,

weed/filesys/wfs_deletion.go

@@ -2,40 +2,10 @@ package filesys
 
 import (
 	"context"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
-func (wfs *WFS) loopProcessingDeletion() {
-
-	ticker := time.NewTicker(2 * time.Second)
-
-	wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		var fileIds []string
-		for {
-			select {
-			case fids := <-wfs.fileIdsDeletionChan:
-				fileIds = append(fileIds, fids...)
-				if len(fileIds) >= 1024 {
-					glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
-					deleteFileIds(context.Background(), client, fileIds)
-					fileIds = fileIds[:0]
-				}
-			case <-ticker.C:
-				if len(fileIds) > 0 {
-					glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
-					deleteFileIds(context.Background(), client, fileIds)
-					fileIds = fileIds[:0]
-				}
-			}
-		}
-	})
-
-}
-
-func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) {
+func (wfs *WFS) deleteFileChunks(ctx context.Context, chunks []*filer_pb.FileChunk) {
 	if len(chunks) == 0 {
 		return
 	}
@@ -45,14 +15,8 @@ func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) {
 		fileIds = append(fileIds, chunk.FileId)
 	}
 
-	var async = false
-	if async {
-		wfs.fileIdsDeletionChan <- fileIds
-		return
-	}
-
-	wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-		deleteFileIds(context.Background(), client, fileIds)
+	wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+		deleteFileIds(ctx, wfs.option.GrpcDialOption, client, fileIds)
 		return nil
 	})
 }

weed/glide.lock

@@ -1,190 +0,0 @@
-hash: 2e3a065472829938d25e879451b6d1aa43e55270e1166a9c044803ef8a3b9eb1
-updated: 2018-06-28T22:01:35.910567-07:00
-imports:
-- name: github.com/seaweedfs/fuse
-  version: 65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e
-  subpackages:
-  - fs
-  - fuseutil
-- name: github.com/boltdb/bolt
-  version: 2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8
-- name: github.com/chrislusf/raft
-  version: 5f7ddd8f479583daf05879d3d3b174aa202c8fb7
-  subpackages:
-  - protobuf
-- name: github.com/dgrijalva/jwt-go
-  version: 06ea1031745cb8b3dab3f6a236daf2b0aa468b7e
-- name: github.com/disintegration/imaging
-  version: bbcee2f5c9d5e94ca42c8b50ec847fec64a6c134
-- name: github.com/fsnotify/fsnotify
-  version: c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9
-- name: github.com/go-redis/redis
-  version: 83fb42932f6145ce52df09860384a4653d2d332a
-  subpackages:
-  - internal
-  - internal/consistenthash
-  - internal/hashtag
-  - internal/pool
-  - internal/proto
-  - internal/singleflight
-  - internal/util
-- name: github.com/go-sql-driver/mysql
-  version: d523deb1b23d913de5bdada721a6071e71283618
-- name: github.com/gocql/gocql
-  version: e06f8c1bcd787e6bf0608288b314522f08cc7848
-  subpackages:
-  - internal/lru
-  - internal/murmur
-  - internal/streams
-- name: github.com/gogo/protobuf
-  version: 30cf7ac33676b5786e78c746683f0d4cd64fa75b
-  subpackages:
-  - proto
-- name: github.com/golang/protobuf
-  version: b4deda0973fb4c70b50d226b1af49f3da59f5265
-  subpackages:
-  - proto
-  - protoc-gen-go/descriptor
-  - ptypes
-  - ptypes/any
-  - ptypes/duration
-  - ptypes/timestamp
-- name: github.com/golang/snappy
-  version: 2e65f85255dbc3072edf28d6b5b8efc472979f5a
-- name: github.com/google/btree
-  version: e89373fe6b4a7413d7acd6da1725b83ef713e6e4
-- name: github.com/gorilla/context
-  version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
-- name: github.com/gorilla/mux
-  version: e3702bed27f0d39777b0b37b664b6280e8ef8fbf
-- name: github.com/hailocab/go-hostpool
-  version: e80d13ce29ede4452c43dea11e79b9bc8a15b478
-- name: github.com/hashicorp/hcl
-  version: ef8a98b0bbce4a65b5aa4c368430a80ddc533168
-  subpackages:
-  - hcl/ast
-  - hcl/parser
-  - hcl/printer
-  - hcl/scanner
-  - hcl/strconv
-  - hcl/token
-  - json/parser
-  - json/scanner
-  - json/token
-- name: github.com/karlseguin/ccache
-  version: b425c9ca005a2050ebe723f6a0cddcb907354ab7
-- name: github.com/klauspost/crc32
-  version: cb6bfca970f6908083f26f39a79009d608efd5cd
-- name: github.com/lib/pq
-  version: 90697d60dd844d5ef6ff15135d0203f65d2f53b8
-  subpackages:
-  - oid
-- name: github.com/magiconair/properties
-  version: c2353362d570a7bfa228149c62842019201cfb71
-- name: github.com/mitchellh/mapstructure
-  version: bb74f1db0675b241733089d5a1faa5dd8b0ef57b
-- name: github.com/pelletier/go-toml
-  version: c01d1270ff3e442a8a57cddc1c92dc1138598194
-- name: github.com/rwcarlsen/goexif
-  version: 8d986c03457a2057c7b0fb0a48113f7dd48f9619
-  subpackages:
-  - exif
-  - tiff
-- name: github.com/soheilhy/cmux
-  version: e09e9389d85d8492d313d73d1469c029e710623f
-- name: github.com/spf13/afero
-  version: 787d034dfe70e44075ccc060d346146ef53270ad
-  subpackages:
-  - mem
-- name: github.com/spf13/cast
-  version: 8965335b8c7107321228e3e3702cab9832751bac
-- name: github.com/spf13/jwalterweatherman
-  version: 7c0cea34c8ece3fbeb2b27ab9b59511d360fb394
-- name: github.com/spf13/pflag
-  version: 3ebe029320b2676d667ae88da602a5f854788a8a
-- name: github.com/spf13/viper
-  version: 15738813a09db5c8e5b60a19d67d3f9bd38da3a4
-- name: github.com/syndtr/goleveldb
-  version: 0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697
-  subpackages:
-  - leveldb
-  - leveldb/cache
-  - leveldb/comparer
-  - leveldb/errors
-  - leveldb/filter
-  - leveldb/iterator
-  - leveldb/journal
-  - leveldb/memdb
-  - leveldb/opt
-  - leveldb/storage
-  - leveldb/table
-  - leveldb/util
-- name: golang.org/x/image
-  version: cc896f830cedae125428bc9fe1b0362aa91b3fb1
-  subpackages:
-  - bmp
-  - tiff
-  - tiff/lzw
-- name: golang.org/x/net
-  version: 4cb1c02c05b0e749b0365f61ae859a8e0cfceed9
-  subpackages:
-  - context
-  - http/httpguts
-  - http2
-  - http2/hpack
-  - idna
-  - internal/timeseries
-  - trace
-- name: golang.org/x/sys
-  version: 7138fd3d9dc8335c567ca206f4333fb75eb05d56
-  subpackages:
-  - unix
-- name: golang.org/x/text
-  version: 5cec4b58c438bd98288aeb248bab2c1840713d21
-  subpackages:
-  - secure/bidirule
-  - transform
-  - unicode/bidi
-  - unicode/norm
-- name: google.golang.org/appengine
-  version: b1f26356af11148e710935ed1ac8a7f5702c7612
-  subpackages:
-  - cloudsql
-- name: google.golang.org/genproto
-  version: ff3583edef7de132f219f0efc00e097cabcc0ec0
-  subpackages:
-  - googleapis/rpc/status
-- name: google.golang.org/grpc
-  version: 168a6198bcb0ef175f7dacec0b8691fc141dc9b8
-  subpackages:
-  - balancer
-  - balancer/base
-  - balancer/roundrobin
-  - codes
-  - connectivity
-  - credentials
-  - encoding
-  - encoding/proto
-  - grpclog
-  - internal
-  - internal/backoff
-  - internal/channelz
-  - internal/grpcrand
-  - keepalive
-  - metadata
-  - naming
-  - peer
-  - reflection
-  - reflection/grpc_reflection_v1alpha
-  - resolver
-  - resolver/dns
-  - resolver/passthrough
-  - stats
-  - status
-  - tap
-  - transport
-- name: gopkg.in/inf.v0
-  version: d2d2541c53f18d2a059457998ce2876cc8e67cbf
-- name: gopkg.in/yaml.v2
-  version: 5420a8b6744d3b0345ab293f6fcba19c978f1183
-testImports: []

weed/glide.yaml

@@ -1,44 +1,81 @@
 package: github.com/chrislusf/seaweedfs/weed
 import:
-- package: github.com/seaweedfs/fuse
+- package: cloud.google.com/go
   subpackages:
-  - fs
-- package: github.com/boltdb/bolt
-  version: ^1.3.1
+  - pubsub
+  - storage
+- package: github.com/Azure/azure-storage-blob-go
+  subpackages:
+  - azblob
+- package: github.com/Shopify/sarama
+- package: github.com/aws/aws-sdk-go
+  subpackages:
+  - aws
+  - aws/awserr
+  - aws/credentials
+  - aws/session
+  - service/s3
+  - service/s3/s3iface
+  - service/sqs
 - package: github.com/chrislusf/raft
+  subpackages:
+  - protobuf
 - package: github.com/dgrijalva/jwt-go
-  version: ^3.2.0
 - package: github.com/disintegration/imaging
-  version: ^1.4.1
+- package: github.com/dustin/go-humanize
+- package: github.com/gabriel-vasile/mimetype
 - package: github.com/go-redis/redis
-  version: ^6.10.2
 - package: github.com/go-sql-driver/mysql
-  version: ^1.3.0
 - package: github.com/gocql/gocql
 - package: github.com/golang/protobuf
-  version: ^1.0.0
   subpackages:
   - proto
 - package: github.com/google/btree
 - package: github.com/gorilla/mux
-  version: ^1.6.1
+- package: github.com/karlseguin/ccache
 - package: github.com/klauspost/crc32
-  version: ^1.1.0
+- package: github.com/kurin/blazer
+  subpackages:
+  - b2
 - package: github.com/lib/pq
+- package: github.com/peterh/liner
+- package: github.com/rakyll/statik
+  subpackages:
+  - fs
 - package: github.com/rwcarlsen/goexif
   subpackages:
   - exif
-- package: github.com/soheilhy/cmux
-  version: ^0.1.4
+- package: github.com/satori/go.uuid
+- package: github.com/seaweedfs/fuse
+  subpackages:
+  - fs
+- package: github.com/spf13/viper
 - package: github.com/syndtr/goleveldb
   subpackages:
   - leveldb
+  - leveldb/opt
   - leveldb/util
+- package: github.com/willf/bloom
+- package: gocloud.dev
+  subpackages:
+  - pubsub
+  - pubsub/awssnssqs
+  - pubsub/azuresb
+  - pubsub/gcppubsub
+  - pubsub/natspubsub
+  - pubsub/rabbitpubsub
 - package: golang.org/x/net
   subpackages:
   - context
-- package: google.golang.org/grpc
-  version: ^1.11.3
+- package: golang.org/x/tools
   subpackages:
+  - godoc/util
+- package: google.golang.org/api
+  subpackages:
+  - option
+- package: google.golang.org/grpc
+  subpackages:
+  - credentials
+  - keepalive
   - peer
   - reflection

weed/notification/gocdk_pub_sub/gocdk_pub_sub.go

@@ -0,0 +1,71 @@
+// Package gocdk_pub_sub supports the Go CDK (Cloud Development Kit) PubSub API,
+// which in turn supports many providers, including Amazon SNS/SQS, Azure Service Bus,
+// Google Cloud PubSub, and RabbitMQ.
+//
+// In the config, select a provider and topic using a URL. See
+// https://godoc.org/gocloud.dev/pubsub and its sub-packages for details.
+//
+// The Go CDK PubSub API does not support administrative operations like topic
+// creation. Create the topic using a UI, CLI or provider-specific API before running
+// weed.
+//
+// The Go CDK obtains credentials via environment variables and other
+// provider-specific default mechanisms. See the provider's documentation for
+// details.
+package gocdk_pub_sub
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/notification"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"gocloud.dev/pubsub"
+	_ "gocloud.dev/pubsub/awssnssqs"
+	_ "gocloud.dev/pubsub/azuresb"
+	_ "gocloud.dev/pubsub/gcppubsub"
+	_ "gocloud.dev/pubsub/natspubsub"
+	_ "gocloud.dev/pubsub/rabbitpubsub"
+)
+
+func init() {
+	notification.MessageQueues = append(notification.MessageQueues, &GoCDKPubSub{})
+}
+
+type GoCDKPubSub struct {
+	topicURL string
+	topic    *pubsub.Topic
+}
+
+func (k *GoCDKPubSub) GetName() string {
+	return "gocdk_pub_sub"
+}
+
+func (k *GoCDKPubSub) Initialize(config util.Configuration) error {
+	k.topicURL = config.GetString("topic_url")
+	glog.V(0).Infof("notification.gocdk_pub_sub.topic_url: %v", k.topicURL)
+	topic, err := pubsub.OpenTopic(context.Background(), k.topicURL)
+	if err != nil {
+		glog.Fatalf("Failed to open topic: %v", err)
+	}
+	k.topic = topic
+	return nil
+}
+
+func (k *GoCDKPubSub) SendMessage(key string, message proto.Message) error {
+	bytes, err := proto.Marshal(message)
+	if err != nil {
+		return err
+	}
+	ctx := context.Background()
+	err = k.topic.Send(ctx, &pubsub.Message{
+		Body:     bytes,
+		Metadata: map[string]string{"key": key},
+	})
+	if err != nil {
+		return fmt.Errorf("send message via Go CDK pubsub %s: %v", k.topicURL, err)
+	}
+	return nil
+}

weed/operation/assign_file_id.go

@@ -3,9 +3,11 @@ package operation
 import (
 	"context"
 	"fmt"
-	"time"
-
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"strings"
 )
 
 type VolumeAssignRequest struct {
@@ -19,14 +21,15 @@ type VolumeAssignRequest struct {
 }
 
 type AssignResult struct {
-	Fid       string `json:"fid,omitempty"`
-	Url       string `json:"url,omitempty"`
-	PublicUrl string `json:"publicUrl,omitempty"`
-	Count     uint64 `json:"count,omitempty"`
-	Error     string `json:"error,omitempty"`
+	Fid       string              `json:"fid,omitempty"`
+	Url       string              `json:"url,omitempty"`
+	PublicUrl string              `json:"publicUrl,omitempty"`
+	Count     uint64              `json:"count,omitempty"`
+	Error     string              `json:"error,omitempty"`
+	Auth      security.EncodedJwt `json:"auth,omitempty"`
 }
 
-func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
+func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
 
 	var requests []*VolumeAssignRequest
 	requests = append(requests, primaryRequest)
@@ -40,9 +43,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 			continue
 		}
 
-		lastError = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
-			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
-			defer cancel()
+		lastError = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 			req := &master_pb.AssignRequest{
 				Count:       primaryRequest.Count,
@@ -53,7 +54,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 				Rack:        primaryRequest.Rack,
 				DataNode:    primaryRequest.DataNode,
 			}
-			resp, grpcErr := masterClient.Assign(ctx, req)
+			resp, grpcErr := masterClient.Assign(context.Background(), req)
 			if grpcErr != nil {
 				return grpcErr
 			}
@@ -63,6 +64,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 			ret.Url = resp.Url
 			ret.PublicUrl = resp.PublicUrl
 			ret.Error = resp.Error
+			ret.Auth = security.EncodedJwt(resp.Auth)
 
 			return nil
 
@@ -81,3 +83,17 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 
 	return ret, lastError
 }
+
+func LookupJwt(master string, fileId string) security.EncodedJwt {
+
+	tokenStr := ""
+
+	if h, e := util.Head(fmt.Sprintf("http://%s/dir/lookup?fileId=%s", master, fileId)); e == nil {
+		bearer := h.Get("Authorization")
+		if len(bearer) > 7 && strings.ToUpper(bearer[0:6]) == "BEARER" {
+			tokenStr = bearer[7:]
+		}
+	}
+
+	return security.EncodedJwt(tokenStr)
+}

weed/operation/chunked_file.go

@@ -5,9 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"sort"
 
+	"google.golang.org/grpc"
+
 	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -53,7 +56,7 @@ func (s ChunkList) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 func LoadChunkManifest(buffer []byte, isGzipped bool) (*ChunkManifest, error) {
 	if isGzipped {
 		var err error
-		if buffer, err = UnGzipData(buffer); err != nil {
+		if buffer, err = util.UnGzipData(buffer); err != nil {
 			return nil, err
 		}
 	}
@@ -69,12 +72,12 @@ func (cm *ChunkManifest) Marshal() ([]byte, error) {
 	return json.Marshal(cm)
 }
 
-func (cm *ChunkManifest) DeleteChunks(master string) error {
+func (cm *ChunkManifest) DeleteChunks(master string, grpcDialOption grpc.DialOption) error {
 	var fileIds []string
 	for _, ci := range cm.Chunks {
 		fileIds = append(fileIds, ci.Fid)
 	}
-	results, err := DeleteFiles(master, fileIds)
+	results, err := DeleteFiles(master, grpcDialOption, fileIds)
 	if err != nil {
 		glog.V(0).Infof("delete %+v: %v", fileIds, err)
 		return fmt.Errorf("chunk delete: %v", err)
@@ -102,7 +105,10 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64) (written int64,
 	if err != nil {
 		return written, err
 	}
-	defer resp.Body.Close()
+	defer func() {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}()
 
 	switch resp.StatusCode {
 	case http.StatusRequestedRangeNotSatisfiable:

weed/operation/data_struts.go

@@ -2,6 +2,5 @@ package operation
 
 type JoinResult struct {
 	VolumeSizeLimit uint64 `json:"VolumeSizeLimit,omitempty"`
-	SecretKey       string `json:"secretKey,omitempty"`
 	Error           string `json:"error,omitempty"`
 }

weed/operation/delete_content.go

@@ -4,12 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
 	"net/http"
 	"strings"
 	"sync"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 )
 
 type DeleteResult struct {
@@ -28,17 +27,17 @@ func ParseFileId(fid string) (vid string, key_cookie string, err error) {
 }
 
 // DeleteFiles batch deletes a list of fileIds
-func DeleteFiles(master string, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
+func DeleteFiles(master string, grpcDialOption grpc.DialOption, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
 
 	lookupFunc := func(vids []string) (map[string]LookupResult, error) {
-		return LookupVolumeIds(master, vids)
+		return LookupVolumeIds(master, grpcDialOption, vids)
 	}
 
-	return DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+	return DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
 
 }
 
-func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []string) (map[string]LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
+func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
 
 	var ret []*volume_server_pb.DeleteResult
 
@@ -48,7 +47,7 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 		vid, _, err := ParseFileId(fileId)
 		if err != nil {
 			ret = append(ret, &volume_server_pb.DeleteResult{
-				FileId: vid,
+				FileId: fileId,
 				Status: http.StatusBadRequest,
 				Error:  err.Error()},
 			)
@@ -92,7 +91,7 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 		go func(server string, fidList []string) {
 			defer wg.Done()
 
-			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, fidList); deleteErr != nil {
+			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList); deleteErr != nil {
 				err = deleteErr
 			} else {
 				ret = append(ret, deleteResults...)
@@ -106,17 +105,15 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 }
 
 // DeleteFilesAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
-func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
+func DeleteFilesAtOneVolumeServer(volumeServer string, grpcDialOption grpc.DialOption, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
 
-	err = WithVolumeServerClient(volumeServer, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
-		defer cancel()
+	err = WithVolumeServerClient(volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
 
 		req := &volume_server_pb.BatchDeleteRequest{
 			FileIds: fileIds,
 		}
 
-		resp, err := volumeServerClient.BatchDelete(ctx, req)
+		resp, err := volumeServerClient.BatchDelete(context.Background(), req)
 
 		// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)
 

weed/operation/grpc_client.go

@@ -1,34 +1,30 @@
 package operation
 
 import (
+	"context"
 	"fmt"
-	"strconv"
-	"strings"
-	"sync"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"strconv"
+	"strings"
 )
 
-var (
-	grpcClients     = make(map[string]*grpc.ClientConn)
-	grpcClientsLock sync.Mutex
-)
+func WithVolumeServerClient(volumeServer string, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error {
 
-func WithVolumeServerClient(volumeServer string, fn func(volume_server_pb.VolumeServerClient) error) error {
+	ctx := context.Background()
 
 	grpcAddress, err := toVolumeServerGrpcAddress(volumeServer)
 	if err != nil {
 		return err
 	}
 
-	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := volume_server_pb.NewVolumeServerClient(grpcConnection)
 		return fn(client)
-	}, grpcAddress)
+	}, grpcAddress, grpcDialOption)
 
 }
 
@@ -42,11 +38,18 @@ func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err err
 	return fmt.Sprintf("%s:%d", volumeServer[0:sepIndex], port+10000), nil
 }
 
-func withMasterServerClient(masterServer string, fn func(masterClient master_pb.SeaweedClient) error) error {
+func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
 
-	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+	ctx := context.Background()
+
+	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer)
+	if parseErr != nil {
+		return fmt.Errorf("failed to parse master grpc %v: %v", masterServer, parseErr)
+	}
+
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := master_pb.NewSeaweedClient(grpcConnection)
 		return fn(client)
-	}, masterServer)
+	}, masterGrpcAddress, grpcDialOption)
 
 }

weed/operation/list_masters.go

@@ -1,32 +0,0 @@
-package operation
-
-import (
-	"encoding/json"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-type ClusterStatusResult struct {
-	IsLeader bool     `json:"IsLeader,omitempty"`
-	Leader   string   `json:"Leader,omitempty"`
-	Peers    []string `json:"Peers,omitempty"`
-}
-
-func ListMasters(server string) (leader string, peers []string, err error) {
-	jsonBlob, err := util.Get("http://" + server + "/cluster/status")
-	glog.V(2).Info("list masters result :", string(jsonBlob))
-	if err != nil {
-		return "", nil, err
-	}
-	var ret ClusterStatusResult
-	err = json.Unmarshal(jsonBlob, &ret)
-	if err != nil {
-		return "", nil, err
-	}
-	peers = ret.Peers
-	if ret.IsLeader {
-		peers = append(peers, ret.Leader)
-	}
-	return ret.Leader, peers, nil
-}

weed/operation/lookup.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"google.golang.org/grpc"
 	"math/rand"
 	"net/url"
 	"strings"
@@ -78,7 +79,7 @@ func LookupFileId(server string, fileId string) (fullUrl string, err error) {
 }
 
 // LookupVolumeIds find volume locations by cache and actual lookup
-func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, error) {
+func LookupVolumeIds(server string, grpcDialOption grpc.DialOption, vids []string) (map[string]LookupResult, error) {
 	ret := make(map[string]LookupResult)
 	var unknown_vids []string
 
@@ -98,14 +99,12 @@ func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, err
 
 	//only query unknown_vids
 
-	err := withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
-		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
-		defer cancel()
+	err := WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 		req := &master_pb.LookupVolumeRequest{
 			VolumeIds: unknown_vids,
 		}
-		resp, grpcErr := masterClient.LookupVolume(ctx, req)
+		resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
 		if grpcErr != nil {
 			return grpcErr
 		}

weed/operation/stats.go

@@ -2,18 +2,16 @@ package operation
 
 import (
 	"context"
-	"time"
+	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
 
-func Statistics(server string, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
+func Statistics(server string, grpcDialOption grpc.DialOption, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
 
-	err = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
-		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
-		defer cancel()
+	err = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
-		grpcResponse, grpcErr := masterClient.Statistics(ctx, req)
+		grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req)
 		if grpcErr != nil {
 			return grpcErr
 		}

weed/operation/submit.go

@@ -2,6 +2,7 @@ package operation
 
 import (
 	"bytes"
+	"google.golang.org/grpc"
 	"io"
 	"mime"
 	"net/url"
@@ -36,10 +37,8 @@ type SubmitResult struct {
 	Error    string `json:"error,omitempty"`
 }
 
-func SubmitFiles(master string, files []FilePart,
-	replication string, collection string, dataCenter string, ttl string, maxMB int,
-	secret security.Secret,
-) ([]SubmitResult, error) {
+func SubmitFiles(master string, grpcDialOption grpc.DialOption, files []FilePart,
+	replication string, collection string, dataCenter string, ttl string, maxMB int) ([]SubmitResult, error) {
 	results := make([]SubmitResult, len(files))
 	for index, file := range files {
 		results[index].FileName = file.FileName
@@ -51,7 +50,7 @@ func SubmitFiles(master string, files []FilePart,
 		DataCenter:  dataCenter,
 		Ttl:         ttl,
 	}
-	ret, err := Assign(master, ar)
+	ret, err := Assign(master, grpcDialOption, ar)
 	if err != nil {
 		for index, _ := range files {
 			results[index].Error = err.Error()
@@ -67,7 +66,7 @@ func SubmitFiles(master string, files []FilePart,
 		file.Replication = replication
 		file.Collection = collection
 		file.DataCenter = dataCenter
-		results[index].Size, err = file.Upload(maxMB, master, secret)
+		results[index].Size, err = file.Upload(maxMB, master, ret.Auth, grpcDialOption)
 		if err != nil {
 			results[index].Error = err.Error()
 		}
@@ -110,8 +109,7 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
 	return ret, nil
 }
 
-func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (retSize uint32, err error) {
-	jwt := security.GenJwt(secret, fi.Fid)
+func (fi FilePart) Upload(maxMB int, master string, jwt security.EncodedJwt, grpcDialOption grpc.DialOption) (retSize uint32, err error) {
 	fileUrl := "http://" + fi.Server + "/" + fi.Fid
 	if fi.ModTime != 0 {
 		fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
@@ -139,7 +137,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 				Collection:  fi.Collection,
 				Ttl:         fi.Ttl,
 			}
-			ret, err = Assign(master, ar)
+			ret, err = Assign(master, grpcDialOption, ar)
 			if err != nil {
 				return
 			}
@@ -152,10 +150,10 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 					Collection:  fi.Collection,
 					Ttl:         fi.Ttl,
 				}
-				ret, err = Assign(master, ar)
+				ret, err = Assign(master, grpcDialOption, ar)
 				if err != nil {
 					// delete all uploaded chunks
-					cm.DeleteChunks(master)
+					cm.DeleteChunks(master, grpcDialOption)
 					return
 				}
 				id = ret.Fid
@@ -170,10 +168,10 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 				baseName+"-"+strconv.FormatInt(i+1, 10),
 				io.LimitReader(fi.Reader, chunkSize),
 				master, fileUrl,
-				jwt)
+				ret.Auth)
 			if e != nil {
 				// delete all uploaded chunks
-				cm.DeleteChunks(master)
+				cm.DeleteChunks(master, grpcDialOption)
 				return 0, e
 			}
 			cm.Chunks = append(cm.Chunks,
@@ -188,7 +186,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 		err = upload_chunked_file_manifest(fileUrl, &cm, jwt)
 		if err != nil {
 			// delete all uploaded chunks
-			cm.DeleteChunks(master)
+			cm.DeleteChunks(master, grpcDialOption)
 		}
 	} else {
 		ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt)

weed/operation/sync_volume.go

@@ -2,63 +2,19 @@ package operation
 
 import (
 	"context"
-	"fmt"
-	"io"
-	"time"
-
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
-	. "github.com/chrislusf/seaweedfs/weed/storage/types"
-	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
 )
 
-func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
+func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
 
-	WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
-		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
-		defer cancel()
+	WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 
-		resp, err = client.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{
-			VolumdId: vid,
+		resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
+			VolumeId: vid,
 		})
 		return nil
 	})
 
 	return
 }
-
-func GetVolumeIdxEntries(server string, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
-
-	return WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
-		stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
-			VolumdId: vid,
-		})
-		if err != nil {
-			return err
-		}
-
-		var indexFileContent []byte
-
-		for {
-			resp, err := stream.Recv()
-			if err == io.EOF {
-				break
-			}
-			if err != nil {
-				return fmt.Errorf("read index entries: %v", err)
-			}
-			indexFileContent = append(indexFileContent, resp.IndexFileContent...)
-		}
-
-		dataSize := len(indexFileContent)
-
-		for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
-			line := indexFileContent[idx : idx+NeedleEntrySize]
-			key := BytesToNeedleId(line[:NeedleIdSize])
-			offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
-			size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
-			eachEntryFn(key, offset, size)
-		}
-
-		return nil
-	})
-}

weed/operation/tail_volume.go

@@ -0,0 +1,82 @@
+package operation
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"google.golang.org/grpc"
+)
+
+func TailVolume(master string, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, timeoutSeconds int, fn func(n *needle.Needle) error) error {
+	// find volume location, replication, ttl info
+	lookup, err := Lookup(master, vid.String())
+	if err != nil {
+		return fmt.Errorf("look up volume %d: %v", vid, err)
+	}
+	if len(lookup.Locations) == 0 {
+		return fmt.Errorf("unable to locate volume %d", vid)
+	}
+
+	volumeServer := lookup.Locations[0].Url
+
+	return TailVolumeFromSource(volumeServer, grpcDialOption, vid, sinceNs, timeoutSeconds, fn)
+}
+
+func TailVolumeFromSource(volumeServer string, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, idleTimeoutSeconds int, fn func(n *needle.Needle) error) error {
+	return WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+
+		stream, err := client.VolumeTailSender(context.Background(), &volume_server_pb.VolumeTailSenderRequest{
+			VolumeId:           uint32(vid),
+			SinceNs:            sinceNs,
+			IdleTimeoutSeconds: uint32(idleTimeoutSeconds),
+		})
+		if err != nil {
+			return err
+		}
+
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			needleHeader := resp.NeedleHeader
+			needleBody := resp.NeedleBody
+
+			if len(needleHeader) == 0 {
+				continue
+			}
+
+			for !resp.IsLastChunk {
+				resp, recvErr = stream.Recv()
+				if recvErr != nil {
+					if recvErr == io.EOF {
+						break
+					} else {
+						return recvErr
+					}
+				}
+				needleBody = append(needleBody, resp.NeedleBody...)
+			}
+
+			n := new(needle.Needle)
+			n.ParseNeedleHeader(needleHeader)
+			n.ReadNeedleBodyBytes(needleBody, needle.CurrentVersion)
+
+			err = fn(n)
+
+			if err != nil {
+				return err
+			}
+
+		}
+		return nil
+	})
+}

weed/operation/upload_content.go

@@ -2,6 +2,8 @@ package operation
 
 import (
 	"bytes"
+	"compress/flate"
+	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -13,10 +15,10 @@ import (
 	"net/textproto"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type UploadResult struct {
@@ -31,21 +33,50 @@ var (
 )
 
 func init() {
-	client = &http.Client{
-		Transport: &http.Transport{MaxIdleConnsPerHost: 1024},
-		Timeout:   5 * time.Second,
-	}
+	client = &http.Client{Transport: &http.Transport{
+		MaxIdleConnsPerHost: 1024,
+	}}
 }
 
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
-// Upload sends a POST request to a volume server to upload the content
-func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
-	return upload_content(uploadUrl, func(w io.Writer) (err error) {
-		_, err = io.Copy(w, reader)
-		return
-	}, filename, isGzipped, mtype, pairMap, jwt)
+// Upload sends a POST request to a volume server to upload the content with adjustable compression level
+func UploadWithLocalCompressionLevel(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) {
+	if compressionLevel < 1 {
+		compressionLevel = 1
+	}
+	if compressionLevel > 9 {
+		compressionLevel = 9
+	}
+	return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, compressionLevel, jwt)
 }
+
+// Upload sends a POST request to a volume server to upload the content with fast compression
+func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
+	return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt)
+}
+
+func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) {
+	contentIsGzipped := isGzipped
+	shouldGzipNow := false
+	if !isGzipped {
+		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
+			shouldGzipNow = true
+			contentIsGzipped = true
+		}
+	}
+	return upload_content(uploadUrl, func(w io.Writer) (err error) {
+		if shouldGzipNow {
+			gzWriter, _ := gzip.NewWriterLevel(w, compression)
+			_, err = io.Copy(gzWriter, reader)
+			gzWriter.Close()
+		} else {
+			_, err = io.Copy(w, reader)
+		}
+		return
+	}, filename, contentIsGzipped, mtype, pairMap, jwt)
+}
+
 func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")
 	body_writer := multipart.NewWriter(body_buf)
@@ -60,9 +91,6 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	if isGzipped {
 		h.Set("Content-Encoding", "gzip")
 	}
-	if jwt != "" {
-		h.Set("Authorization", "BEARER "+string(jwt))
-	}
 
 	file_writer, cp_err := body_writer.CreatePart(h)
 	if cp_err != nil {
@@ -88,18 +116,15 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	for k, v := range pairMap {
 		req.Header.Set(k, v)
 	}
+	if jwt != "" {
+		req.Header.Set("Authorization", "BEARER "+string(jwt))
+	}
 	resp, post_err := client.Do(req)
 	if post_err != nil {
 		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
 		return nil, post_err
 	}
 	defer resp.Body.Close()
-
-	if resp.StatusCode < http.StatusOK ||
-		resp.StatusCode > http.StatusIMUsed {
-		return nil, errors.New(http.StatusText(resp.StatusCode))
-	}
-
 	etag := getEtag(resp)
 	resp_body, ra_err := ioutil.ReadAll(resp.Body)
 	if ra_err != nil {

weed/pb/filer.proto

@@ -24,6 +24,9 @@ service SeaweedFiler {
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
+    rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) {
+    }
+
     rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
     }
 
@@ -69,19 +72,33 @@ message Entry {
     map<string, bytes> extended = 5;
 }
 
+message FullEntry {
+    string dir = 1;
+    Entry entry = 2;
+}
+
 message EventNotification {
     Entry old_entry = 1;
     Entry new_entry = 2;
     bool delete_chunks = 3;
+    string new_parent_path = 4;
 }
 
 message FileChunk {
-    string file_id = 1;
+    string file_id = 1; // to be deprecated
     int64 offset = 2;
     uint64 size = 3;
     int64 mtime = 4;
     string e_tag = 5;
-    string source_file_id = 6;
+    string source_file_id = 6; // to be deprecated
+    FileId fid = 7;
+    FileId source_fid = 8;
+}
+
+message FileId {
+    uint32 volume_id = 1;
+    uint64 file_key = 2;
+    fixed32 cookie = 3;
 }
 
 message FuseAttributes {
@@ -126,6 +143,16 @@ message DeleteEntryRequest {
 message DeleteEntryResponse {
 }
 
+message AtomicRenameEntryRequest {
+    string old_directory = 1;
+    string old_name = 2;
+    string new_directory = 3;
+    string new_name = 4;
+}
+
+message AtomicRenameEntryResponse {
+}
+
 message AssignVolumeRequest {
     int32 count = 1;
     string collection = 2;
@@ -139,6 +166,7 @@ message AssignVolumeResponse {
     string url = 2;
     string public_url = 3;
     int32 count = 4;
+    string auth = 5;
 }
 
 message LookupVolumeRequest {

weed/pb/filer_pb/filer.pb.go

@@ -14,8 +14,10 @@ It has these top-level messages:
 	ListEntriesRequest
 	ListEntriesResponse
 	Entry
+	FullEntry
 	EventNotification
 	FileChunk
+	FileId
 	FuseAttributes
 	CreateEntryRequest
 	CreateEntryResponse
@@ -23,6 +25,8 @@ It has these top-level messages:
 	UpdateEntryResponse
 	DeleteEntryRequest
 	DeleteEntryResponse
+	AtomicRenameEntryRequest
+	AtomicRenameEntryResponse
 	AssignVolumeRequest
 	AssignVolumeResponse
 	LookupVolumeRequest
@@ -208,16 +212,41 @@ func (m *Entry) GetExtended() map[string][]byte {
 	return nil
 }
 
+type FullEntry struct {
+	Dir   string `protobuf:"bytes,1,opt,name=dir" json:"dir,omitempty"`
+	Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
+}
+
+func (m *FullEntry) Reset()                    { *m = FullEntry{} }
+func (m *FullEntry) String() string            { return proto.CompactTextString(m) }
+func (*FullEntry) ProtoMessage()               {}
+func (*FullEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+func (m *FullEntry) GetDir() string {
+	if m != nil {
+		return m.Dir
+	}
+	return ""
+}
+
+func (m *FullEntry) GetEntry() *Entry {
+	if m != nil {
+		return m.Entry
+	}
+	return nil
+}
+
 type EventNotification struct {
-	OldEntry     *Entry `protobuf:"bytes,1,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"`
-	NewEntry     *Entry `protobuf:"bytes,2,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"`
-	DeleteChunks bool   `protobuf:"varint,3,opt,name=delete_chunks,json=deleteChunks" json:"delete_chunks,omitempty"`
+	OldEntry      *Entry `protobuf:"bytes,1,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"`
+	NewEntry      *Entry `protobuf:"bytes,2,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"`
+	DeleteChunks  bool   `protobuf:"varint,3,opt,name=delete_chunks,json=deleteChunks" json:"delete_chunks,omitempty"`
+	NewParentPath string `protobuf:"bytes,4,opt,name=new_parent_path,json=newParentPath" json:"new_parent_path,omitempty"`
 }
 
 func (m *EventNotification) Reset()                    { *m = EventNotification{} }
 func (m *EventNotification) String() string            { return proto.CompactTextString(m) }
 func (*EventNotification) ProtoMessage()               {}
-func (*EventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+func (*EventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
 
 func (m *EventNotification) GetOldEntry() *Entry {
 	if m != nil {
@@ -240,19 +269,28 @@ func (m *EventNotification) GetDeleteChunks() bool {
 	return false
 }
 
+func (m *EventNotification) GetNewParentPath() string {
+	if m != nil {
+		return m.NewParentPath
+	}
+	return ""
+}
+
 type FileChunk struct {
-	FileId       string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
-	Offset       int64  `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
-	Size         uint64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
-	Mtime        int64  `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
-	ETag         string `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
-	SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
+	FileId       string  `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
+	Offset       int64   `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
+	Size         uint64  `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
+	Mtime        int64   `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
+	ETag         string  `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
+	SourceFileId string  `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
+	Fid          *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"`
+	SourceFid    *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"`
 }
 
 func (m *FileChunk) Reset()                    { *m = FileChunk{} }
 func (m *FileChunk) String() string            { return proto.CompactTextString(m) }
 func (*FileChunk) ProtoMessage()               {}
-func (*FileChunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+func (*FileChunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
 
 func (m *FileChunk) GetFileId() string {
 	if m != nil {
@@ -296,6 +334,52 @@ func (m *FileChunk) GetSourceFileId() string {
 	return ""
 }
 
+func (m *FileChunk) GetFid() *FileId {
+	if m != nil {
+		return m.Fid
+	}
+	return nil
+}
+
+func (m *FileChunk) GetSourceFid() *FileId {
+	if m != nil {
+		return m.SourceFid
+	}
+	return nil
+}
+
+type FileId struct {
+	VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+	FileKey  uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"`
+	Cookie   uint32 `protobuf:"fixed32,3,opt,name=cookie" json:"cookie,omitempty"`
+}
+
+func (m *FileId) Reset()                    { *m = FileId{} }
+func (m *FileId) String() string            { return proto.CompactTextString(m) }
+func (*FileId) ProtoMessage()               {}
+func (*FileId) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *FileId) GetVolumeId() uint32 {
+	if m != nil {
+		return m.VolumeId
+	}
+	return 0
+}
+
+func (m *FileId) GetFileKey() uint64 {
+	if m != nil {
+		return m.FileKey
+	}
+	return 0
+}
+
+func (m *FileId) GetCookie() uint32 {
+	if m != nil {
+		return m.Cookie
+	}
+	return 0
+}
+
 type FuseAttributes struct {
 	FileSize      uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
 	Mtime         int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
@@ -315,7 +399,7 @@ type FuseAttributes struct {
 func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
 func (m *FuseAttributes) String() string            { return proto.CompactTextString(m) }
 func (*FuseAttributes) ProtoMessage()               {}
-func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
 
 func (m *FuseAttributes) GetFileSize() uint64 {
 	if m != nil {
@@ -416,7 +500,7 @@ type CreateEntryRequest struct {
 func (m *CreateEntryRequest) Reset()                    { *m = CreateEntryRequest{} }
 func (m *CreateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryRequest) ProtoMessage()               {}
-func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
 
 func (m *CreateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -438,7 +522,7 @@ type CreateEntryResponse struct {
 func (m *CreateEntryResponse) Reset()                    { *m = CreateEntryResponse{} }
 func (m *CreateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryResponse) ProtoMessage()               {}
-func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
 
 type UpdateEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@@ -448,7 +532,7 @@ type UpdateEntryRequest struct {
 func (m *UpdateEntryRequest) Reset()                    { *m = UpdateEntryRequest{} }
 func (m *UpdateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryRequest) ProtoMessage()               {}
-func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
 
 func (m *UpdateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -470,7 +554,7 @@ type UpdateEntryResponse struct {
 func (m *UpdateEntryResponse) Reset()                    { *m = UpdateEntryResponse{} }
 func (m *UpdateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryResponse) ProtoMessage()               {}
-func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
 
 type DeleteEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@@ -483,7 +567,7 @@ type DeleteEntryRequest struct {
 func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
 func (m *DeleteEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryRequest) ProtoMessage()               {}
-func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
 
 func (m *DeleteEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -519,7 +603,55 @@ type DeleteEntryResponse struct {
 func (m *DeleteEntryResponse) Reset()                    { *m = DeleteEntryResponse{} }
 func (m *DeleteEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryResponse) ProtoMessage()               {}
-func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+
+type AtomicRenameEntryRequest struct {
+	OldDirectory string `protobuf:"bytes,1,opt,name=old_directory,json=oldDirectory" json:"old_directory,omitempty"`
+	OldName      string `protobuf:"bytes,2,opt,name=old_name,json=oldName" json:"old_name,omitempty"`
+	NewDirectory string `protobuf:"bytes,3,opt,name=new_directory,json=newDirectory" json:"new_directory,omitempty"`
+	NewName      string `protobuf:"bytes,4,opt,name=new_name,json=newName" json:"new_name,omitempty"`
+}
+
+func (m *AtomicRenameEntryRequest) Reset()                    { *m = AtomicRenameEntryRequest{} }
+func (m *AtomicRenameEntryRequest) String() string            { return proto.CompactTextString(m) }
+func (*AtomicRenameEntryRequest) ProtoMessage()               {}
+func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+
+func (m *AtomicRenameEntryRequest) GetOldDirectory() string {
+	if m != nil {
+		return m.OldDirectory
+	}
+	return ""
+}
+
+func (m *AtomicRenameEntryRequest) GetOldName() string {
+	if m != nil {
+		return m.OldName
+	}
+	return ""
+}
+
+func (m *AtomicRenameEntryRequest) GetNewDirectory() string {
+	if m != nil {
+		return m.NewDirectory
+	}
+	return ""
+}
+
+func (m *AtomicRenameEntryRequest) GetNewName() string {
+	if m != nil {
+		return m.NewName
+	}
+	return ""
+}
+
+type AtomicRenameEntryResponse struct {
+}
+
+func (m *AtomicRenameEntryResponse) Reset()                    { *m = AtomicRenameEntryResponse{} }
+func (m *AtomicRenameEntryResponse) String() string            { return proto.CompactTextString(m) }
+func (*AtomicRenameEntryResponse) ProtoMessage()               {}
+func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
 
 type AssignVolumeRequest struct {
 	Count       int32  `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
@@ -532,7 +664,7 @@ type AssignVolumeRequest struct {
 func (m *AssignVolumeRequest) Reset()                    { *m = AssignVolumeRequest{} }
 func (m *AssignVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeRequest) ProtoMessage()               {}
-func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
 
 func (m *AssignVolumeRequest) GetCount() int32 {
 	if m != nil {
@@ -574,12 +706,13 @@ type AssignVolumeResponse struct {
 	Url       string `protobuf:"bytes,2,opt,name=url" json:"url,omitempty"`
 	PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"`
 	Count     int32  `protobuf:"varint,4,opt,name=count" json:"count,omitempty"`
+	Auth      string `protobuf:"bytes,5,opt,name=auth" json:"auth,omitempty"`
 }
 
 func (m *AssignVolumeResponse) Reset()                    { *m = AssignVolumeResponse{} }
 func (m *AssignVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeResponse) ProtoMessage()               {}
-func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
 
 func (m *AssignVolumeResponse) GetFileId() string {
 	if m != nil {
@@ -609,6 +742,13 @@ func (m *AssignVolumeResponse) GetCount() int32 {
 	return 0
 }
 
+func (m *AssignVolumeResponse) GetAuth() string {
+	if m != nil {
+		return m.Auth
+	}
+	return ""
+}
+
 type LookupVolumeRequest struct {
 	VolumeIds []string `protobuf:"bytes,1,rep,name=volume_ids,json=volumeIds" json:"volume_ids,omitempty"`
 }
@@ -616,7 +756,7 @@ type LookupVolumeRequest struct {
 func (m *LookupVolumeRequest) Reset()                    { *m = LookupVolumeRequest{} }
 func (m *LookupVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeRequest) ProtoMessage()               {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
 
 func (m *LookupVolumeRequest) GetVolumeIds() []string {
 	if m != nil {
@@ -632,7 +772,7 @@ type Locations struct {
 func (m *Locations) Reset()                    { *m = Locations{} }
 func (m *Locations) String() string            { return proto.CompactTextString(m) }
 func (*Locations) ProtoMessage()               {}
-func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
 
 func (m *Locations) GetLocations() []*Location {
 	if m != nil {
@@ -649,7 +789,7 @@ type Location struct {
 func (m *Location) Reset()                    { *m = Location{} }
 func (m *Location) String() string            { return proto.CompactTextString(m) }
 func (*Location) ProtoMessage()               {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
 
 func (m *Location) GetUrl() string {
 	if m != nil {
@@ -672,7 +812,7 @@ type LookupVolumeResponse struct {
 func (m *LookupVolumeResponse) Reset()                    { *m = LookupVolumeResponse{} }
 func (m *LookupVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse) ProtoMessage()               {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
 
 func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations {
 	if m != nil {
@@ -688,7 +828,7 @@ type DeleteCollectionRequest struct {
 func (m *DeleteCollectionRequest) Reset()                    { *m = DeleteCollectionRequest{} }
 func (m *DeleteCollectionRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionRequest) ProtoMessage()               {}
-func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
 
 func (m *DeleteCollectionRequest) GetCollection() string {
 	if m != nil {
@@ -703,7 +843,7 @@ type DeleteCollectionResponse struct {
 func (m *DeleteCollectionResponse) Reset()                    { *m = DeleteCollectionResponse{} }
 func (m *DeleteCollectionResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionResponse) ProtoMessage()               {}
-func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
 type StatisticsRequest struct {
 	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
@@ -714,7 +854,7 @@ type StatisticsRequest struct {
 func (m *StatisticsRequest) Reset()                    { *m = StatisticsRequest{} }
 func (m *StatisticsRequest) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsRequest) ProtoMessage()               {}
-func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
 
 func (m *StatisticsRequest) GetReplication() string {
 	if m != nil {
@@ -749,7 +889,7 @@ type StatisticsResponse struct {
 func (m *StatisticsResponse) Reset()                    { *m = StatisticsResponse{} }
 func (m *StatisticsResponse) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsResponse) ProtoMessage()               {}
-func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
 
 func (m *StatisticsResponse) GetReplication() string {
 	if m != nil {
@@ -799,8 +939,10 @@ func init() {
 	proto.RegisterType((*ListEntriesRequest)(nil), "filer_pb.ListEntriesRequest")
 	proto.RegisterType((*ListEntriesResponse)(nil), "filer_pb.ListEntriesResponse")
 	proto.RegisterType((*Entry)(nil), "filer_pb.Entry")
+	proto.RegisterType((*FullEntry)(nil), "filer_pb.FullEntry")
 	proto.RegisterType((*EventNotification)(nil), "filer_pb.EventNotification")
 	proto.RegisterType((*FileChunk)(nil), "filer_pb.FileChunk")
+	proto.RegisterType((*FileId)(nil), "filer_pb.FileId")
 	proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes")
 	proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest")
 	proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse")
@@ -808,6 +950,8 @@ func init() {
 	proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse")
 	proto.RegisterType((*DeleteEntryRequest)(nil), "filer_pb.DeleteEntryRequest")
 	proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse")
+	proto.RegisterType((*AtomicRenameEntryRequest)(nil), "filer_pb.AtomicRenameEntryRequest")
+	proto.RegisterType((*AtomicRenameEntryResponse)(nil), "filer_pb.AtomicRenameEntryResponse")
 	proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest")
 	proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse")
 	proto.RegisterType((*LookupVolumeRequest)(nil), "filer_pb.LookupVolumeRequest")
@@ -836,6 +980,7 @@ type SeaweedFilerClient interface {
 	CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
 	UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
 	DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
+	AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error)
 	AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
 	LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error)
 	DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
@@ -895,6 +1040,15 @@ func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryReq
 	return out, nil
 }
 
+func (c *seaweedFilerClient) AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error) {
+	out := new(AtomicRenameEntryResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AtomicRenameEntry", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) {
 	out := new(AssignVolumeResponse)
 	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AssignVolume", in, out, c.cc, opts...)
@@ -939,6 +1093,7 @@ type SeaweedFilerServer interface {
 	CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
 	UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
 	DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
+	AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error)
 	AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
 	LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error)
 	DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
@@ -1039,6 +1194,24 @@ func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedFiler_AtomicRenameEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(AtomicRenameEntryRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/AtomicRenameEntry",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).AtomicRenameEntry(ctx, req.(*AtomicRenameEntryRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _SeaweedFiler_AssignVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(AssignVolumeRequest)
 	if err := dec(in); err != nil {
@@ -1135,6 +1308,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			MethodName: "DeleteEntry",
 			Handler:    _SeaweedFiler_DeleteEntry_Handler,
 		},
+		{
+			MethodName: "AtomicRenameEntry",
+			Handler:    _SeaweedFiler_AtomicRenameEntry_Handler,
+		},
 		{
 			MethodName: "AssignVolume",
 			Handler:    _SeaweedFiler_AssignVolume_Handler,
@@ -1159,86 +1336,100 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1291 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0x4d, 0x8f, 0xdc, 0x44,
-	0x13, 0x8e, 0xe7, 0x2b, 0xe3, 0x9a, 0x99, 0xbc, 0xbb, 0x3d, 0xfb, 0x12, 0x6b, 0xb2, 0x1b, 0x26,
-	0x86, 0xa0, 0x8d, 0x88, 0x46, 0x51, 0xe0, 0x90, 0x10, 0x21, 0x91, 0x6c, 0x36, 0x52, 0xa4, 0x4d,
-	0x82, 0xbc, 0x09, 0x12, 0xe2, 0x60, 0x79, 0xed, 0x9e, 0xa1, 0xb5, 0x1e, 0x7b, 0x70, 0xb7, 0x37,
-	0x09, 0x7f, 0x82, 0x0b, 0x57, 0x0e, 0x9c, 0xf8, 0x17, 0x5c, 0xf8, 0x3f, 0xdc, 0xb9, 0xa1, 0xae,
-	0x6e, 0x7b, 0xda, 0x63, 0xef, 0x06, 0x84, 0x72, 0xeb, 0x7e, 0xaa, 0xba, 0xbe, 0xfa, 0xe9, 0x2a,
-	0x1b, 0x06, 0x73, 0x16, 0xd3, 0x6c, 0xb6, 0xca, 0x52, 0x91, 0x92, 0x3e, 0x6e, 0xfc, 0xd5, 0x89,
-	0xfb, 0x02, 0xae, 0x1d, 0xa5, 0xe9, 0x69, 0xbe, 0x7a, 0xcc, 0x32, 0x1a, 0x8a, 0x34, 0x7b, 0x7b,
-	0x98, 0x88, 0xec, 0xad, 0x47, 0x7f, 0xc8, 0x29, 0x17, 0x64, 0x17, 0xec, 0xa8, 0x10, 0x38, 0xd6,
-	0xd4, 0xda, 0xb7, 0xbd, 0x35, 0x40, 0x08, 0x74, 0x92, 0x60, 0x49, 0x9d, 0x16, 0x0a, 0x70, 0xed,
-	0x1e, 0xc2, 0x6e, 0xb3, 0x41, 0xbe, 0x4a, 0x13, 0x4e, 0xc9, 0x4d, 0xe8, 0x52, 0x09, 0xa0, 0xb5,
-	0xc1, 0xdd, 0xff, 0xcd, 0x8a, 0x50, 0x66, 0x4a, 0x4f, 0x49, 0xdd, 0xdf, 0x2d, 0x20, 0x47, 0x8c,
-	0x0b, 0x09, 0x32, 0xca, 0xff, 0x59, 0x3c, 0x1f, 0x40, 0x6f, 0x95, 0xd1, 0x39, 0x7b, 0xa3, 0x23,
-	0xd2, 0x3b, 0x72, 0x1b, 0xb6, 0xb9, 0x08, 0x32, 0xf1, 0x24, 0x4b, 0x97, 0x4f, 0x58, 0x4c, 0x9f,
-	0xcb, 0xa0, 0xdb, 0xa8, 0x52, 0x17, 0x90, 0x19, 0x10, 0x96, 0x84, 0x71, 0xce, 0xd9, 0x19, 0x3d,
-	0x2e, 0xa4, 0x4e, 0x67, 0x6a, 0xed, 0xf7, 0xbd, 0x06, 0x09, 0xd9, 0x81, 0x6e, 0xcc, 0x96, 0x4c,
-	0x38, 0xdd, 0xa9, 0xb5, 0x3f, 0xf2, 0xd4, 0xc6, 0xfd, 0x0a, 0xc6, 0x95, 0xf8, 0x75, 0xfa, 0xb7,
-	0xe0, 0x32, 0x55, 0x90, 0x63, 0x4d, 0xdb, 0x4d, 0x05, 0x28, 0xe4, 0xee, 0x2f, 0x2d, 0xe8, 0x22,
-	0x54, 0xd6, 0xd9, 0x5a, 0xd7, 0x99, 0xdc, 0x80, 0x21, 0xe3, 0xfe, 0xba, 0x18, 0x2d, 0x8c, 0x6f,
-	0xc0, 0x78, 0x59, 0x77, 0xf2, 0x29, 0xf4, 0xc2, 0xef, 0xf3, 0xe4, 0x94, 0x3b, 0x6d, 0x74, 0x35,
-	0x5e, 0xbb, 0x92, 0xc9, 0x1e, 0x48, 0x99, 0xa7, 0x55, 0xc8, 0x3d, 0x80, 0x40, 0x88, 0x8c, 0x9d,
-	0xe4, 0x82, 0x72, 0xcc, 0x76, 0x70, 0xd7, 0x31, 0x0e, 0xe4, 0x9c, 0x3e, 0x2c, 0xe5, 0x9e, 0xa1,
-	0x4b, 0xee, 0x43, 0x9f, 0xbe, 0x11, 0x34, 0x89, 0x68, 0xe4, 0x74, 0xd1, 0xd1, 0xde, 0x46, 0x4e,
-	0xb3, 0x43, 0x2d, 0x57, 0x19, 0x96, 0xea, 0x93, 0x07, 0x30, 0xaa, 0x88, 0xc8, 0x16, 0xb4, 0x4f,
-	0x69, 0x71, 0xb3, 0x72, 0x29, 0xab, 0x7b, 0x16, 0xc4, 0xb9, 0x22, 0xd9, 0xd0, 0x53, 0x9b, 0x2f,
-	0x5a, 0xf7, 0x2c, 0xf7, 0x67, 0x0b, 0xb6, 0x0f, 0xcf, 0x68, 0x22, 0x9e, 0xa7, 0x82, 0xcd, 0x59,
-	0x18, 0x08, 0x96, 0x26, 0xe4, 0x36, 0xd8, 0x69, 0x1c, 0xf9, 0x17, 0x72, 0xac, 0x9f, 0xc6, 0xda,
-	0xdf, 0x6d, 0xb0, 0x13, 0xfa, 0x5a, 0x6b, 0xb7, 0xce, 0xd1, 0x4e, 0xe8, 0x6b, 0xa5, 0xfd, 0x11,
-	0x8c, 0x22, 0x1a, 0x53, 0x41, 0xfd, 0xb2, 0xae, 0xb2, 0xe8, 0x43, 0x05, 0x62, 0x3d, 0xb9, 0xfb,
-	0xab, 0x05, 0x76, 0x59, 0x5e, 0x72, 0x15, 0x2e, 0x4b, 0x73, 0x3e, 0x8b, 0x74, 0x52, 0x3d, 0xb9,
-	0x7d, 0x1a, 0x49, 0xae, 0xa6, 0xf3, 0x39, 0xa7, 0x02, 0xdd, 0xb6, 0x3d, 0xbd, 0x93, 0x77, 0xcd,
-	0xd9, 0x8f, 0x8a, 0x9e, 0x1d, 0x0f, 0xd7, 0xb2, 0x06, 0x4b, 0xc1, 0x96, 0x14, 0xaf, 0xa5, 0xed,
-	0xa9, 0x0d, 0x19, 0x43, 0x97, 0xfa, 0x22, 0x58, 0x20, 0xef, 0x6c, 0xaf, 0x43, 0x5f, 0x06, 0x0b,
-	0xf2, 0x31, 0x5c, 0xe1, 0x69, 0x9e, 0x85, 0xd4, 0x2f, 0xdc, 0xf6, 0x50, 0x3a, 0x54, 0xe8, 0x13,
-	0x74, 0xee, 0xfe, 0xd9, 0x82, 0x2b, 0xd5, 0x1b, 0x25, 0xd7, 0xc0, 0xc6, 0x13, 0xe8, 0xdc, 0x42,
-	0xe7, 0xd8, 0x25, 0x8e, 0x2b, 0x01, 0xb4, 0xcc, 0x00, 0x8a, 0x23, 0xcb, 0x34, 0x52, 0xf1, 0x8e,
-	0xd4, 0x91, 0x67, 0x69, 0x44, 0xe5, 0x4d, 0xe6, 0x2c, 0xc2, 0x88, 0x47, 0x9e, 0x5c, 0x4a, 0x64,
-	0xc1, 0x22, 0xfd, 0x4a, 0xe4, 0x52, 0xd6, 0x20, 0xcc, 0xd0, 0x6e, 0x4f, 0xd5, 0x40, 0xed, 0x64,
-	0x0d, 0x96, 0x12, 0xbd, 0xac, 0x12, 0x93, 0x6b, 0x32, 0x85, 0x41, 0x46, 0x57, 0xb1, 0xbe, 0x66,
-	0xa7, 0x8f, 0x22, 0x13, 0x22, 0xd7, 0x01, 0xc2, 0x34, 0x8e, 0x69, 0x88, 0x0a, 0x36, 0x2a, 0x18,
-	0x88, 0xbc, 0x0a, 0x21, 0x62, 0x9f, 0xd3, 0xd0, 0x81, 0xa9, 0xb5, 0xdf, 0xf5, 0x7a, 0x42, 0xc4,
-	0xc7, 0x34, 0x94, 0x79, 0xe4, 0x9c, 0x66, 0x3e, 0xbe, 0xb1, 0x01, 0x9e, 0xeb, 0x4b, 0x00, 0xbb,
-	0xc1, 0x1e, 0xc0, 0x22, 0x4b, 0xf3, 0x95, 0x92, 0x0e, 0xa7, 0x6d, 0xd9, 0x72, 0x10, 0x41, 0xf1,
-	0x4d, 0xb8, 0xc2, 0xdf, 0x2e, 0x63, 0x96, 0x9c, 0xfa, 0x22, 0xc8, 0x16, 0x54, 0x38, 0x23, 0x34,
-	0x30, 0xd2, 0xe8, 0x4b, 0x04, 0xdd, 0x6f, 0x81, 0x1c, 0x64, 0x34, 0x10, 0xf4, 0x5f, 0x74, 0xd7,
-	0xb2, 0x53, 0xb6, 0x2e, 0xec, 0x94, 0xff, 0x87, 0x71, 0xc5, 0xb4, 0x6a, 0x34, 0xd2, 0xe3, 0xab,
-	0x55, 0xf4, 0xbe, 0x3c, 0x56, 0x4c, 0x6b, 0x8f, 0x3f, 0x59, 0x40, 0x1e, 0xe3, 0x4b, 0xf8, 0x6f,
-	0x23, 0x44, 0x72, 0x58, 0xb6, 0x36, 0xf5, 0xd2, 0xa2, 0x40, 0x04, 0xba, 0xf9, 0x0e, 0x19, 0x57,
-	0xf6, 0x1f, 0x07, 0x22, 0xd0, 0x0d, 0x30, 0xa3, 0x61, 0x9e, 0xc9, 0x7e, 0x8c, 0xbc, 0xc2, 0x06,
-	0xe8, 0x15, 0x90, 0x0c, 0xb4, 0x12, 0x90, 0x0e, 0xf4, 0x37, 0x0b, 0xc6, 0x0f, 0x39, 0x67, 0x8b,
-	0xe4, 0x9b, 0x34, 0xce, 0x97, 0xb4, 0x88, 0x74, 0x07, 0xba, 0x61, 0x9a, 0x27, 0x02, 0xa3, 0xec,
-	0x7a, 0x6a, 0xb3, 0x41, 0xab, 0x56, 0x8d, 0x56, 0x1b, 0xc4, 0x6c, 0xd7, 0x89, 0x69, 0x10, 0xaf,
-	0x53, 0x21, 0xde, 0x87, 0x30, 0x90, 0xe9, 0xf9, 0x21, 0x4d, 0x04, 0xcd, 0xf4, 0x3b, 0x06, 0x09,
-	0x1d, 0x20, 0xe2, 0x9e, 0xc1, 0x4e, 0x35, 0x50, 0x3d, 0x45, 0xce, 0xed, 0x2a, 0xf2, 0xd5, 0x65,
-	0xb1, 0x8e, 0x52, 0x2e, 0x25, 0x7f, 0x57, 0xf9, 0x49, 0xcc, 0x42, 0x5f, 0x0a, 0x54, 0x74, 0xb6,
-	0x42, 0x5e, 0x65, 0xf1, 0x3a, 0xe7, 0x8e, 0x91, 0xb3, 0xfb, 0x39, 0x8c, 0xd5, 0x10, 0xaf, 0x16,
-	0x68, 0x0f, 0xe0, 0x0c, 0x01, 0x9f, 0x45, 0x6a, 0x7e, 0xd9, 0x9e, 0xad, 0x90, 0xa7, 0x11, 0x77,
-	0xbf, 0x04, 0xfb, 0x28, 0x55, 0x39, 0x73, 0x72, 0x07, 0xec, 0xb8, 0xd8, 0xe8, 0x51, 0x47, 0xd6,
-	0x7c, 0x2a, 0xf4, 0xbc, 0xb5, 0x92, 0xfb, 0x00, 0xfa, 0x05, 0x5c, 0xe4, 0x61, 0x9d, 0x97, 0x47,
-	0x6b, 0x23, 0x0f, 0xf7, 0x0f, 0x0b, 0x76, 0xaa, 0x21, 0xeb, 0x52, 0xbd, 0x82, 0x51, 0xe9, 0xc2,
-	0x5f, 0x06, 0x2b, 0x1d, 0xcb, 0x1d, 0x33, 0x96, 0xfa, 0xb1, 0x32, 0x40, 0xfe, 0x2c, 0x58, 0x29,
-	0xf6, 0x0c, 0x63, 0x03, 0x9a, 0xbc, 0x84, 0xed, 0x9a, 0x4a, 0xc3, 0xf4, 0xba, 0x65, 0x4e, 0xaf,
-	0xca, 0x04, 0x2e, 0x4f, 0x9b, 0x23, 0xed, 0x3e, 0x5c, 0x55, 0x84, 0x3d, 0x28, 0xf9, 0x55, 0xd4,
-	0xbe, 0x4a, 0x43, 0x6b, 0x93, 0x86, 0xee, 0x04, 0x9c, 0xfa, 0x51, 0x4d, 0xf8, 0x05, 0x6c, 0x1f,
-	0x8b, 0x40, 0x30, 0x2e, 0x58, 0x58, 0x7e, 0x4a, 0x6d, 0xf0, 0xd6, 0x7a, 0x57, 0x43, 0xad, 0x33,
-	0x7f, 0x0b, 0xda, 0x42, 0x14, 0x9c, 0x92, 0x4b, 0x79, 0x0b, 0xc4, 0xf4, 0xa4, 0xef, 0xe0, 0x3d,
-	0xb8, 0x92, 0x7c, 0x10, 0xa9, 0x08, 0x62, 0x35, 0xb0, 0x3a, 0x38, 0xb0, 0x6c, 0x44, 0x70, 0x62,
-	0xa9, 0x9e, 0x1e, 0x29, 0x69, 0x57, 0x8d, 0x33, 0x09, 0xa0, 0x70, 0x0f, 0x00, 0x9f, 0x8f, 0x62,
-	0x7e, 0x4f, 0x9d, 0x95, 0xc8, 0x81, 0x04, 0xee, 0xfe, 0xd5, 0x85, 0xe1, 0x31, 0x0d, 0x5e, 0x53,
-	0x1a, 0xc9, 0x79, 0x99, 0x91, 0x45, 0xc1, 0xad, 0xea, 0x37, 0x2d, 0xb9, 0xb9, 0x49, 0xa2, 0xc6,
-	0x8f, 0xe8, 0xc9, 0x27, 0xef, 0x52, 0xd3, 0xd7, 0x74, 0x89, 0x1c, 0xc1, 0xc0, 0xf8, 0x68, 0x24,
-	0xbb, 0xc6, 0xc1, 0xda, 0xb7, 0xf0, 0x64, 0xef, 0x1c, 0xa9, 0x69, 0xcd, 0x98, 0x0c, 0xa6, 0xb5,
-	0xfa, 0x2c, 0x32, 0xad, 0x35, 0x8d, 0x13, 0xb4, 0x66, 0x74, 0x7d, 0xd3, 0x5a, 0x7d, 0xce, 0x98,
-	0xd6, 0x9a, 0x46, 0x05, 0x5a, 0x33, 0x5a, 0xb3, 0x69, 0xad, 0x3e, 0x42, 0x4c, 0x6b, 0x4d, 0xfd,
-	0xfc, 0x12, 0x79, 0x01, 0x43, 0xb3, 0x4f, 0x12, 0xe3, 0x40, 0x43, 0xa3, 0x9f, 0x5c, 0x3f, 0x4f,
-	0x6c, 0x1a, 0x34, 0xdb, 0x82, 0x69, 0xb0, 0xa1, 0x31, 0x9a, 0x06, 0x9b, 0xba, 0x89, 0x7b, 0x89,
-	0x7c, 0x07, 0x5b, 0x9b, 0xcf, 0x93, 0xdc, 0xd8, 0x4c, 0xab, 0xf6, 0xea, 0x27, 0xee, 0x45, 0x2a,
-	0xa5, 0xf1, 0xa7, 0x00, 0xeb, 0x57, 0x47, 0xae, 0xad, 0xcf, 0xd4, 0x5e, 0xfd, 0x64, 0xb7, 0x59,
-	0x58, 0x98, 0x7a, 0x74, 0x1d, 0xb6, 0xb8, 0xa2, 0xfe, 0x9c, 0xcf, 0xc2, 0x98, 0xd1, 0x44, 0x3c,
-	0x02, 0x7c, 0x05, 0x5f, 0xcb, 0x3f, 0xc7, 0x93, 0x1e, 0xfe, 0x40, 0x7e, 0xf6, 0x77, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0x8d, 0x38, 0xa9, 0x9f, 0x4f, 0x0e, 0x00, 0x00,
+	// 1506 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x4b, 0x6f, 0xdb, 0xc6,
+	0x16, 0x0e, 0xf5, 0xe6, 0x91, 0x94, 0xd8, 0x63, 0xdf, 0x1b, 0x46, 0xb6, 0x73, 0x15, 0xfa, 0x26,
+	0x70, 0x70, 0x03, 0xdf, 0x20, 0xed, 0x22, 0x69, 0x50, 0xa0, 0x89, 0x1f, 0x80, 0x51, 0xe7, 0x01,
+	0x3a, 0x29, 0x5a, 0x14, 0x28, 0x41, 0x93, 0x23, 0x79, 0x60, 0x8a, 0xa3, 0x92, 0x43, 0x3b, 0xe9,
+	0x4f, 0xe8, 0xa6, 0xfb, 0x02, 0x5d, 0xf7, 0x4f, 0x14, 0xdd, 0xf4, 0x57, 0xf4, 0x4f, 0x74, 0xd9,
+	0x75, 0x31, 0x67, 0x48, 0x6a, 0x28, 0xca, 0x4e, 0x8a, 0x22, 0x3b, 0xce, 0x79, 0x7c, 0xe7, 0x31,
+	0xe7, 0x31, 0x12, 0x74, 0x47, 0x2c, 0xa4, 0xf1, 0xf6, 0x34, 0xe6, 0x82, 0x93, 0x0e, 0x1e, 0xdc,
+	0xe9, 0xb1, 0xfd, 0x02, 0xd6, 0x0e, 0x39, 0x3f, 0x4d, 0xa7, 0xbb, 0x2c, 0xa6, 0xbe, 0xe0, 0xf1,
+	0xdb, 0xbd, 0x48, 0xc4, 0x6f, 0x1d, 0xfa, 0x6d, 0x4a, 0x13, 0x41, 0xd6, 0xc1, 0x0c, 0x72, 0x86,
+	0x65, 0x0c, 0x8d, 0x2d, 0xd3, 0x99, 0x11, 0x08, 0x81, 0x46, 0xe4, 0x4d, 0xa8, 0x55, 0x43, 0x06,
+	0x7e, 0xdb, 0x7b, 0xb0, 0xbe, 0x18, 0x30, 0x99, 0xf2, 0x28, 0xa1, 0xe4, 0x36, 0x34, 0xa9, 0x24,
+	0x20, 0x5a, 0xf7, 0xc1, 0xb5, 0xed, 0xdc, 0x95, 0x6d, 0x25, 0xa7, 0xb8, 0xf6, 0xaf, 0x06, 0x90,
+	0x43, 0x96, 0x08, 0x49, 0x64, 0x34, 0x79, 0x3f, 0x7f, 0xfe, 0x0d, 0xad, 0x69, 0x4c, 0x47, 0xec,
+	0x4d, 0xe6, 0x51, 0x76, 0x22, 0xf7, 0x60, 0x39, 0x11, 0x5e, 0x2c, 0xf6, 0x63, 0x3e, 0xd9, 0x67,
+	0x21, 0x7d, 0x2e, 0x9d, 0xae, 0xa3, 0x48, 0x95, 0x41, 0xb6, 0x81, 0xb0, 0xc8, 0x0f, 0xd3, 0x84,
+	0x9d, 0xd1, 0xa3, 0x9c, 0x6b, 0x35, 0x86, 0xc6, 0x56, 0xc7, 0x59, 0xc0, 0x21, 0xab, 0xd0, 0x0c,
+	0xd9, 0x84, 0x09, 0xab, 0x39, 0x34, 0xb6, 0xfa, 0x8e, 0x3a, 0xd8, 0x9f, 0xc1, 0x4a, 0xc9, 0xff,
+	0x2c, 0xfc, 0xbb, 0xd0, 0xa6, 0x8a, 0x64, 0x19, 0xc3, 0xfa, 0xa2, 0x04, 0xe4, 0x7c, 0xfb, 0xa7,
+	0x1a, 0x34, 0x91, 0x54, 0xe4, 0xd9, 0x98, 0xe5, 0x99, 0xdc, 0x82, 0x1e, 0x4b, 0xdc, 0x59, 0x32,
+	0x6a, 0xe8, 0x5f, 0x97, 0x25, 0x45, 0xde, 0xc9, 0xff, 0xa0, 0xe5, 0x9f, 0xa4, 0xd1, 0x69, 0x62,
+	0xd5, 0xd1, 0xd4, 0xca, 0xcc, 0x94, 0x0c, 0x76, 0x47, 0xf2, 0x9c, 0x4c, 0x84, 0x3c, 0x04, 0xf0,
+	0x84, 0x88, 0xd9, 0x71, 0x2a, 0x68, 0x82, 0xd1, 0x76, 0x1f, 0x58, 0x9a, 0x42, 0x9a, 0xd0, 0x27,
+	0x05, 0xdf, 0xd1, 0x64, 0xc9, 0x23, 0xe8, 0xd0, 0x37, 0x82, 0x46, 0x01, 0x0d, 0xac, 0x26, 0x1a,
+	0xda, 0x98, 0x8b, 0x69, 0x7b, 0x2f, 0xe3, 0xab, 0x08, 0x0b, 0xf1, 0xc1, 0x63, 0xe8, 0x97, 0x58,
+	0x64, 0x09, 0xea, 0xa7, 0x34, 0xbf, 0x59, 0xf9, 0x29, 0xb3, 0x7b, 0xe6, 0x85, 0xa9, 0x2a, 0xb2,
+	0x9e, 0xa3, 0x0e, 0x9f, 0xd4, 0x1e, 0x1a, 0xf6, 0x2e, 0x98, 0xfb, 0x69, 0x18, 0x16, 0x8a, 0x01,
+	0x8b, 0x73, 0xc5, 0x80, 0xc5, 0xb3, 0x42, 0xab, 0x5d, 0x5a, 0x68, 0xbf, 0x18, 0xb0, 0xbc, 0x77,
+	0x46, 0x23, 0xf1, 0x9c, 0x0b, 0x36, 0x62, 0xbe, 0x27, 0x18, 0x8f, 0xc8, 0x3d, 0x30, 0x79, 0x18,
+	0xb8, 0x97, 0x56, 0x6a, 0x87, 0x87, 0x99, 0xd7, 0xf7, 0xc0, 0x8c, 0xe8, 0xb9, 0x7b, 0xa9, 0xb9,
+	0x4e, 0x44, 0xcf, 0x95, 0xf4, 0x26, 0xf4, 0x03, 0x1a, 0x52, 0x41, 0xdd, 0xe2, 0x76, 0xe4, 0xd5,
+	0xf5, 0x14, 0x71, 0x47, 0x5d, 0xc7, 0x1d, 0xb8, 0x26, 0x21, 0xa7, 0x5e, 0x4c, 0x23, 0xe1, 0x4e,
+	0x3d, 0x71, 0x82, 0x77, 0x62, 0x3a, 0xfd, 0x88, 0x9e, 0xbf, 0x44, 0xea, 0x4b, 0x4f, 0x9c, 0xd8,
+	0x7f, 0x1a, 0x60, 0x16, 0x97, 0x49, 0xae, 0x43, 0x5b, 0x9a, 0x75, 0x59, 0x90, 0x65, 0xa2, 0x25,
+	0x8f, 0x07, 0x81, 0xec, 0x0c, 0x3e, 0x1a, 0x25, 0x54, 0xa0, 0x7b, 0x75, 0x27, 0x3b, 0xc9, 0xca,
+	0x4a, 0xd8, 0x77, 0xaa, 0x19, 0x1a, 0x0e, 0x7e, 0xcb, 0x8c, 0x4f, 0x04, 0x9b, 0x50, 0x34, 0x58,
+	0x77, 0xd4, 0x81, 0xac, 0x40, 0x93, 0xba, 0xc2, 0x1b, 0x63, 0x95, 0x9b, 0x4e, 0x83, 0xbe, 0xf2,
+	0xc6, 0xe4, 0xbf, 0x70, 0x35, 0xe1, 0x69, 0xec, 0x53, 0x37, 0x37, 0xdb, 0x42, 0x6e, 0x4f, 0x51,
+	0xf7, 0x95, 0x71, 0x1b, 0xea, 0x23, 0x16, 0x58, 0x6d, 0x4c, 0xcc, 0x52, 0xb9, 0x08, 0x0f, 0x02,
+	0x47, 0x32, 0xc9, 0xff, 0x01, 0x0a, 0xa4, 0xc0, 0xea, 0x5c, 0x20, 0x6a, 0xe6, 0xb8, 0x81, 0xfd,
+	0x25, 0xb4, 0x32, 0xf8, 0x35, 0x30, 0xcf, 0x78, 0x98, 0x4e, 0x8a, 0xb0, 0xfb, 0x4e, 0x47, 0x11,
+	0x0e, 0x02, 0x72, 0x03, 0x70, 0xd6, 0xb9, 0xb2, 0xaa, 0x6a, 0x18, 0x24, 0x66, 0xe8, 0x73, 0x8a,
+	0xd3, 0xc2, 0xe7, 0xfc, 0x94, 0xa9, 0xe8, 0xdb, 0x4e, 0x76, 0xb2, 0xff, 0xa8, 0xc1, 0xd5, 0x72,
+	0xb9, 0x4b, 0x13, 0x88, 0x82, 0xb9, 0x32, 0x10, 0x06, 0x61, 0x8f, 0x4a, 0xf9, 0xaa, 0xe9, 0xf9,
+	0xca, 0x55, 0x26, 0x3c, 0x50, 0x06, 0xfa, 0x4a, 0xe5, 0x19, 0x0f, 0xa8, 0xac, 0xd6, 0x94, 0x05,
+	0x98, 0xe0, 0xbe, 0x23, 0x3f, 0x25, 0x65, 0xcc, 0x82, 0x6c, 0x84, 0xc8, 0x4f, 0x74, 0x2f, 0x46,
+	0xdc, 0x96, 0xba, 0x32, 0x75, 0x92, 0x57, 0x36, 0x91, 0xd4, 0xb6, 0xba, 0x07, 0xf9, 0x4d, 0x86,
+	0xd0, 0x8d, 0xe9, 0x34, 0xcc, 0xaa, 0x17, 0xd3, 0x67, 0x3a, 0x3a, 0x89, 0xdc, 0x04, 0xf0, 0x79,
+	0x18, 0x52, 0x1f, 0x05, 0x4c, 0x14, 0xd0, 0x28, 0xb2, 0x72, 0x84, 0x08, 0xdd, 0x84, 0xfa, 0x16,
+	0x0c, 0x8d, 0xad, 0xa6, 0xd3, 0x12, 0x22, 0x3c, 0xa2, 0xbe, 0x8c, 0x23, 0x4d, 0x68, 0xec, 0xe2,
+	0x00, 0xea, 0xa2, 0x5e, 0x47, 0x12, 0x70, 0x54, 0x6e, 0x00, 0x8c, 0x63, 0x9e, 0x4e, 0x15, 0xb7,
+	0x37, 0xac, 0xcb, 0x79, 0x8c, 0x14, 0x64, 0xdf, 0x86, 0xab, 0xc9, 0xdb, 0x49, 0xc8, 0xa2, 0x53,
+	0x57, 0x78, 0xf1, 0x98, 0x0a, 0xab, 0xaf, 0x6a, 0x38, 0xa3, 0xbe, 0x42, 0xa2, 0xfd, 0x15, 0x90,
+	0x9d, 0x98, 0x7a, 0x82, 0xfe, 0x8d, 0xd5, 0xf3, 0x9e, 0xdd, 0xfd, 0x2f, 0x58, 0x29, 0x41, 0xab,
+	0x29, 0x2c, 0x2d, 0xbe, 0x9e, 0x06, 0x1f, 0xca, 0x62, 0x09, 0x3a, 0xb3, 0xf8, 0x83, 0x01, 0x64,
+	0x17, 0x1b, 0xfc, 0x9f, 0xed, 0x57, 0xd9, 0x72, 0x72, 0xee, 0xab, 0x01, 0x12, 0x78, 0xc2, 0xcb,
+	0x36, 0x53, 0x8f, 0x25, 0x0a, 0x7f, 0xd7, 0x13, 0x5e, 0xb6, 0x1d, 0x62, 0xea, 0xa7, 0xb1, 0x5c,
+	0x56, 0x58, 0x57, 0xb8, 0x1d, 0x9c, 0x9c, 0x24, 0x1d, 0x2d, 0x39, 0x94, 0x39, 0xfa, 0xa3, 0x01,
+	0xd6, 0x13, 0xc1, 0x27, 0xcc, 0x77, 0xa8, 0x34, 0x58, 0x72, 0x77, 0x13, 0xfa, 0x72, 0x2c, 0xce,
+	0xbb, 0xdc, 0xe3, 0x61, 0x30, 0x5b, 0x3b, 0x37, 0x40, 0x4e, 0x46, 0x57, 0xf3, 0xbc, 0xcd, 0xc3,
+	0x00, 0x0b, 0x62, 0x13, 0xe4, 0xf8, 0xd2, 0xf4, 0xd5, 0x12, 0xee, 0x45, 0xf4, 0xbc, 0xa4, 0x2f,
+	0x85, 0x50, 0x5f, 0xcd, 0xbc, 0x76, 0x44, 0xcf, 0xa5, 0xbe, 0xbd, 0x06, 0x37, 0x16, 0xf8, 0x96,
+	0x79, 0xfe, 0xb3, 0x01, 0x2b, 0x4f, 0x92, 0x84, 0x8d, 0xa3, 0x2f, 0xb0, 0xfb, 0x73, 0xa7, 0x57,
+	0xa1, 0xe9, 0xf3, 0x34, 0x12, 0xe8, 0x6c, 0xd3, 0x51, 0x87, 0xb9, 0x86, 0xa8, 0x55, 0x1a, 0x62,
+	0xae, 0xa5, 0xea, 0xd5, 0x96, 0xd2, 0x5a, 0xa6, 0x51, 0x6a, 0x99, 0xff, 0x40, 0x57, 0x5e, 0x8c,
+	0xeb, 0xd3, 0x48, 0xd0, 0x38, 0x1b, 0x98, 0x20, 0x49, 0x3b, 0x48, 0xb1, 0xbf, 0x37, 0x60, 0xb5,
+	0xec, 0x69, 0xf6, 0x3a, 0xb8, 0x70, 0x7e, 0xcb, 0x81, 0x11, 0x87, 0x99, 0x9b, 0xf2, 0x53, 0xb6,
+	0xde, 0x34, 0x3d, 0x0e, 0x99, 0xef, 0x4a, 0x86, 0x72, 0xcf, 0x54, 0x94, 0xd7, 0x71, 0x38, 0x0b,
+	0xba, 0xa1, 0x07, 0x4d, 0xa0, 0xe1, 0xa5, 0xe2, 0x24, 0x9f, 0xe1, 0xf2, 0xdb, 0xfe, 0x18, 0x56,
+	0xd4, 0x83, 0xad, 0x9c, 0xb5, 0x0d, 0x80, 0x62, 0xaa, 0xaa, 0xb7, 0x8a, 0xe9, 0x98, 0xf9, 0x58,
+	0x4d, 0xec, 0x4f, 0xc1, 0x3c, 0xe4, 0x2a, 0x11, 0x09, 0xb9, 0x0f, 0x66, 0x98, 0x1f, 0xb2, 0x67,
+	0x0d, 0x99, 0xb5, 0x47, 0x2e, 0xe7, 0xcc, 0x84, 0xec, 0xc7, 0xd0, 0xc9, 0xc9, 0x79, 0x6c, 0xc6,
+	0x45, 0xb1, 0xd5, 0xe6, 0x62, 0xb3, 0x7f, 0x33, 0x60, 0xb5, 0xec, 0x72, 0x96, 0xbe, 0xd7, 0xd0,
+	0x2f, 0x4c, 0xb8, 0x13, 0x6f, 0x9a, 0xf9, 0x72, 0x5f, 0xf7, 0xa5, 0xaa, 0x56, 0x38, 0x98, 0x3c,
+	0xf3, 0xa6, 0xaa, 0xa4, 0x7a, 0xa1, 0x46, 0x1a, 0xbc, 0x82, 0xe5, 0x8a, 0xc8, 0x82, 0x97, 0xca,
+	0x5d, 0xfd, 0xa5, 0x52, 0x7a, 0x6d, 0x15, 0xda, 0xfa, 0xf3, 0xe5, 0x11, 0x5c, 0x57, 0xfd, 0xb7,
+	0x53, 0x14, 0x5d, 0x9e, 0xfb, 0x72, 0x6d, 0x1a, 0xf3, 0xb5, 0x69, 0x0f, 0xc0, 0xaa, 0xaa, 0x66,
+	0x5d, 0x30, 0x86, 0xe5, 0x23, 0xe1, 0x09, 0x96, 0x08, 0xe6, 0x17, 0xcf, 0xe6, 0xb9, 0x62, 0x36,
+	0xde, 0xb5, 0x1f, 0xaa, 0xed, 0xb0, 0x04, 0x75, 0x21, 0xf2, 0x3a, 0x93, 0x9f, 0xf2, 0x16, 0x88,
+	0x6e, 0x29, 0xbb, 0x83, 0x0f, 0x60, 0x4a, 0xd6, 0x83, 0xe0, 0xc2, 0x0b, 0xd5, 0xfe, 0x6d, 0xe0,
+	0xfe, 0x35, 0x91, 0x82, 0x0b, 0x58, 0xad, 0xa8, 0x40, 0x71, 0x9b, 0x6a, 0x3b, 0x4b, 0x02, 0x32,
+	0x37, 0x00, 0xb0, 0xa5, 0x54, 0x37, 0xb4, 0x94, 0xae, 0xa4, 0xec, 0x48, 0xc2, 0x83, 0xdf, 0x5b,
+	0xd0, 0x3b, 0xa2, 0xde, 0x39, 0xa5, 0x81, 0x7c, 0x4e, 0xc4, 0x64, 0x9c, 0xd7, 0x56, 0xf9, 0xf7,
+	0x0b, 0xb9, 0x3d, 0x5f, 0x44, 0x0b, 0x7f, 0x30, 0x0d, 0xee, 0xbc, 0x4b, 0x2c, 0xbb, 0xa6, 0x2b,
+	0xe4, 0x10, 0xba, 0xda, 0x0f, 0x04, 0xb2, 0xae, 0x29, 0x56, 0x7e, 0xf7, 0x0c, 0x36, 0x2e, 0xe0,
+	0xea, 0x68, 0xda, 0xa2, 0xd3, 0xd1, 0xaa, 0xab, 0x55, 0x47, 0x5b, 0xb4, 0x1d, 0x11, 0x4d, 0x5b,
+	0x62, 0x3a, 0x5a, 0x75, 0x6d, 0xea, 0x68, 0x8b, 0x36, 0x1f, 0xa2, 0x69, 0x9b, 0x46, 0x47, 0xab,
+	0x6e, 0x44, 0x1d, 0x6d, 0xd1, 0x7a, 0xba, 0x42, 0xbe, 0x81, 0xe5, 0xca, 0x0e, 0x20, 0xf6, 0x4c,
+	0xeb, 0xa2, 0xe5, 0x35, 0xd8, 0xbc, 0x54, 0xa6, 0xc0, 0x7f, 0x01, 0x3d, 0x7d, 0x36, 0x13, 0xcd,
+	0xa1, 0x05, 0xdb, 0x65, 0x70, 0xf3, 0x22, 0xb6, 0x0e, 0xa8, 0x8f, 0x1d, 0x1d, 0x70, 0xc1, 0xe0,
+	0xd5, 0x01, 0x17, 0x4d, 0x2b, 0xfb, 0x0a, 0xf9, 0x1a, 0x96, 0xe6, 0xdb, 0x9f, 0xdc, 0x9a, 0x4f,
+	0x5b, 0x65, 0xaa, 0x0c, 0xec, 0xcb, 0x44, 0x0a, 0xf0, 0x03, 0x80, 0x59, 0x57, 0x93, 0xb5, 0x99,
+	0x4e, 0x65, 0xaa, 0x0c, 0xd6, 0x17, 0x33, 0x73, 0xa8, 0xa7, 0x37, 0x61, 0x29, 0x51, 0xad, 0x35,
+	0x4a, 0xb6, 0xfd, 0x90, 0xd1, 0x48, 0x3c, 0x05, 0xec, 0xb2, 0x97, 0x31, 0x17, 0xfc, 0xb8, 0x85,
+	0x7f, 0x46, 0x7c, 0xf4, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xce, 0x1a, 0x06, 0x9b, 0x10,
+	0x00, 0x00,
 }

weed/pb/filer_pb/filer_pb_helper.go

@@ -0,0 +1,58 @@
+package filer_pb
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+)
+
+func toFileId(fileIdStr string) (*FileId, error) {
+	t, err := needle.ParseFileIdFromString(fileIdStr)
+	if err != nil {
+		return nil, err
+	}
+	return &FileId{
+		VolumeId: uint32(t.VolumeId),
+		Cookie:   uint32(t.Cookie),
+		FileKey:  uint64(t.Key),
+	}, nil
+
+}
+
+func (fid *FileId) toFileId() string {
+	return needle.NewFileId(needle.VolumeId(fid.VolumeId), fid.FileKey, fid.Cookie).String()
+}
+
+func BeforeEntrySerialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.FileId != "" {
+			if fid, err := toFileId(chunk.FileId); err == nil {
+				chunk.Fid = fid
+				chunk.FileId = ""
+			}
+		}
+
+		if chunk.SourceFileId != "" {
+			if fid, err := toFileId(chunk.SourceFileId); err == nil {
+				chunk.SourceFid = fid
+				chunk.SourceFileId = ""
+			}
+		}
+
+	}
+}
+
+func AfterEntryDeserialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.Fid != nil && chunk.FileId == "" {
+			chunk.FileId = chunk.Fid.toFileId()
+		}
+
+		if chunk.SourceFid != nil && chunk.SourceFileId == "" {
+			chunk.SourceFileId = chunk.SourceFid.toFileId()
+		}
+
+	}
+}

weed/pb/filer_pb/filer_pb_helper_test.go

@@ -0,0 +1,17 @@
+package filer_pb
+
+import (
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+)
+
+func TestFileIdSize(t *testing.T) {
+	fileIdStr := "11745,0293434534cbb9892b"
+
+	fid, _ := toFileId(fileIdStr)
+	bytes, _ := proto.Marshal(fid)
+
+	println(len(fileIdStr))
+	println(len(bytes))
+}

weed/pb/master.proto

@@ -15,6 +15,14 @@ service Seaweed {
     }
     rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
     }
+    rpc CollectionList (CollectionListRequest) returns (CollectionListResponse) {
+    }
+    rpc CollectionDelete (CollectionDeleteRequest) returns (CollectionDeleteResponse) {
+    }
+    rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
+    }
+    rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -29,14 +37,22 @@ message Heartbeat {
     string rack = 7;
     uint32 admin_port = 8;
     repeated VolumeInformationMessage volumes = 9;
-    // delta volume ids
-    repeated uint32 new_vids = 10;
-    repeated uint32 deleted_vids = 11;
+    // delta volumes
+    repeated VolumeShortInformationMessage new_volumes = 10;
+    repeated VolumeShortInformationMessage deleted_volumes = 11;
+    bool has_no_volumes = 12;
+
+    // erasure coding
+    repeated VolumeEcShardInformationMessage ec_shards = 16;
+    // delta erasure coding shards
+    repeated VolumeEcShardInformationMessage new_ec_shards = 17;
+    repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
+    bool has_no_ec_shards = 19;
+
 }
 
 message HeartbeatResponse {
     uint64 volumeSizeLimit = 1;
-    string secretKey = 2;
     string leader = 3;
 }
 
@@ -51,6 +67,22 @@ message VolumeInformationMessage {
     uint32 replica_placement = 8;
     uint32 version = 9;
     uint32 ttl = 10;
+    uint32 compact_revision = 11;
+    int64 modified_at_second = 12;
+}
+
+message VolumeShortInformationMessage {
+    uint32 id = 1;
+    string collection = 3;
+    uint32 replica_placement = 8;
+    uint32 version = 9;
+    uint32 ttl = 10;
+}
+
+message VolumeEcShardInformationMessage {
+    uint32 id = 1;
+    string collection = 2;
+    uint32 ec_index_bits = 3;
 }
 
 message Empty {
@@ -109,6 +141,7 @@ message AssignResponse {
     string public_url = 3;
     uint64 count = 4;
     string error = 5;
+    string auth = 6;
 }
 
 message StatisticsRequest {
@@ -124,3 +157,83 @@ message StatisticsResponse {
     uint64 used_size = 5;
     uint64 file_count = 6;
 }
+
+//
+// collection related
+//
+
+message StorageType {
+    string replication = 1;
+    string ttl = 2;
+}
+message Collection {
+    string name = 1;
+}
+message CollectionListRequest {
+    bool include_normal_volumes = 1;
+    bool include_ec_volumes = 2;
+}
+message CollectionListResponse {
+    repeated Collection collections = 1;
+}
+
+message CollectionDeleteRequest {
+    string name = 1;
+}
+message CollectionDeleteResponse {
+}
+
+//
+// volume related
+//
+message DataNodeInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated VolumeInformationMessage volume_infos = 6;
+    repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
+}
+message RackInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated DataNodeInfo data_node_infos = 6;
+}
+message DataCenterInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated RackInfo rack_infos = 6;
+}
+message TopologyInfo {
+    string id = 1;
+    uint64 volume_count = 2;
+    uint64 max_volume_count = 3;
+    uint64 free_volume_count = 4;
+    uint64 active_volume_count = 5;
+    repeated DataCenterInfo data_center_infos = 6;
+}
+message VolumeListRequest {
+}
+message VolumeListResponse {
+    TopologyInfo topology_info = 1;
+    uint64 volume_size_limit_mb = 2;
+}
+
+message LookupEcVolumeRequest {
+    uint32 volume_id = 1;
+}
+message LookupEcVolumeResponse {
+    uint32 volume_id = 1;
+    message EcShardIdLocation {
+        uint32 shard_id = 1;
+        repeated Location locations = 2;
+    }
+    repeated EcShardIdLocation shard_id_locations = 2;
+}

weed/pb/volume_server.proto

@@ -19,14 +19,12 @@ service VolumeServer {
 
     rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
     }
-    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
+    rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
     }
 
     rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
     }
-    rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) {
-    }
-    rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) {
+    rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
     }
 
     rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
@@ -36,7 +34,34 @@ service VolumeServer {
     rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
     }
 
-    // rpc VolumeUiPage (VolumeUiPageRequest) returns (VolumeUiPageResponse) {}
+    // copy the .idx .dat files, and mount this volume
+    rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
+    }
+    rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
+    }
+    rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
+    }
+
+    rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
+    }
+    rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
+    }
+
+    // erasure coding
+    rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
+    }
+    rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
+    }
+    rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
+    }
+    rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
+    }
+    rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
+    }
+    rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
+    }
+    rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
+    }
 
 }
 
@@ -60,27 +85,27 @@ message Empty {
 }
 
 message VacuumVolumeCheckRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VacuumVolumeCheckResponse {
     double garbage_ratio = 1;
 }
 
 message VacuumVolumeCompactRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
     int64 preallocate = 2;
 }
 message VacuumVolumeCompactResponse {
 }
 
 message VacuumVolumeCommitRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VacuumVolumeCommitResponse {
 }
 
 message VacuumVolumeCleanupRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VacuumVolumeCleanupResponse {
 }
@@ -91,21 +116,21 @@ message DeleteCollectionRequest {
 message DeleteCollectionResponse {
 }
 
-message AssignVolumeRequest {
-    uint32 volumd_id = 1;
+message AllocateVolumeRequest {
+    uint32 volume_id = 1;
     string collection = 2;
     int64 preallocate = 3;
     string replication = 4;
     string ttl = 5;
 }
-message AssignVolumeResponse {
+message AllocateVolumeResponse {
 }
 
 message VolumeSyncStatusRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeSyncStatusResponse {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
     string collection = 2;
     string replication = 4;
     string ttl = 5;
@@ -114,45 +139,145 @@ message VolumeSyncStatusResponse {
     uint64 idx_file_size = 8;
 }
 
-message VolumeSyncIndexRequest {
-    uint32 volumd_id = 1;
+message VolumeIncrementalCopyRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
 }
-message VolumeSyncIndexResponse {
-    bytes index_file_content = 1;
-}
-
-message VolumeSyncDataRequest {
-    uint32 volumd_id = 1;
-    uint32 revision = 2;
-    uint32 offset = 3;
-    uint32 size = 4;
-    string needle_id = 5;
-}
-message VolumeSyncDataResponse {
+message VolumeIncrementalCopyResponse {
     bytes file_content = 1;
 }
 
 message VolumeMountRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeMountResponse {
 }
 
 message VolumeUnmountRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeUnmountResponse {
 }
 
 message VolumeDeleteRequest {
-    uint32 volumd_id = 1;
+    uint32 volume_id = 1;
 }
 message VolumeDeleteResponse {
 }
 
-message VolumeUiPageRequest {
+message VolumeCopyRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    string replication = 3;
+    string ttl = 4;
+    string source_data_node = 5;
 }
-message VolumeUiPageResponse {
+message VolumeCopyResponse {
+    uint64 last_append_at_ns = 1;
+}
+
+message CopyFileRequest {
+    uint32 volume_id = 1;
+    string ext = 2;
+    uint32 compaction_revision = 3;
+    uint64 stop_offset = 4;
+    string collection = 5;
+    bool is_ec_volume = 6;
+}
+message CopyFileResponse {
+    bytes file_content = 1;
+}
+
+message VolumeTailSenderRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+}
+message VolumeTailSenderResponse {
+    bytes needle_header = 1;
+    bytes needle_body = 2;
+    bool is_last_chunk = 3;
+}
+
+message VolumeTailReceiverRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+    string source_volume_server = 4;
+}
+message VolumeTailReceiverResponse {
+}
+
+message VolumeEcShardsGenerateRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsGenerateResponse {
+}
+
+message VolumeEcShardsRebuildRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsRebuildResponse {
+    repeated uint32 rebuilt_shard_ids = 1;
+}
+
+message VolumeEcShardsCopyRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+    bool copy_ecx_file = 4;
+    string source_data_node = 5;
+}
+message VolumeEcShardsCopyResponse {
+}
+
+message VolumeEcShardsDeleteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsDeleteResponse {
+}
+
+message VolumeEcShardsMountRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsMountResponse {
+}
+
+message VolumeEcShardsUnmountRequest {
+    uint32 volume_id = 1;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsUnmountResponse {
+}
+
+message VolumeEcShardReadRequest {
+    uint32 volume_id = 1;
+    uint32 shard_id = 2;
+    int64 offset = 3;
+    int64 size = 4;
+}
+message VolumeEcShardReadResponse {
+    bytes data = 1;
+}
+
+message ReadVolumeFileStatusRequest {
+    uint32 volume_id = 1;
+}
+message ReadVolumeFileStatusResponse {
+    uint32 volume_id = 1;
+    uint64 idx_file_timestamp_seconds = 2;
+    uint64 idx_file_size = 3;
+    uint64 dat_file_timestamp_seconds = 4;
+    uint64 dat_file_size = 5;
+    uint64 file_count = 6;
+    uint32 compaction_revision = 7;
+    string collection = 8;
 }
 
 message DiskStatus {