1.41

fix https://github.com/chrislusf/seaweedfs/issues/1001

.gitignore

@@ -80,3 +80,6 @@ build
 target
 *.class
 other/java/hdfs/dependency-reduced-pom.xml
+
+# binary file
+weed/weed

.travis.yml

@@ -1,19 +1,21 @@
 sudo: false
 language: go
 go:
-- 1.10.x
-- 1.11.x
-- 1.12.x
-- tip
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
+  # - tip
 
 before_install:
-- export PATH=/home/travis/gopath/bin:$PATH
+  - export PATH=/home/travis/gopath/bin:$PATH
 
 install:
-- go get ./weed/...
+  - export CGO_ENABLED="0"
+  - go env
+  - go get -u ./weed/...
 
 script:
-- go test ./weed/...
+  - go test ./weed/...
 
 before_deploy:
   - make release
@@ -23,23 +25,26 @@ deploy:
   api_key:
     secure: ERL986+ncQ8lwAJUYDrQ8s2/FxF/cyNIwJIFCqspnWxQgGNNyokET9HapmlPSxjpFRF0q6L2WCg9OY3mSVRq4oI6hg1igOQ12KlLyN71XSJ3c8w0Ay5ho48TQ9l3f3Iu97mntBCe9l0R9pnT8wj1VI8YJxloXwUMG2yeTjA9aBI=
   file:
-  - build/linux_arm.tar.gz
-  - build/linux_arm64.tar.gz
-  - build/linux_386.tar.gz
-  - build/linux_amd64.tar.gz
-  - build/darwin_amd64.tar.gz
-  - build/windows_386.zip
-  - build/windows_amd64.zip
-  - build/freebsd_arm.tar.gz
-  - build/freebsd_amd64.tar.gz
-  - build/freebsd_386.tar.gz
-  - build/netbsd_arm.tar.gz
-  - build/netbsd_amd64.tar.gz
-  - build/netbsd_386.tar.gz
-  - build/openbsd_arm.tar.gz
-  - build/openbsd_amd64.tar.gz
-  - build/openbsd_386.tar.gz
+    - build/linux_arm.tar.gz
+    - build/linux_arm64.tar.gz
+    - build/linux_386.tar.gz
+    - build/linux_amd64.tar.gz
+    - build/linux_amd64_large_disk.tar.gz
+    - build/darwin_amd64.tar.gz
+    - build/darwin_amd64_large_disk.tar.gz
+    - build/windows_386.zip
+    - build/windows_amd64.zip
+    - build/windows_amd64_large_disk.zip
+    - build/freebsd_arm.tar.gz
+    - build/freebsd_amd64.tar.gz
+    - build/freebsd_386.tar.gz
+    - build/netbsd_arm.tar.gz
+    - build/netbsd_amd64.tar.gz
+    - build/netbsd_386.tar.gz
+    - build/openbsd_arm.tar.gz
+    - build/openbsd_amd64.tar.gz
+    - build/openbsd_386.tar.gz
   on:
     tags: true
     repo: chrislusf/seaweedfs
-    go: tip
+    go: 1.12.x

Makefile

@@ -12,6 +12,9 @@ build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -stat
 tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
 zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
 
+build_large = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -tags 5BytesOffset -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
+tar_large = cd build && tar -cvzf $(1)_$(2)_large_disk.tar.gz $(appname)$(3) && rm $(appname)$(3)
+zip_large = cd build && zip $(1)_$(2)_large_disk.zip $(appname)$(3) && rm $(appname)$(3)
 
 all: build
 
@@ -32,9 +35,21 @@ linux: deps
 	mkdir -p linux
 	GOOS=linux GOARCH=amd64 go build $(GO_FLAGS) -o linux/$(BINARY) $(SOURCE_DIR)
 
-release: deps windows_build darwin_build linux_build bsd_build
+release: deps windows_build darwin_build linux_build bsd_build 5_byte_linux_build 5_byte_darwin_build 5_byte_windows_build
 
 ##### LINUX BUILDS #####
+5_byte_linux_build:
+	$(call build_large,linux,amd64,)
+	$(call tar_large,linux,amd64)
+
+5_byte_darwin_build:
+	$(call build_large,darwin,amd64,)
+	$(call tar_large,darwin,amd64)
+
+5_byte_windows_build:
+	$(call build_large,windows,amd64,.exe)
+	$(call zip_large,windows,amd64,.exe)
+
 linux_build: build/linux_arm.tar.gz build/linux_arm64.tar.gz build/linux_386.tar.gz build/linux_amd64.tar.gz
 
 build/linux_386.tar.gz: $(sources)

README.md

@@ -10,18 +10,16 @@
 
 <h2 align="center">Supporting SeaweedFS</h2>
 
-SeaweedFS is an independent Apache-licensed open source project with its ongoing development made 
-possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
+SeaweedFS is an independent Apache-licensed open source project with its ongoing development made
+possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md).
 If you'd like to grow SeaweedFS even stronger, please consider joining our
 <a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
 
-Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
-Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
-
 Your support will be really appreciated by me and other supporters!
 
 <h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
 
+<!--
 <h4 align="center">Platinum</h4>
 
 <p align="center">
@@ -45,6 +43,8 @@ Your support will be really appreciated by me and other supporters!
   </tbody>
 </table>
 
+-->
+
 ---
 
 
@@ -52,9 +52,29 @@ Your support will be really appreciated by me and other supporters!
 - [SeaweedFS on Slack](https://join.slack.com/t/seaweedfs/shared_invite/enQtMzI4MTMwMjU2MzA3LTc4MmVlYmFlNjBmZTgzZmJlYmI1MDE1YzkyNWYyZjkwZDFiM2RlMDdjNjVlNjdjYzc4NGFhZGIyYzEyMzJkYTA)
 - [SeaweedFS Mailing List](https://groups.google.com/d/forum/seaweedfs)
 - [Wiki Documentation](https://github.com/chrislusf/seaweedfs/wiki)
+- [SeaweedFS Introduction Slides](https://www.slideshare.net/chrislusf/seaweedfs-introduction)
 
+Table of Contents
+=================
 
-## Introduction
+* [Introduction](#introduction)
+* [Features](#features)
+    * [Additional Features](#additional-features)
+    * [Filer Features](#filer-features)
+* [Example Usage](#example-usage)
+* [Architecture](#architecture)
+* [Compared to Other File Systems](#compared-to-other-file-systems)
+    * [Compared to HDFS](#compared-to-hdfs)
+    * [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
+    * [Compared to GlusterFS](#compared-to-glusterfs)
+    * [Compared to Ceph](#compared-to-ceph)
+* [Dev Plan](#dev-plan)
+* [Installation Guide](#installation-guide)
+* [Disk Related Topics](#disk-related-topics)
+* [Benchmark](#Benchmark)
+* [License](#license)
+
+## Introduction ##
 
 SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
 
@@ -65,41 +85,57 @@ SeaweedFS started as an Object Store to handle small files efficiently. Instead
 
 There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
-SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
+SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf). Also, SeaweedFS implements erasure coding with ideas from [f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf)
 
 SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
 
-## Additional Features
-* Can choose no replication or different replication levels, rack and data center aware
-* Automatic master servers failover - no single point of failure (SPOF)
-* Automatic Gzip compression depending on file mime type
-* Automatic compaction to reclaim disk space after deletion or update
-* Servers in the same cluster can have different disk spaces, file systems, OS etc.
-* Adding/Removing servers does **not** cause any data re-balancing
-* Optionally fix the orientation for jpeg pictures
-* Support Etag, Accept-Range, Last-Modified, etc.
-* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
+[Back to TOC](#table-of-contents)
 
-## Filer Features
+## Features ##
+
+[Back to TOC](#table-of-contents)
+
+## Additional Features ##
+* Can choose no replication or different replication levels, rack and data center aware.
+* Automatic master servers failover - no single point of failure (SPOF).
+* Automatic Gzip compression depending on file mime type.
+* Automatic compaction to reclaim disk space after deletion or update.
+* Servers in the same cluster can have different disk spaces, file systems, OS etc.
+* Adding/Removing servers does **not** cause any data re-balancing.
+* Optionally fix the orientation for jpeg pictures.
+* Support ETag, Accept-Range, Last-Modified, etc.
+* Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
+* Support rebalancing the writable and readonly volumes.
+
+[Back to TOC](#table-of-contents)
+
+## Filer Features ##
 * [filer server][Filer] provide "normal" directories and files via http.
 * [mount filer][Mount] to read and write files directly as a local directory via FUSE.
 * [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
+* [Erasure Coding for warm storage][ErasureCoding]  Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
 * [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
 * [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+* [WebDAV] access as a mapped drive on Mac and Windows, or from mobile devices.
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
 [Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
 [AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
 [BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
 [Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
+[WebDAV]: https://github.com/chrislusf/seaweedfs/wiki/WebDAV
+[ErasureCoding]: https://github.com/chrislusf/seaweedfs/wiki/Erasure-coding-for-warm-storage
+
+[Back to TOC](#table-of-contents)
+
+## Example Usage ##
 
-## Example Usage
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.
 Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
 
 SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
 
-### Start Master Server
+### Start Master Server ###
 
 ```
 > ./weed master
@@ -125,7 +161,7 @@ Second, to store the file content, send a HTTP multi-part POST request to `url +
 
 ```
 > curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
-{"size": 43234}
+{"name":"myphoto.jpg","size":43234,"eTag":"1cc0118e"}
 ```
 
 To update, send another POST request with updated file content.
@@ -135,6 +171,7 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 ```
 > curl -X DELETE http://127.0.0.1:8080/3,01637037d6
 ```
+
 ### Save File Id ###
 
 Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
@@ -157,7 +194,7 @@ First look up the volume server's URLs by the file's volumeId:
 
 ```
 > curl http://localhost:9333/dir/lookup?volumeId=3
-{"locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
+{"volumeId":"3","locations":[{"publicUrl":"localhost:8080","url":"localhost:8080"}]}
 ```
 
 Since (usually) there are not too many volume servers, and volumes don't move often, you can cache the results most of the time. Depending on the replication type, one volume can have multiple replica locations. Just randomly pick one location to read.
@@ -213,7 +250,7 @@ More details about replication can be found [on the wiki][Replication].
 
 You can also set the default replication strategy when starting the master server.
 
-### Allocate File Key on specific data center ###
+### Allocate File Key on Specific Data Center ###
 
 Volume servers can be started with a specific data center name:
 
@@ -239,6 +276,8 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
 [feat-3]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#upload-large-files
 [feat-4]: https://github.com/chrislusf/seaweedfs/wiki/Optimization#collection-as-a-simple-name-space
 
+[Back to TOC](#table-of-contents)
+
 ## Architecture ##
 
 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
@@ -279,12 +318,16 @@ Each individual file size is limited to the volume size.
 
 All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
+[Back to TOC](#table-of-contents)
+
 ## Compared to Other File Systems ##
 
 Most other distributed file systems seem more complicated than necessary.
 
 SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
 
+[Back to TOC](#table-of-contents)
+
 ### Compared to HDFS ###
 
 HDFS uses the chunk approach for each file, and is ideal for storing large files.
@@ -293,6 +336,7 @@ SeaweedFS is ideal for serving relatively smaller files quickly and concurrently
 
 SeaweedFS can also store extra large files by splitting them into manageable data chunks, and store the file ids of the data chunks into a meta chunk. This is managed by "weed upload/download" tool, and the weed master or volume servers are agnostic about it.
 
+[Back to TOC](#table-of-contents)
 
 ### Compared to GlusterFS, Ceph ###
 
@@ -310,17 +354,21 @@ The architectures are mostly the same. SeaweedFS aims to store and read files fa
 | GlusterFS      | hashing          |                  | FUSE, NFS          |          |                           |
 | Ceph           | hashing + rules  |                  | FUSE               | Yes      |                           |
 
+[Back to TOC](#table-of-contents)
+
 ### Compared to GlusterFS ###
 
 GlusterFS stores files, both directories and content, in configurable volumes called "bricks".
 
 GlusterFS hashes the path and filename into ids, and assigned to virtual volumes, and then mapped to "bricks".
 
+[Back to TOC](#table-of-contents)
+
 ### Compared to Ceph ###
 
 Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
 
-SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. 
+SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage.
 
 Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
 
@@ -336,16 +384,26 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Cassa
 | Volume  | OSD | optimized for small files |
 | Filer  | Ceph FS | linearly scalable, Customizable, O(1) or O(logN) |
 
+[Back to TOC](#table-of-contents)
 
-## Dev plan ##
+## Dev Plan ##
 
 More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
 Other key features include: Erasure Encoding, JWT security.
 
 This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
+BTW, We suggest run the code style check script `util/gostd` before you push your branch to remote, it will make SeaweedFS easy to review, maintain and develop:
 
-## Installation guide for users who are not familiar with golang
+```
+$ ./util/gostd
+```
+
+[Back to TOC](#table-of-contents)
+
+## Installation Guide ##
+
+> Installation guide for users who are not familiar with golang
 
 Step 1: install go on your machine and setup the environment by following the instructions at:
 
@@ -366,23 +424,27 @@ go get github.com/chrislusf/seaweedfs/weed
 
 Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-Step 4: after you modify your code locally, you could start a local build by calling `go install` under 
+Step 4: after you modify your code locally, you could start a local build by calling `go install` under
 
 ```
 $GOPATH/src/github.com/chrislusf/seaweedfs/weed
 ```
 
-## Disk Related topics ##
+[Back to TOC](#table-of-contents)
+
+## Disk Related Topics ##
 
 ### Hard Drive Performance ###
 
 When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
 
-### Solid State Disk
+### Solid State Disk ###
 
 To modify or delete small files, SSD must delete a whole block at a time, and move content in existing blocks to a new block. SSD is fast when brand new, but will get fragmented over time and you have to garbage collect, compacting blocks. SeaweedFS is friendly to SSD since it is append-only. Deletion and compaction are done on volume level in the background, not slowing reading and not causing fragmentation.
 
-## Benchmark
+[Back to TOC](#table-of-contents)
+
+## Benchmark ##
 
 My Own Unscientific Single Machine Results on Mac Book with Solid State Disk, CPU: 1 Intel Core i7 2.6GHz.
 
@@ -435,8 +497,9 @@ Percentage of the requests served within a certain time (ms)
   100%     20.7 ms
 ```
 
+[Back to TOC](#table-of-contents)
 
-## License
+## License ##
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -450,7 +513,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
+[Back to TOC](#table-of-contents)
 
-## Stargazers over time
+## Stargazers over time ##
 
 [![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)

docker/Dockerfile

@@ -1,9 +1,20 @@
 FROM frolvlad/alpine-glibc
 
+# Supercronic install settings
+ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.1.8/supercronic-linux-amd64 \
+    SUPERCRONIC=supercronic-linux-amd64 \
+    SUPERCRONIC_SHA1SUM=be43e64c45acd6ec4fce5831e03759c89676a0ea
+
+# Install SeaweedFS and Supercronic ( for cron job mode )
 # Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
 RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
     wget -P /tmp https://github.com/$(curl -s -L https://github.com/chrislusf/seaweedfs/releases/latest | egrep -o 'chrislusf/seaweedfs/releases/download/.*/linux_amd64.tar.gz') && \
     tar -C /usr/bin/ -xzvf /tmp/linux_amd64.tar.gz && \
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+    echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+    chmod +x "$SUPERCRONIC" && \
+    mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+    ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
     apk del build-dependencies && \
     rm -rf /tmp/*
 
@@ -22,7 +33,7 @@ EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
 
-RUN mkdir -p /data/filerdb
+RUN mkdir -p /data/filerldb2
 
 VOLUME /data
 

docker/Dockerfile.go_build

@@ -16,7 +16,7 @@ EXPOSE 9333
 # s3 server http port
 EXPOSE 8333
 
-RUN mkdir -p /data/filerdb
+RUN mkdir -p /data/filerldb2
 
 VOLUME /data
 

docker/entrypoint.sh

@@ -29,7 +29,7 @@ case "$1" in
   	;;
 
   'filer')
-  	ARGS="-ip `hostname -i` "
+  	ARGS=""
   	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
 		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
 	fi
@@ -44,6 +44,16 @@ case "$1" in
   	exec /usr/bin/weed $@ $ARGS
 	;;
 
+  'cronjob')
+	MASTER=${WEED_MASTER-localhost:9333}
+	FIX_REPLICATION_CRON_SCHEDULE=${CRON_SCHEDULE-*/7 * * * * *}
+	echo "$FIX_REPLICATION_CRON_SCHEDULE" 'echo "volume.fix.replication" | weed shell -master='$MASTER > /crontab
+	BALANCING_CRON_SCHEDULE=${CRON_SCHEDULE-25 * * * * *}
+	echo "$BALANCING_CRON_SCHEDULE" 'echo "volume.balance -c ALL -force" | weed shell -master='$MASTER >> /crontab
+	echo "Running Crontab:"
+	cat /crontab
+	exec supercronic /crontab
+	;;
   *)
   	exec /usr/bin/weed $@
 	;;

docker/filer.toml

@@ -1,3 +1,3 @@
-[leveldb]
+[leveldb2]
 enabled = true
-dir = "/data/filerdb"
+dir = "/data/filerldb2"

docker/seaweedfs-compose.yml

@@ -26,6 +26,16 @@ services:
     depends_on:
     - master
     - volume
+  cronjob:
+    image: chrislusf/seaweedfs # use a remote image
+    command: 'cronjob'
+    environment:
+      # Run re-replication every 2 minutes
+      CRON_SCHEDULE: '*/2 * * * * *' # Default: '*/5 * * * * *'
+      WEED_MASTER: master:9333 # Default: localhost:9333
+    depends_on:
+    - master
+    - volume
   s3:
     image: chrislusf/seaweedfs # use a remote image
     ports:

other/java/client/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.0.9</version>
+    <version>1.1.0</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -173,21 +173,27 @@ public class FilerClient {
     }
 
     public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
-        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+        List<FilerProto.Entry> entries = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
                 .setDirectory(path)
                 .setPrefix(entryPrefix)
                 .setStartFromFileName(lastEntryName)
                 .setLimit(limit)
                 .build()).getEntriesList();
+        List<FilerProto.Entry> fixedEntries = new ArrayList<>(entries.size());
+        for (FilerProto.Entry entry : entries) {
+            fixedEntries.add(fixEntryAfterReading(entry));
+        }
+        return fixedEntries;
     }
 
     public FilerProto.Entry lookupEntry(String directory, String entryName) {
         try {
-            return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+            FilerProto.Entry entry = filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
                     FilerProto.LookupDirectoryEntryRequest.newBuilder()
                             .setDirectory(directory)
                             .setName(entryName)
                             .build()).getEntry();
+            return fixEntryAfterReading(entry);
         } catch (Exception e) {
             LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
             return null;
@@ -251,4 +257,24 @@ public class FilerClient {
         return true;
     }
 
+    private FilerProto.Entry fixEntryAfterReading(FilerProto.Entry entry) {
+        if (entry.getChunksList().size() <= 0) {
+            return entry;
+        }
+        String fileId = entry.getChunks(0).getFileId();
+        if (fileId != null && fileId.length() != 0) {
+            return entry;
+        }
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        entryBuilder.clearChunks();
+        for (FilerProto.FileChunk chunk : entry.getChunksList()) {
+            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
+            FilerProto.FileId fid = chunk.getFid();
+            fileId = String.format("%d,%d%x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie());
+            chunkBuilder.setFileId(fileId);
+            entryBuilder.addChunks(chunkBuilder);
+        }
+        return entryBuilder.build();
+    }
+
 }

other/java/client/src/main/proto/filer.proto

@@ -39,6 +39,9 @@ service SeaweedFiler {
     rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
     }
 
+    rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
+    }
+
 }
 
 //////////////////////////////////////////////////
@@ -72,19 +75,33 @@ message Entry {
     map<string, bytes> extended = 5;
 }
 
+message FullEntry {
+    string dir = 1;
+    Entry entry = 2;
+}
+
 message EventNotification {
     Entry old_entry = 1;
     Entry new_entry = 2;
     bool delete_chunks = 3;
+    string new_parent_path = 4;
 }
 
 message FileChunk {
-    string file_id = 1;
+    string file_id = 1; // to be deprecated
     int64 offset = 2;
     uint64 size = 3;
     int64 mtime = 4;
     string e_tag = 5;
-    string source_file_id = 6;
+    string source_file_id = 6; // to be deprecated
+    FileId fid = 7;
+    FileId source_fid = 8;
+}
+
+message FileId {
+    uint32 volume_id = 1;
+    uint64 file_key = 2;
+    fixed32 cookie = 3;
 }
 
 message FuseAttributes {
@@ -191,3 +208,12 @@ message StatisticsResponse {
     uint64 used_size = 5;
     uint64 file_count = 6;
 }
+
+message GetFilerConfigurationRequest {
+}
+message GetFilerConfigurationResponse {
+    repeated string masters = 1;
+    string replication = 2;
+    string collection = 3;
+    uint32 max_mb = 4;
+}

other/java/hdfs/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.0.9</seaweedfs.client.version>
+        <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 

unmaintained/change_superblock/change_superblock.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 var (
@@ -73,7 +74,7 @@ func main() {
 	}
 
 	if *targetTTL != "" {
-		ttl, err := storage.ReadTTL(*targetTTL)
+		ttl, err := needle.ReadTTL(*targetTTL)
 
 		if err != nil {
 			glog.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)

unmaintained/compact_leveldb/compact_leveldb.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	dir = flag.String("dir", ".", "data directory to store leveldb files")
+)
+
+func main() {
+
+	flag.Parse()
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+		OpenFilesCacheCapacity:        -1,
+	}
+
+	db, err := leveldb.OpenFile(*dir, opts)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+	if err := db.CompactRange(util.Range{}); err != nil {
+		log.Fatal(err)
+	}
+}

unmaintained/fix_dat/fix_dat.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -61,7 +62,7 @@ func main() {
 	}
 	newDatFile.Write(superBlock.Bytes())
 
-	iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) {
+	iterateEntries(datFile, indexFile, func(n *needle.Needle, offset int64) {
 		fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
 		_, s, _, e := n.Append(newDatFile, superBlock.Version())
 		fmt.Printf("size %d error %v\n", s, e)
@@ -69,7 +70,7 @@ func main() {
 
 }
 
-func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needle, offset int64)) {
+func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) {
 	// start to read index file
 	var readerOffset int64
 	bytes := make([]byte, 16)
@@ -84,7 +85,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 	}
 	offset := int64(superBlock.BlockSize())
 	version := superBlock.Version()
-	n, rest, err := storage.ReadNeedleHeader(datFile, version, offset)
+	n, _, rest, err := needle.ReadNeedleHeader(datFile, version, offset)
 	if err != nil {
 		fmt.Printf("cannot read needle header: %v", err)
 		return
@@ -106,7 +107,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 
 		fmt.Printf("key: %d offsetFromIndex %d n.Size %d sizeFromIndex:%d\n", key, offsetFromIndex, n.Size, sizeFromIndex)
 
-		rest = storage.NeedleBodyLength(sizeFromIndex, version)
+		rest = needle.NeedleBodyLength(sizeFromIndex, version)
 
 		func() {
 			defer func() {
@@ -114,7 +115,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 					fmt.Println("Recovered in f", r)
 				}
 			}()
-			if err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleEntrySize), rest); err != nil {
+			if _, err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleHeaderSize), rest); err != nil {
 				fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err)
 			}
 		}()
@@ -124,9 +125,9 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 		}
 		visitNeedle(n, offset)
 
-		offset += types.NeedleEntrySize + rest
+		offset += types.NeedleHeaderSize + rest
 		//fmt.Printf("==> new entry offset %d\n", offset)
-		if n, rest, err = storage.ReadNeedleHeader(datFile, version, offset); err != nil {
+		if n, _, rest, err = needle.ReadNeedleHeader(datFile, version, offset); err != nil {
 			if err == io.EOF {
 				return
 			}

unmaintained/load_test/load_test_leveldb.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"crypto/md5"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+var (
+	dir = flag.String("dir", "./t", "directory to store level db files")
+	useHash = flag.Bool("isHash", false, "hash the path as the key")
+	dbCount = flag.Int("dbCount", 1, "the number of leveldb")
+)
+
+func main() {
+
+	flag.Parse()
+
+	totalTenants := 300
+	totalYears := 3
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 4,
+	}
+
+	var dbs []*leveldb.DB
+	var chans []chan string
+	for d := 0 ; d < *dbCount; d++ {
+		dbFolder := fmt.Sprintf("%s/%02d", *dir, d)
+		os.MkdirAll(dbFolder, 0755)
+		db, err := leveldb.OpenFile(dbFolder, opts)
+		if err != nil {
+			log.Printf("filer store open dir %s: %v", *dir, err)
+			return
+		}
+		dbs = append(dbs, db)
+		chans = append(chans, make(chan string, 1024))
+	}
+
+	var wg sync.WaitGroup
+	for d := 0 ; d < *dbCount; d++ {
+		wg.Add(1)
+		go func(d int){
+			defer wg.Done()
+
+			ch := chans[d]
+			db := dbs[d]
+
+			for p := range ch {
+				if *useHash {
+					insertAsHash(db, p)
+				}else{
+					insertAsFullPath(db, p)
+				}
+			}
+		}(d)
+	}
+
+
+	counter := int64(0)
+	lastResetTime := time.Now()
+
+	r := rand.New(rand.NewSource(35))
+
+	for y := 0; y < totalYears; y++ {
+		for m := 0; m < 12; m++ {
+			for d := 0; d < 31; d++ {
+				for h := 0; h < 24; h++ {
+					for min := 0; min < 60; min++ {
+						for i := 0; i < totalTenants; i++ {
+							p := fmt.Sprintf("tenent%03d/%4d/%02d/%02d/%02d/%02d", i, 2015+y, 1+m, 1+d, h, min)
+
+							x := r.Intn(*dbCount)
+
+							chans[x] <- p
+
+							counter++
+						}
+
+						t := time.Now()
+						if lastResetTime.Add(time.Second).Before(t) {
+							p := fmt.Sprintf("%4d/%02d/%02d/%02d/%02d", 2015+y, 1+m, 1+d, h, min)
+							fmt.Printf("%s = %4d put/sec\n", p, counter)
+							counter = 0
+							lastResetTime = t
+						}
+					}
+				}
+			}
+		}
+	}
+
+	for d := 0 ; d < *dbCount; d++ {
+		close(chans[d])
+	}
+
+	wg.Wait()
+
+}
+
+func insertAsFullPath(db *leveldb.DB, p string) {
+	_, getErr := db.Get([]byte(p), nil)
+	if getErr == leveldb.ErrNotFound {
+		putErr := db.Put([]byte(p), []byte(p), nil)
+		if putErr != nil {
+			log.Printf("failed to put %s", p)
+		}
+	}
+}
+
+func insertAsHash(db *leveldb.DB, p string) {
+	key := fmt.Sprintf("%d:%s", hashToLong(p), p)
+	_, getErr := db.Get([]byte(key), nil)
+	if getErr == leveldb.ErrNotFound {
+		putErr := db.Put([]byte(key), []byte(p), nil)
+		if putErr != nil {
+			log.Printf("failed to put %s", p)
+		}
+	}
+}
+
+func hashToLong(dir string) (v int64) {
+	h := md5.New()
+	io.WriteString(h, dir)
+
+	b := h.Sum(nil)
+
+	v += int64(b[0])
+	v <<= 8
+	v += int64(b[1])
+	v <<= 8
+	v += int64(b[2])
+	v <<= 8
+	v += int64(b[3])
+	v <<= 8
+	v += int64(b[4])
+	v <<= 8
+	v += int64(b[5])
+	v <<= 8
+	v += int64(b[6])
+	v <<= 8
+	v += int64(b[7])
+
+	return
+}

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -4,12 +4,12 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
 	"log"
 	"math/rand"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
@@ -22,7 +22,7 @@ var (
 func main() {
 	flag.Parse()
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
 
 	for i := 0; i < *repeat; i++ {

unmaintained/see_dat/see_dat.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"flag"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+
 	"time"
 )
 
@@ -14,7 +17,7 @@ var (
 )
 
 type VolumeFileScanner4SeeDat struct {
-	version storage.Version
+	version needle.Version
 }
 
 func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
@@ -26,7 +29,7 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
 	return true
 }
 
-func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *storage.Needle, offset int64) error {
+func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64) error {
 	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
 	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x appendedAt %v", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie, t)
 	return nil
@@ -35,7 +38,7 @@ func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *storage.Needle, offset i
 func main() {
 	flag.Parse()
 
-	vid := storage.VolumeId(*volumeId)
+	vid := needle.VolumeId(*volumeId)
 
 	scanner := &VolumeFileScanner4SeeDat{}
 	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)

unmaintained/see_idx/see_idx.go

@@ -8,7 +8,7 @@ import (
 	"strconv"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/idx"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
@@ -35,7 +35,7 @@ func main() {
 	}
 	defer indexFile.Close()
 
-	storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
+	idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
 		fmt.Printf("key:%v offset:%v size:%v\n", key, offset, size)
 		return nil
 	})

unmaintained/see_meta/see_meta.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+)
+
+var (
+	metaFile = flag.String("meta", "", "meta file generated via fs.meta.save")
+)
+
+func main() {
+	flag.Parse()
+
+	dst, err := os.OpenFile(*metaFile, os.O_RDONLY, 0644)
+	if err != nil {
+		log.Fatalf("failed to open %s: %v", *metaFile, err)
+	}
+	defer dst.Close()
+
+	err = walkMetaFile(dst)
+	if err != nil {
+		log.Fatalf("failed to visit %s: %v", *metaFile, err)
+	}
+
+}
+
+func walkMetaFile(dst *os.File) error {
+
+	sizeBuf := make([]byte, 4)
+
+	for {
+		if n, err := dst.Read(sizeBuf); n != 4 {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+
+		size := util.BytesToUint32(sizeBuf)
+
+		data := make([]byte, int(size))
+
+		if n, err := dst.Read(data); n != len(data) {
+			return err
+		}
+
+		fullEntry := &filer_pb.FullEntry{}
+		if err := proto.Unmarshal(data, fullEntry); err != nil {
+			return err
+		}
+
+		fmt.Fprintf(os.Stdout, "file %s %v\n", filer2.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
+		for i, chunk := range fullEntry.Entry.Chunks {
+			fmt.Fprintf(os.Stdout, "  chunk %d %v\n", i+1, chunk.String())
+		}
+
+	}
+
+}

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	size        = flag.Int("size", 1024, "file size")
+	concurrency = flag.Int("c", 4, "concurrent number of uploads")
+	times       = flag.Int("n", 1024, "repeated number of times")
+	fileCount   = flag.Int("fileCount", 1, "number of files to write")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+
+	flag.Parse()
+
+	data := make([]byte, *size)
+	println("data len", len(data))
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		go func(x int) {
+			defer wg.Done()
+
+			client := &http.Client{Transport: &http.Transport{
+				MaxConnsPerHost:     1024,
+				MaxIdleConnsPerHost: 1024,
+			}}
+			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
+
+			for t := 0; t < *times; t++ {
+				for f := 0; f < *fileCount; f++ {
+					fn := r.Intn(*fileCount)
+					if size, err := uploadFileToFiler(client, data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					} else {
+						log.Fatalf("client %d upload %d times: %v", x, t, err)
+					}
+				}
+			}
+		}(x)
+	}
+
+	go func() {
+		ticker := time.NewTicker(1000 * time.Millisecond)
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(client *http.Client, data []byte, filename, destination string) (size int64, err error) {
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", filename)
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", filename, err)
+	}
+
+	part.Write(data)
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", filename, err)
+	}
+
+	uri := destination + filename
+
+	request, err := http.NewRequest("POST", uri, body)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	// request.Close = true  // can not use this, which do not reuse http connection, impacting filer->volume also.
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return int64(len(data)), nil
+}

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -0,0 +1,150 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	dir         = flag.String("dir", ".", "upload files under this directory")
+	concurrency = flag.Int("c", 1, "concurrent number of uploads")
+	times       = flag.Int("n", 1, "repeated number of times")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+
+	flag.Parse()
+
+	var fileNames []string
+
+	files, err := ioutil.ReadDir(*dir)
+	if err != nil {
+		log.Fatalf("fail to read dir %v: %v", *dir, err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		fileNames = append(fileNames, filepath.Join(*dir, file.Name()))
+	}
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		client := &http.Client{}
+
+		go func() {
+			defer wg.Done()
+			rand.Shuffle(len(fileNames), func(i, j int) {
+				fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
+			})
+			for t := 0; t < *times; t++ {
+				for _, filename := range fileNames {
+					if size, err := uploadFileToFiler(client, filename, *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					}
+				}
+			}
+		}()
+	}
+
+	go func() {
+		ticker := time.NewTicker(500 * time.Millisecond)
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(client *http.Client, filename, destination string) (size int64, err error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", file.Name())
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", file.Name(), err)
+	}
+	_, err = io.Copy(part, file)
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	uri := destination + file.Name()
+
+	request, err := http.NewRequest("POST", uri, body)
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+
+	resp, err := client.Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return fi.Size(), nil
+}

unmaintained/volume_tailer/volume_tailer.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	util2 "github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+	"golang.org/x/tools/godoc/util"
+)
+
+var (
+	master         = flag.String("master", "localhost:9333", "master server host and port")
+	volumeId       = flag.Int("volumeId", -1, "a volume id")
+	rewindDuration = flag.Duration("rewind", -1, "rewind back in time. -1 means from the first entry. 0 means from now.")
+	timeoutSeconds = flag.Int("timeoutSeconds", 0, "disconnect if no activity after these seconds")
+	showTextFile   = flag.Bool("showTextFile", false, "display textual file content")
+)
+
+func main() {
+	flag.Parse()
+
+	util2.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
+	vid := needle.VolumeId(*volumeId)
+
+	var sinceTimeNs int64
+	if *rewindDuration == 0 {
+		sinceTimeNs = time.Now().UnixNano()
+	} else if *rewindDuration == -1 {
+		sinceTimeNs = 0
+	} else if *rewindDuration > 0 {
+		sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
+	}
+
+	err := operation.TailVolume(*master, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
+		if n.Size == 0 {
+			println("-", n.String())
+			return nil
+		} else {
+			println("+", n.String())
+		}
+
+		if *showTextFile {
+
+			data := n.Data
+			if n.IsGzipped() {
+				if data, err = util2.UnGzipData(data); err != nil {
+					return err
+				}
+			}
+			if util.IsText(data) {
+				println(string(data))
+			}
+
+			println("-", n.String(), "compressed", n.IsGzipped(), "original size", len(data))
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("Error VolumeTailSender volume %d: %v", vid, err)
+	}
+
+}

util/gostd

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+############################  GLOBAL VARIABLES
+regex=' '
+branch="master"
+max_length=150
+
+REGEX_SUFFIX_GO=".+\.go$"
+
+############################  FUNCTIONS
+msg() {
+    printf '%b' "$1" >&2
+}
+
+die() {
+    msg "\33[31m[✘]\33[0m ${1}${2}"
+    exit 1
+}
+
+succ() {
+    msg "\33[34m[√]\33[0m ${1}${2}"
+}
+
+gostd() {
+    local branch=$1
+    local reg4exclude=$2
+    local max_length=$3
+
+    for file in `git diff $branch --name-only`
+    do
+        if ! [[ $file =~ $REGEX_SUFFIX_GO ]] || [[ $file =~ $reg4exclude ]]; then
+            continue
+        fi
+
+        error=`go fmt $file 2>&1`
+        if ! [ $? -eq 0 ]; then
+            die "go fmt $file:" "$error"
+        fi
+
+        succ "$file\n"
+
+        grep -n -E --color=always ".{$max_length}" $file | awk '{ printf ("%4s %s\n", "", $0) }'
+    done
+}
+
+get_options() {
+    while getopts "b:e:hl:" opts
+    do
+        case $opts in
+            b)
+                branch=$OPTARG
+                ;;
+            e)
+                regex=$OPTARG
+                ;;
+            h)
+                usage
+                exit 0
+                ;;
+            l)
+                max_length=$OPTARG
+                ;;
+            \?)
+                usage
+                exit 1
+                ;;
+        esac
+    done
+}
+
+usage () {
+    cat << _EOC_
+Usage:
+    gostd [options]
+
+Options:
+    -b <branch/commit>        Specify the git diff branch or commit.
+                              (default: master)
+    -e <regex>                Regex for excluding file or directory.
+    -h                        Print this usage.
+    -l <length>               Show files that exceed the limit line length.
+                              (default: 150)
+
+Examples:
+    gostd
+    gostd -b master -l 100
+    gostd -b 59d532a -e weed/pb -l 100
+_EOC_
+}
+
+main() {
+    get_options "$@"
+
+    gostd "$branch" "$regex" "$max_length"
+}
+
+############################  MAIN()
+main "$@"

weed/command/backup.go

@@ -2,8 +2,10 @@ package command
 
 import (
 	"fmt"
+
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/spf13/viper"
 
 	"github.com/chrislusf/seaweedfs/weed/operation"
@@ -33,16 +35,16 @@ var cmdBackup = &Command{
 	UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333",
 	Short:     "incrementally backup a volume to local folder",
 	Long: `Incrementally backup volume data.
-	
+
 	It is expected that you use this inside a script, to loop through
 	all possible volume ids that needs to be backup to local folder.
-	
+
 	The volume id does not need to exist locally or even remotely.
 	This will help to backup future new volumes.
-	
+
 	Usually backing up is just copying the .dat (and .idx) files.
 	But it's tricky to incrementally copy the differences.
-	
+
 	The complexity comes when there are multiple addition, deletion and compaction.
 	This tool will handle them correctly and efficiently, avoiding unnecessary data transportation.
   `,
@@ -50,13 +52,13 @@ var cmdBackup = &Command{
 
 func runBackup(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
 
 	if *s.volumeId == -1 {
 		return false
 	}
-	vid := storage.VolumeId(*s.volumeId)
+	vid := needle.VolumeId(*s.volumeId)
 
 	// find volume location, replication, ttl info
 	lookup, err := operation.Lookup(*s.master, vid.String())
@@ -71,7 +73,7 @@ func runBackup(cmd *Command, args []string) bool {
 		fmt.Printf("Error get volume %d status: %v\n", vid, err)
 		return true
 	}
-	ttl, err := storage.ReadTTL(stats.Ttl)
+	ttl, err := needle.ReadTTL(stats.Ttl)
 	if err != nil {
 		fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err)
 		return true
@@ -88,8 +90,8 @@ func runBackup(cmd *Command, args []string) bool {
 		return true
 	}
 
-	if v.SuperBlock.CompactRevision < uint16(stats.CompactRevision) {
-		if err = v.Compact(0); err != nil {
+	if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
+		if err = v.Compact(0, 0); err != nil {
 			fmt.Printf("Compact Volume before synchronizing %v\n", err)
 			return true
 		}
@@ -97,11 +99,13 @@ func runBackup(cmd *Command, args []string) bool {
 			fmt.Printf("Commit Compact before synchronizing %v\n", err)
 			return true
 		}
-		v.SuperBlock.CompactRevision = uint16(stats.CompactRevision)
+		v.SuperBlock.CompactionRevision = uint16(stats.CompactRevision)
 		v.DataFile().WriteAt(v.SuperBlock.Bytes(), 0)
 	}
 
-	if uint64(v.Size()) > stats.TailOffset {
+	datSize, _, _ := v.FileStat()
+
+	if datSize > stats.TailOffset {
 		// remove the old data
 		v.Destroy()
 		// recreate an empty volume
@@ -113,7 +117,7 @@ func runBackup(cmd *Command, args []string) bool {
 	}
 	defer v.Close()
 
-	if err := v.Follow(volumeServer, grpcDialOption); err != nil {
+	if err := v.IncrementalBackup(volumeServer, grpcDialOption); err != nil {
 		fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
 		return true
 	}

weed/command/benchmark.go

@@ -4,9 +4,6 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
-	"google.golang.org/grpc"
 	"io"
 	"math"
 	"math/rand"
@@ -18,6 +15,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/security"
@@ -108,7 +108,7 @@ var (
 
 func runBenchmark(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 	b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)

weed/command/command.go

@@ -13,7 +13,6 @@ var Commands = []*Command{
 	cmdCompact,
 	cmdCopy,
 	cmdFix,
-	cmdFilerExport,
 	cmdFilerReplicate,
 	cmdServer,
 	cmdMaster,
@@ -27,6 +26,7 @@ var Commands = []*Command{
 	cmdVolume,
 	cmdExport,
 	cmdMount,
+	cmdWebDav,
 }
 
 type Command struct {

weed/command/compact.go

@@ -3,6 +3,7 @@ package command
 import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 func init() {
@@ -35,14 +36,14 @@ func runCompact(cmd *Command, args []string) bool {
 
 	preallocate := *compactVolumePreallocate * (1 << 20)
 
-	vid := storage.VolumeId(*compactVolumeId)
+	vid := needle.VolumeId(*compactVolumeId)
 	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid,
 		storage.NeedleMapInMemory, nil, nil, preallocate)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}
 	if *compactMethod == 0 {
-		if err = v.Compact(preallocate); err != nil {
+		if err = v.Compact(preallocate, 0); err != nil {
 			glog.Fatalf("Compact Volume [ERROR] %s\n", err)
 		}
 	} else {

weed/command/export.go

@@ -12,10 +12,12 @@ import (
 	"text/template"
 	"time"
 
+	"io"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
-	"io"
 )
 
 const (
@@ -66,10 +68,10 @@ var (
 	localLocation, _             = time.LoadLocation("Local")
 )
 
-func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Version, deleted bool) {
-	key := storage.NewFileIdFromNeedle(vid, n).String()
+func printNeedle(vid needle.VolumeId, n *needle.Needle, version needle.Version, deleted bool) {
+	key := needle.NewFileIdFromNeedle(vid, n).String()
 	size := n.DataSize
-	if version == storage.Version1 {
+	if version == needle.Version1 {
 		size = n.Size
 	}
 	fmt.Printf("%s\t%s\t%d\t%t\t%s\t%s\t%s\t%t\n",
@@ -85,10 +87,10 @@ func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Versio
 }
 
 type VolumeFileScanner4Export struct {
-	version   storage.Version
+	version   needle.Version
 	counter   int
 	needleMap *storage.NeedleMap
-	vid       storage.VolumeId
+	vid       needle.VolumeId
 }
 
 func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
@@ -100,14 +102,14 @@ func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
 	return true
 }
 
-func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset int64) error {
+func (scanner *VolumeFileScanner4Export) VisitNeedle(n *needle.Needle, offset int64) error {
 	needleMap := scanner.needleMap
 	vid := scanner.vid
 
 	nv, ok := needleMap.Get(n.Id)
 	glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
 		n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
-	if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && int64(nv.Offset)*types.NeedlePaddingSize == offset {
+	if ok && nv.Size > 0 && nv.Size != types.TombstoneFileSize && nv.Offset.ToAcutalOffset() == offset {
 		if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
 			glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
 				n.LastModified, newerThanUnix)
@@ -189,7 +191,7 @@ func runExport(cmd *Command, args []string) bool {
 	if *export.collection != "" {
 		fileName = *export.collection + "_" + fileName
 	}
-	vid := storage.VolumeId(*export.volumeId)
+	vid := needle.VolumeId(*export.volumeId)
 	indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644)
 	if err != nil {
 		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
@@ -225,8 +227,8 @@ type nameParams struct {
 	Ext  string
 }
 
-func writeFile(vid storage.VolumeId, n *storage.Needle) (err error) {
-	key := storage.NewFileIdFromNeedle(vid, n).String()
+func writeFile(vid needle.VolumeId, n *needle.Needle) (err error) {
+	key := needle.NewFileIdFromNeedle(vid, n).String()
 	fileNameTemplateBuffer.Reset()
 	if err = fileNameTemplate.Execute(fileNameTemplateBuffer,
 		nameParams{

weed/command/filer.go

@@ -1,13 +1,14 @@
 package command
 
 import (
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/spf13/viper"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
@@ -23,7 +24,6 @@ type FilerOptions struct {
 	masters                 *string
 	ip                      *string
 	port                    *int
-	grpcPort                *int
 	publicPort              *int
 	collection              *string
 	defaultReplicaPlacement *string
@@ -45,8 +45,7 @@ func init() {
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
 	f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
-	f.grpcPort = cmdFiler.Flag.Int("port.grpc", 0, "filer grpc server listen port, default to http port + 10000")
-	f.publicPort = cmdFiler.Flag.Int("port.public", 0, "port opened to public")
+	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
 	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
@@ -79,7 +78,7 @@ var cmdFiler = &Command{
 
 func runFiler(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 
 	f.startFiler()
 
@@ -95,9 +94,9 @@ func (fo *FilerOptions) startFiler() {
 		publicVolumeMux = http.NewServeMux()
 	}
 
-	defaultLevelDbDirectory := "./filerdb"
+	defaultLevelDbDirectory := "./filerldb2"
 	if fo.defaultLevelDbDirectory != nil {
-		defaultLevelDbDirectory = *fo.defaultLevelDbDirectory + "/filerdb"
+		defaultLevelDbDirectory = *fo.defaultLevelDbDirectory + "/filerldb2"
 	}
 
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
@@ -111,6 +110,7 @@ func (fo *FilerOptions) startFiler() {
 		DataCenter:         *fo.dataCenter,
 		DefaultLevelDbDir:  defaultLevelDbDirectory,
 		DisableHttp:        *fo.disableHttp,
+		Port:               *fo.port,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -132,7 +132,7 @@ func (fo *FilerOptions) startFiler() {
 
 	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.VERSION, *fo.ip, *fo.port)
 	filerListener, e := util.NewListener(
-		":"+strconv.Itoa(*fo.port),
+		*fo.ip+":"+strconv.Itoa(*fo.port),
 		time.Duration(10)*time.Second,
 	)
 	if e != nil {
@@ -140,10 +140,7 @@ func (fo *FilerOptions) startFiler() {
 	}
 
 	// starting grpc server
-	grpcPort := *fo.grpcPort
-	if grpcPort == 0 {
-		grpcPort = *fo.port + 10000
-	}
+	grpcPort := *fo.port + 10000
 	grpcL, err := util.NewListener(":"+strconv.Itoa(grpcPort), 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)

weed/command/filer_copy.go

@@ -1,54 +1,56 @@
 package command
 
 import (
+	"context"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/chrislusf/seaweedfs/weed/wdclient"
-	"github.com/spf13/viper"
-	"google.golang.org/grpc"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"sync"
+	"time"
 
-	"context"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 )
 
 var (
-	copy CopyOptions
+	copy      CopyOptions
+	waitGroup sync.WaitGroup
 )
 
 type CopyOptions struct {
-	filerGrpcPort  *int
-	master         *string
-	include        *string
-	replication    *string
-	collection     *string
-	ttl            *string
-	maxMB          *int
-	grpcDialOption grpc.DialOption
-	masterClient   *wdclient.MasterClient
+	include          *string
+	replication      *string
+	collection       *string
+	ttl              *string
+	maxMB            *int
+	masterClient     *wdclient.MasterClient
+	concurrency      *int
+	compressionLevel *int
+	grpcDialOption   grpc.DialOption
+	masters          []string
 }
 
 func init() {
 	cmdCopy.Run = runCopy // break init cycle
 	cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information")
-	copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location")
 	copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir")
 	copy.replication = cmdCopy.Flag.String("replication", "", "replication type")
 	copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name")
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 	copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
-	copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
+	copy.concurrency = cmdCopy.Flag.Int("c", 8, "concurrent file copy goroutines")
+	copy.compressionLevel = cmdCopy.Flag.Int("compressionLevel", 9, "local file compression level 1 ~ 9")
 }
 
 var cmdCopy = &Command{
@@ -69,7 +71,7 @@ var cmdCopy = &Command{
 
 func runCopy(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 
 	if len(args) <= 1 {
 		return false
@@ -100,72 +102,170 @@ func runCopy(cmd *Command, args []string) bool {
 	}
 
 	filerGrpcPort := filerPort + 10000
-	if *copy.filerGrpcPort != 0 {
-		filerGrpcPort = uint64(*copy.filerGrpcPort)
-	}
-
 	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
 	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 
-	copy.masterClient = wdclient.NewMasterClient(context.Background(), copy.grpcDialOption, "client", strings.Split(*copy.master, ","))
+	ctx := context.Background()
+
+	masters, collection, replication, maxMB, err := readFilerConfiguration(ctx, copy.grpcDialOption, filerGrpcAddress)
+	if err != nil {
+		fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err)
+		return false
+	}
+	if *copy.collection == "" {
+		*copy.collection = collection
+	}
+	if *copy.replication == "" {
+		*copy.replication = replication
+	}
+	if *copy.maxMB == 0 {
+		*copy.maxMB = int(maxMB)
+	}
+	copy.masters = masters
+
+	copy.masterClient = wdclient.NewMasterClient(ctx, copy.grpcDialOption, "client", copy.masters)
 	go copy.masterClient.KeepConnectedToMaster()
 	copy.masterClient.WaitUntilConnected()
 
-	for _, fileOrDir := range fileOrDirs {
-		if !doEachCopy(context.Background(), fileOrDir, filerUrl.Host, filerGrpcAddress, copy.grpcDialOption, urlPath) {
-			return false
-		}
+	if *cmdCopy.IsDebug {
+		util.SetupProfiling("filer.copy.cpu.pprof", "filer.copy.mem.pprof")
 	}
+
+	fileCopyTaskChan := make(chan FileCopyTask, *copy.concurrency)
+
+	go func() {
+		defer close(fileCopyTaskChan)
+		for _, fileOrDir := range fileOrDirs {
+			if err := genFileCopyTask(fileOrDir, urlPath, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "gen file list error: %v\n", err)
+				break
+			}
+		}
+	}()
+	for i := 0; i < *copy.concurrency; i++ {
+		waitGroup.Add(1)
+		go func() {
+			defer waitGroup.Done()
+			worker := FileCopyWorker{
+				options:          &copy,
+				filerHost:        filerUrl.Host,
+				filerGrpcAddress: filerGrpcAddress,
+			}
+			if err := worker.copyFiles(ctx, fileCopyTaskChan); err != nil {
+				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
+				return
+			}
+		}()
+	}
+	waitGroup.Wait()
+
 	return true
 }
 
-func doEachCopy(ctx context.Context, fileOrDir string, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, path string) bool {
-	f, err := os.Open(fileOrDir)
-	if err != nil {
-		fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
-		return false
-	}
-	defer f.Close()
+func readFilerConfiguration(ctx context.Context, grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, maxMB uint32, err error) {
+	err = withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+		}
+		masters, collection, replication, maxMB = resp.Masters, resp.Collection, resp.Replication, resp.MaxMb
+		return nil
+	})
+	return
+}
 
-	fi, err := f.Stat()
+func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan FileCopyTask) error {
+
+	fi, err := os.Stat(fileOrDir)
 	if err != nil {
-		fmt.Printf("Failed to get stat for file %s: %v\n", fileOrDir, err)
-		return false
+		fmt.Fprintf(os.Stderr, "Failed to get stat for file %s: %v\n", fileOrDir, err)
+		return nil
 	}
 
 	mode := fi.Mode()
 	if mode.IsDir() {
 		files, _ := ioutil.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
-			if !doEachCopy(ctx, fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, grpcDialOption, path+fi.Name()+"/") {
-				return false
+			if err = genFileCopyTask(fileOrDir+"/"+subFileOrDir.Name(), destPath+fi.Name()+"/", fileCopyTaskChan); err != nil {
+				return err
 			}
 		}
-		return true
+		return nil
 	}
 
+	uid, gid := util.GetFileUidGid(fi)
+
+	fileCopyTaskChan <- FileCopyTask{
+		sourceLocation:     fileOrDir,
+		destinationUrlPath: destPath,
+		fileSize:           fi.Size(),
+		fileMode:           fi.Mode(),
+		uid:                uid,
+		gid:                gid,
+	}
+
+	return nil
+}
+
+type FileCopyWorker struct {
+	options          *CopyOptions
+	filerHost        string
+	filerGrpcAddress string
+}
+
+func (worker *FileCopyWorker) copyFiles(ctx context.Context, fileCopyTaskChan chan FileCopyTask) error {
+	for task := range fileCopyTaskChan {
+		if err := worker.doEachCopy(ctx, task); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type FileCopyTask struct {
+	sourceLocation     string
+	destinationUrlPath string
+	fileSize           int64
+	fileMode           os.FileMode
+	uid                uint32
+	gid                uint32
+}
+
+func (worker *FileCopyWorker) doEachCopy(ctx context.Context, task FileCopyTask) error {
+
+	f, err := os.Open(task.sourceLocation)
+	if err != nil {
+		fmt.Printf("Failed to open file %s: %v\n", task.sourceLocation, err)
+		if _, ok := err.(*os.PathError); ok {
+			fmt.Printf("skipping %s\n", task.sourceLocation)
+			return nil
+		}
+		return err
+	}
+	defer f.Close()
+
 	// this is a regular file
-	if *copy.include != "" {
-		if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok {
-			return true
+	if *worker.options.include != "" {
+		if ok, _ := filepath.Match(*worker.options.include, filepath.Base(task.sourceLocation)); !ok {
+			return nil
 		}
 	}
 
 	// find the chunk count
-	chunkSize := int64(*copy.maxMB * 1024 * 1024)
+	chunkSize := int64(*worker.options.maxMB * 1024 * 1024)
 	chunkCount := 1
-	if chunkSize > 0 && fi.Size() > chunkSize {
-		chunkCount = int(fi.Size()/chunkSize) + 1
+	if chunkSize > 0 && task.fileSize > chunkSize {
+		chunkCount = int(task.fileSize/chunkSize) + 1
 	}
 
 	if chunkCount == 1 {
-		return uploadFileAsOne(ctx, filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi)
+		return worker.uploadFileAsOne(ctx, task, f)
 	}
 
-	return uploadFileInChunks(ctx, filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi, chunkCount, chunkSize)
+	return worker.uploadFileInChunks(ctx, task, f, chunkCount, chunkSize)
 }
 
-func uploadFileAsOne(ctx context.Context, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo) bool {
+func (worker *FileCopyWorker) uploadFileAsOne(ctx context.Context, task FileCopyTask, f *os.File) error {
 
 	// upload the file content
 	fileName := filepath.Base(f.Name())
@@ -173,29 +273,27 @@ func uploadFileAsOne(ctx context.Context, filerAddress, filerGrpcAddress string,
 
 	var chunks []*filer_pb.FileChunk
 
-	if fi.Size() > 0 {
+	if task.fileSize > 0 {
 
 		// assign a volume
-		assignResult, err := operation.Assign(copy.masterClient.GetMaster(), grpcDialOption, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
-			Replication: *copy.replication,
-			Collection:  *copy.collection,
-			Ttl:         *copy.ttl,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
 		})
 		if err != nil {
-			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
+			fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
 		}
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
 
-		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth)
+		uploadResult, err := operation.UploadWithLocalCompressionLevel(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth, *worker.options.compressionLevel)
 		if err != nil {
-			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-			return false
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 		}
 		if uploadResult.Error != "" {
-			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-			return false
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
 		}
 		fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
 
@@ -207,25 +305,25 @@ func uploadFileAsOne(ctx context.Context, filerAddress, filerGrpcAddress string,
 			ETag:   uploadResult.ETag,
 		})
 
-		fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
+		fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
 	}
 
-	if err := withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
-			Directory: urlFolder,
+			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
 				Name: fileName,
 				Attributes: &filer_pb.FuseAttributes{
 					Crtime:      time.Now().Unix(),
 					Mtime:       time.Now().Unix(),
-					Gid:         uint32(os.Getgid()),
-					Uid:         uint32(os.Getuid()),
-					FileSize:    uint64(fi.Size()),
-					FileMode:    uint32(fi.Mode()),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
 					Mime:        mimeType,
-					Replication: *copy.replication,
-					Collection:  *copy.collection,
-					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
 				},
 				Chunks: chunks,
 			},
@@ -236,14 +334,13 @@ func uploadFileAsOne(ctx context.Context, filerAddress, filerGrpcAddress string,
 		}
 		return nil
 	}); err != nil {
-		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
-		return false
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
 	}
 
-	return true
+	return nil
 }
 
-func uploadFileInChunks(ctx context.Context, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
+func (worker *FileCopyWorker) uploadFileInChunks(ctx context.Context, task FileCopyTask, f *os.File, chunkCount int, chunkSize int64) error {
 
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
@@ -253,14 +350,14 @@ func uploadFileInChunks(ctx context.Context, filerAddress, filerGrpcAddress stri
 	for i := int64(0); i < int64(chunkCount); i++ {
 
 		// assign a volume
-		assignResult, err := operation.Assign(copy.masterClient.GetMaster(), grpcDialOption, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(worker.options.masterClient.GetMaster(), worker.options.grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
-			Replication: *copy.replication,
-			Collection:  *copy.collection,
-			Ttl:         *copy.ttl,
+			Replication: *worker.options.replication,
+			Collection:  *worker.options.collection,
+			Ttl:         *worker.options.ttl,
 		})
 		if err != nil {
-			fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err)
+			fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
 		}
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
@@ -270,12 +367,10 @@ func uploadFileInChunks(ctx context.Context, filerAddress, filerGrpcAddress stri
 			io.LimitReader(f, chunkSize),
 			false, "application/octet-stream", nil, assignResult.Auth)
 		if err != nil {
-			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
-			return false
+			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 		}
 		if uploadResult.Error != "" {
-			fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-			return false
+			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
 		}
 		chunks = append(chunks, &filer_pb.FileChunk{
 			FileId: assignResult.Fid,
@@ -287,22 +382,22 @@ func uploadFileInChunks(ctx context.Context, filerAddress, filerGrpcAddress stri
 		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 	}
 
-	if err := withFilerClient(ctx, filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(ctx, worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
-			Directory: urlFolder,
+			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
 				Name: fileName,
 				Attributes: &filer_pb.FuseAttributes{
 					Crtime:      time.Now().Unix(),
 					Mtime:       time.Now().Unix(),
-					Gid:         uint32(os.Getgid()),
-					Uid:         uint32(os.Getuid()),
-					FileSize:    uint64(fi.Size()),
-					FileMode:    uint32(fi.Mode()),
+					Gid:         task.gid,
+					Uid:         task.uid,
+					FileSize:    uint64(task.fileSize),
+					FileMode:    uint32(task.fileMode),
 					Mime:        mimeType,
-					Replication: *copy.replication,
-					Collection:  *copy.collection,
-					TtlSec:      int32(util.ParseInt(*copy.ttl, 0)),
+					Replication: *worker.options.replication,
+					Collection:  *worker.options.collection,
+					TtlSec:      int32(util.ParseInt(*worker.options.ttl, 0)),
 				},
 				Chunks: chunks,
 			},
@@ -313,13 +408,12 @@ func uploadFileInChunks(ctx context.Context, filerAddress, filerGrpcAddress stri
 		}
 		return nil
 	}); err != nil {
-		fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerAddress, urlFolder, fileName, err)
-		return false
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
 	}
 
-	fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
+	fmt.Printf("copied %s => http://%s%s%s\n", fileName, worker.filerHost, task.destinationUrlPath, fileName)
 
-	return true
+	return nil
 }
 
 func detectMimeType(f *os.File) string {
@@ -340,13 +434,9 @@ func detectMimeType(f *os.File) string {
 
 func withFilerClient(ctx context.Context, filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(ctx, filerAddress, grpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", filerAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(clientConn *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(clientConn)
+		return fn(client)
+	}, filerAddress, grpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }

weed/command/filer_export.go

@@ -1,191 +0,0 @@
-package command
-
-import (
-	"context"
-	"github.com/chrislusf/seaweedfs/weed/filer2"
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/notification"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
-)
-
-func init() {
-	cmdFilerExport.Run = runFilerExport // break init cycle
-}
-
-var cmdFilerExport = &Command{
-	UsageLine: "filer.export -sourceStore=mysql -targetStore=cassandra",
-	Short:     "export meta data in filer store",
-	Long: `Iterate the file tree and export all metadata out
-
-	Both source and target store:
-        * should be a store name already specified in filer.toml
-        * do not need to be enabled state
-
-	If target store is empty, only the directory tree will be listed.
-
-	If target store is "notification", the list of entries will be sent to notification.
-	This is usually used to bootstrap filer replication to a remote system.
-
-  `,
-}
-
-var (
-	// filerExportOutputFile  = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
-	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
-	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
-	dir                    = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
-	dirListLimit           = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
-	dryRun                 = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
-	verboseFilerExport     = cmdFilerExport.Flag.Bool("v", false, "verbose entry details")
-)
-
-type statistics struct {
-	directoryCount int
-	fileCount      int
-}
-
-func runFilerExport(cmd *Command, args []string) bool {
-
-	weed_server.LoadConfiguration("filer", true)
-	config := viper.GetViper()
-
-	var sourceStore, targetStore filer2.FilerStore
-
-	for _, store := range filer2.Stores {
-		if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
-			viperSub := config.Sub(store.GetName())
-			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize source store for %s: %+v",
-					store.GetName(), err)
-			} else {
-				sourceStore = store
-			}
-			break
-		}
-	}
-
-	for _, store := range filer2.Stores {
-		if store.GetName() == *filerExportTargetStore {
-			viperSub := config.Sub(store.GetName())
-			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize target store for %s: %+v",
-					store.GetName(), err)
-			} else {
-				targetStore = store
-			}
-			break
-		}
-	}
-
-	if sourceStore == nil {
-		glog.Errorf("Failed to find source store %s", *filerExportSourceStore)
-		println("existing data sources are:")
-		for _, store := range filer2.Stores {
-			println("    " + store.GetName())
-		}
-		return false
-	}
-
-	if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
-		glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
-		println("existing data sources are:")
-		for _, store := range filer2.Stores {
-			println("    " + store.GetName())
-		}
-		return false
-	}
-
-	ctx := context.Background()
-
-	stat := statistics{}
-
-	var fn func(level int, entry *filer2.Entry) error
-
-	if *filerExportTargetStore == "notification" {
-		weed_server.LoadConfiguration("notification", false)
-		v := viper.GetViper()
-		notification.LoadConfiguration(v.Sub("notification"))
-
-		fn = func(level int, entry *filer2.Entry) error {
-			printout(level, entry)
-			if *dryRun {
-				return nil
-			}
-			return notification.Queue.SendMessage(
-				string(entry.FullPath),
-				&filer_pb.EventNotification{
-					NewEntry: entry.ToProtoEntry(),
-				},
-			)
-		}
-	} else if targetStore == nil {
-		fn = printout
-	} else {
-		fn = func(level int, entry *filer2.Entry) error {
-			printout(level, entry)
-			if *dryRun {
-				return nil
-			}
-			return targetStore.InsertEntry(ctx, entry)
-		}
-	}
-
-	doTraverse(ctx, &stat, sourceStore, filer2.FullPath(*dir), 0, fn)
-
-	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
-
-	return true
-}
-
-func doTraverse(ctx context.Context, stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
-
-	limit := *dirListLimit
-	lastEntryName := ""
-	for {
-		entries, err := filerStore.ListDirectoryEntries(ctx, parentPath, lastEntryName, false, limit)
-		if err != nil {
-			break
-		}
-		for _, entry := range entries {
-			if fnErr := fn(level, entry); fnErr != nil {
-				glog.Errorf("failed to process entry: %s", entry.FullPath)
-			}
-			if entry.IsDirectory() {
-				stat.directoryCount++
-				doTraverse(ctx, stat, filerStore, entry.FullPath, level+1, fn)
-			} else {
-				stat.fileCount++
-			}
-			lastEntryName = entry.Name()
-		}
-		if len(entries) < limit {
-			break
-		}
-	}
-}
-
-func printout(level int, entry *filer2.Entry) error {
-	for i := 0; i < level; i++ {
-		if i == level-1 {
-			print("+-")
-		} else {
-			print("| ")
-		}
-	}
-	print(entry.FullPath.Name())
-	if *verboseFilerExport {
-		for _, chunk := range entry.Chunks {
-			print("[")
-			print(chunk.FileId)
-			print(",")
-			print(chunk.Offset)
-			print(",")
-			print(chunk.Size)
-			print(")")
-		}
-	}
-	println()
-	return nil
-}

weed/command/filer_replication.go

@@ -13,7 +13,7 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
 	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
 	"github.com/chrislusf/seaweedfs/weed/replication/sub"
-	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/spf13/viper"
 )
 
@@ -36,9 +36,9 @@ var cmdFilerReplicate = &Command{
 
 func runFilerReplicate(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
-	weed_server.LoadConfiguration("replication", true)
-	weed_server.LoadConfiguration("notification", true)
+	util.LoadConfiguration("security", false)
+	util.LoadConfiguration("replication", true)
+	util.LoadConfiguration("notification", true)
 	config := viper.GetViper()
 
 	var notificationInput sub.NotificationInput

weed/command/fix.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
@@ -29,7 +30,7 @@ var (
 )
 
 type VolumeFileScanner4Fix struct {
-	version storage.Version
+	version needle.Version
 	nm      *storage.NeedleMap
 }
 
@@ -42,14 +43,14 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
 	return false
 }
 
-func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
+func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64) error {
 	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
 	if n.Size > 0 && n.Size != types.TombstoneFileSize {
-		pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
+		pe := scanner.nm.Put(n.Id, types.ToOffset(offset), n.Size)
 		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
 	} else {
 		glog.V(2).Infof("skipping deleted file ...")
-		return scanner.nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
+		return scanner.nm.Delete(n.Id, types.ToOffset(offset))
 	}
 	return nil
 }
@@ -74,7 +75,7 @@ func runFix(cmd *Command, args []string) bool {
 	nm := storage.NewBtreeNeedleMap(indexFile)
 	defer nm.Close()
 
-	vid := storage.VolumeId(*fixVolumeId)
+	vid := needle.VolumeId(*fixVolumeId)
 	scanner := &VolumeFileScanner4Fix{
 		nm: nm,
 	}

weed/command/master.go

@@ -1,6 +1,12 @@
 package command
 
 import (
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+
 	"github.com/chrislusf/raft/protobuf"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
@@ -10,15 +16,45 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc/reflection"
-	"net/http"
-	"os"
-	"runtime"
-	"strconv"
-	"strings"
 )
 
+var (
+	m MasterOptions
+)
+
+type MasterOptions struct {
+	port               *int
+	ip                 *string
+	ipBind             *string
+	metaFolder         *string
+	peers              *string
+	volumeSizeLimitMB  *uint
+	volumePreallocate  *bool
+	pulseSeconds       *int
+	defaultReplication *string
+	garbageThreshold   *float64
+	whiteList          *string
+	disableHttp        *bool
+	metricsAddress     *string
+	metricsIntervalSec *int
+}
+
 func init() {
 	cmdMaster.Run = runMaster // break init cycle
+	m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
+	m.ip = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
+	m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
+	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
+	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
+	m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
+	m.defaultReplication = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
+	m.garbageThreshold = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
+	m.whiteList = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
+	m.disableHttp = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
+	m.metricsAddress = cmdMaster.Flag.String("metrics.address", "", "Prometheus gateway address")
+	m.metricsIntervalSec = cmdMaster.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 }
 
 var cmdMaster = &Command{
@@ -34,55 +70,34 @@ var cmdMaster = &Command{
 }
 
 var (
-	mport                   = cmdMaster.Flag.Int("port", 9333, "http listen port")
-	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
-	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
-	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
-	masterPeers             = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
-	volumeSizeLimitMB       = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
-	volumePreallocate       = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
-	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
-	// mTimeout                = cmdMaster.Flag.Int("idleTimeout", 30, "connection idle seconds")
-	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	garbageThreshold      = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
-	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	disableHttp           = cmdMaster.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
-	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
-	masterMemProfile      = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
+	masterCpuProfile = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
+	masterMemProfile = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
 
 	masterWhiteList []string
 )
 
 func runMaster(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
+	util.LoadConfiguration("master", false)
 
-	if *mMaxCpu < 1 {
-		*mMaxCpu = runtime.NumCPU()
-	}
-	runtime.GOMAXPROCS(*mMaxCpu)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 	util.SetupProfiling(*masterCpuProfile, *masterMemProfile)
 
-	if err := util.TestFolderWritable(*metaFolder); err != nil {
-		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err)
+	if err := util.TestFolderWritable(*m.metaFolder); err != nil {
+		glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *m.metaFolder, err)
 	}
-	if *masterWhiteListOption != "" {
-		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
+	if *m.whiteList != "" {
+		masterWhiteList = strings.Split(*m.whiteList, ",")
 	}
-	if *volumeSizeLimitMB > 30*1000 {
+	if *m.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
 		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
 	}
 
 	r := mux.NewRouter()
-	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
-		*volumeSizeLimitMB, *volumePreallocate,
-		*mpulse, *defaultReplicaPlacement, *garbageThreshold,
-		masterWhiteList,
-		*disableHttp,
-	)
+	ms := weed_server.NewMasterServer(r, m.toMasterOption(masterWhiteList))
 
-	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
+	listeningAddress := *m.ipBind + ":" + strconv.Itoa(*m.port)
 
 	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
 
@@ -93,15 +108,18 @@ func runMaster(cmd *Command, args []string) bool {
 
 	go func() {
 		// start raftServer
-		myMasterAddress, peers := checkPeers(*masterIp, *mport, *masterPeers)
+		myMasterAddress, peers := checkPeers(*m.ip, *m.port, *m.peers)
 		raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
-			peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
+			peers, myMasterAddress, *m.metaFolder, ms.Topo, *m.pulseSeconds)
+		if raftServer == nil {
+			glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717", *m.metaFolder)
+		}
 		ms.SetRaftServer(raftServer)
 		r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 
 		// starting grpc server
-		grpcPort := *mport + 10000
-		grpcL, err := util.NewListener(*masterBindIp+":"+strconv.Itoa(grpcPort), 0)
+		grpcPort := *m.port + 10000
+		grpcL, err := util.NewListener(*m.ipBind+":"+strconv.Itoa(grpcPort), 0)
 		if err != nil {
 			glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 		}
@@ -111,7 +129,7 @@ func runMaster(cmd *Command, args []string) bool {
 		protobuf.RegisterRaftServer(grpcS, raftServer)
 		reflection.Register(grpcS)
 
-		glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *masterBindIp, grpcPort)
+		glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *m.ipBind, grpcPort)
 		grpcS.Serve(grpcL)
 	}()
 
@@ -147,3 +165,19 @@ func checkPeers(masterIp string, masterPort int, peers string) (masterAddress st
 	}
 	return
 }
+
+func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
+	return &weed_server.MasterOption{
+		Port:                    *m.port,
+		MetaFolder:              *m.metaFolder,
+		VolumeSizeLimitMB:       *m.volumeSizeLimitMB,
+		VolumePreallocate:       *m.volumePreallocate,
+		PulseSeconds:            *m.pulseSeconds,
+		DefaultReplicaPlacement: *m.defaultReplication,
+		GarbageThreshold:        *m.garbageThreshold,
+		WhiteList:               whiteList,
+		DisableHttp:             *m.disableHttp,
+		MetricsAddress:          *m.metricsAddress,
+		MetricsIntervalSec:      *m.metricsIntervalSec,
+	}
+}

weed/command/mount_std.go

@@ -4,16 +4,18 @@ package command
 
 import (
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
 	"os"
 	"os/user"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/jacobsa/daemonize"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -23,41 +25,66 @@ import (
 
 func runMount(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
+
+	return RunMount(
+		*mountOptions.filer,
+		*mountOptions.filerMountRootPath,
+		*mountOptions.dir,
+		*mountOptions.collection,
+		*mountOptions.replication,
+		*mountOptions.dataCenter,
+		*mountOptions.chunkSizeLimitMB,
+		*mountOptions.allowOthers,
+		*mountOptions.ttlSec,
+		*mountOptions.dirListingLimit,
+	)
+}
+
+func RunMount(filer, filerMountRootPath, dir, collection, replication, dataCenter string, chunkSizeLimitMB int,
+	allowOthers bool, ttlSec int, dirListingLimit int) bool {
+
+	util.LoadConfiguration("security", false)
 
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
-	if *mountOptions.dir == "" {
+	if dir == "" {
 		fmt.Printf("Please specify the mount directory via \"-dir\"")
 		return false
 	}
-	if *mountOptions.chunkSizeLimitMB <= 0 {
+	if chunkSizeLimitMB <= 0 {
 		fmt.Printf("Please specify a reasonable buffer size.")
 		return false
 	}
 
-	fuse.Unmount(*mountOptions.dir)
+	fuse.Unmount(dir)
+
+	uid, gid := uint32(0), uint32(0)
 
 	// detect mount folder mode
 	mountMode := os.ModeDir | 0755
-	if fileInfo, err := os.Stat(*mountOptions.dir); err == nil {
+	fileInfo, err := os.Stat(dir)
+	if err == nil {
 		mountMode = os.ModeDir | fileInfo.Mode()
+		uid, gid = util.GetFileUidGid(fileInfo)
+		fmt.Printf("mount point owner uid=%d gid=%d mode=%s\n", uid, gid, fileInfo.Mode())
 	}
 
-	// detect current user
-	uid, gid := uint32(0), uint32(0)
-	if u, err := user.Current(); err == nil {
-		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
-			uid = uint32(parsedId)
-		}
-		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
-			gid = uint32(parsedId)
+	if uid == 0 {
+		if u, err := user.Current(); err == nil {
+			if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+				uid = uint32(parsedId)
+			}
+			if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+				gid = uint32(parsedId)
+			}
+			fmt.Printf("current uid=%d gid=%d\n", uid, gid)
 		}
 	}
 
-	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
+	mountName := path.Base(dir)
 
 	options := []fuse.MountOption{
-		fuse.VolumeName("SeaweedFS"),
+		fuse.VolumeName(mountName),
 		fuse.FSName("SeaweedFS"),
 		fuse.Subtype("SeaweedFS"),
 		fuse.NoAppleDouble(),
@@ -71,56 +98,64 @@ func runMount(cmd *Command, args []string) bool {
 		fuse.MaxReadahead(1024 * 128),
 		fuse.AsyncRead(),
 		fuse.WritebackCache(),
+		fuse.AllowNonEmptyMount(),
 	}
-	if *mountOptions.allowOthers {
+	if allowOthers {
 		options = append(options, fuse.AllowOther())
 	}
 
-	c, err := fuse.Mount(*mountOptions.dir, options...)
+	c, err := fuse.Mount(dir, options...)
 	if err != nil {
 		glog.Fatal(err)
+		daemonize.SignalOutcome(err)
 		return false
 	}
 
 	util.OnInterrupt(func() {
-		fuse.Unmount(*mountOptions.dir)
+		fuse.Unmount(dir)
 		c.Close()
 	})
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*mountOptions.filer)
+	filerGrpcAddress, err := parseFilerGrpcAddress(filer)
 	if err != nil {
 		glog.Fatal(err)
+		daemonize.SignalOutcome(err)
 		return false
 	}
 
-	mountRoot := *mountOptions.filerMountRootPath
+	mountRoot := filerMountRootPath
 	if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
 		mountRoot = mountRoot[0 : len(mountRoot)-1]
 	}
 
+	daemonize.SignalOutcome(nil)
+
 	err = fs.Serve(c, filesys.NewSeaweedFileSystem(&filesys.Option{
 		FilerGrpcAddress:   filerGrpcAddress,
 		GrpcDialOption:     security.LoadClientTLS(viper.Sub("grpc"), "client"),
 		FilerMountRootPath: mountRoot,
-		Collection:         *mountOptions.collection,
-		Replication:        *mountOptions.replication,
-		TtlSec:             int32(*mountOptions.ttlSec),
-		ChunkSizeLimit:     int64(*mountOptions.chunkSizeLimitMB) * 1024 * 1024,
-		DataCenter:         *mountOptions.dataCenter,
-		DirListingLimit:    *mountOptions.dirListingLimit,
+		Collection:         collection,
+		Replication:        replication,
+		TtlSec:             int32(ttlSec),
+		ChunkSizeLimit:     int64(chunkSizeLimitMB) * 1024 * 1024,
+		DataCenter:         dataCenter,
+		DirListingLimit:    dirListingLimit,
 		EntryCacheTtl:      3 * time.Second,
 		MountUid:           uid,
 		MountGid:           gid,
 		MountMode:          mountMode,
+		MountCtime:         fileInfo.ModTime(),
+		MountMtime:         time.Now(),
 	}))
 	if err != nil {
-		fuse.Unmount(*mountOptions.dir)
+		fuse.Unmount(dir)
 	}
 
 	// check if the mount process has an error to report
 	<-c.Ready
 	if err := c.MountError; err != nil {
 		glog.Fatal(err)
+		daemonize.SignalOutcome(err)
 	}
 
 	return true

weed/command/s3.go

@@ -1,13 +1,14 @@
 package command
 
 import (
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
 	"net/http"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"fmt"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/s3api"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -15,12 +16,11 @@ import (
 )
 
 var (
-	s3options S3Options
+	s3StandaloneOptions S3Options
 )
 
 type S3Options struct {
 	filer            *string
-	filerGrpcPort    *int
 	filerBucketsPath *string
 	port             *int
 	domainName       *string
@@ -30,13 +30,12 @@ type S3Options struct {
 
 func init() {
 	cmdS3.Run = runS3 // break init cycle
-	s3options.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
-	s3options.filerGrpcPort = cmdS3.Flag.Int("filer.grpcPort", 0, "filer server grpc port, default to filer http port plus 10000")
-	s3options.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
-	s3options.port = cmdS3.Flag.Int("port", 8333, "s3options server http listen port")
-	s3options.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
-	s3options.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
-	s3options.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
+	s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
+	s3StandaloneOptions.filerBucketsPath = cmdS3.Flag.String("filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
+	s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
+	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}")
+	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
+	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 }
 
 var cmdS3 = &Command{
@@ -49,9 +48,15 @@ var cmdS3 = &Command{
 
 func runS3(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 
-	filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer)
+	return s3StandaloneOptions.startS3Server()
+
+}
+
+func (s3opt *S3Options) startS3Server() bool {
+
+	filerGrpcAddress, err := parseFilerGrpcAddress(*s3opt.filer)
 	if err != nil {
 		glog.Fatal(err)
 		return false
@@ -60,10 +65,10 @@ func runS3(cmd *Command, args []string) bool {
 	router := mux.NewRouter().SkipClean(true)
 
 	_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
-		Filer:            *s3options.filer,
+		Filer:            *s3opt.filer,
 		FilerGrpcAddress: filerGrpcAddress,
-		DomainName:       *s3options.domainName,
-		BucketsPath:      *s3options.filerBucketsPath,
+		DomainName:       *s3opt.domainName,
+		BucketsPath:      *s3opt.filerBucketsPath,
 		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
 	})
 	if s3ApiServer_err != nil {
@@ -72,22 +77,22 @@ func runS3(cmd *Command, args []string) bool {
 
 	httpS := &http.Server{Handler: router}
 
-	listenAddress := fmt.Sprintf(":%d", *s3options.port)
+	listenAddress := fmt.Sprintf(":%d", *s3opt.port)
 	s3ApiListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
 	if err != nil {
 		glog.Fatalf("S3 API Server listener on %s error: %v", listenAddress, err)
 	}
 
-	if *s3options.tlsPrivateKey != "" {
-		if err = httpS.ServeTLS(s3ApiListener, *s3options.tlsCertificate, *s3options.tlsPrivateKey); err != nil {
+	if *s3opt.tlsPrivateKey != "" {
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.VERSION, *s3opt.port)
+		if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
-		glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.VERSION, *s3options.port)
 	} else {
+		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.VERSION, *s3opt.port)
 		if err = httpS.Serve(s3ApiListener); err != nil {
 			glog.Fatalf("S3 API Server Fail to serve: %v", err)
 		}
-		glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.VERSION, *s3options.port)
 	}
 
 	return true

weed/command/scaffold.go

@@ -10,7 +10,7 @@ func init() {
 }
 
 var cmdScaffold = &Command{
-	UsageLine: "scaffold -config=[filer|notification|replication|security]",
+	UsageLine: "scaffold -config=[filer|notification|replication|security|master]",
 	Short:     "generate basic configuration files",
 	Long: `Generate filer.toml with all possible configurations for you to customize.
 
@@ -19,7 +19,7 @@ var cmdScaffold = &Command{
 
 var (
 	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
-	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security] the configuration file to generate")
+	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security|master] the configuration file to generate")
 )
 
 func runScaffold(cmd *Command, args []string) bool {
@@ -34,6 +34,8 @@ func runScaffold(cmd *Command, args []string) bool {
 		content = REPLICATION_TOML_EXAMPLE
 	case "security":
 		content = SECURITY_TOML_EXAMPLE
+	case "master":
+		content = MASTER_TOML_EXAMPLE
 	}
 	if content == "" {
 		println("need a valid -config option")
@@ -63,6 +65,12 @@ enabled = false
 
 [leveldb]
 # local on disk, mostly for simple single-machine setup, fairly scalable
+enabled = false
+dir = "."					# directory to store level db files
+
+[leveldb2]
+# local on disk, mostly for simple single-machine setup, fairly scalable
+# faster than previous leveldb, recommended.
 enabled = true
 dir = "."					# directory to store level db files
 
@@ -72,12 +80,13 @@ dir = "."					# directory to store level db files
 
 [mysql]
 # CREATE TABLE IF NOT EXISTS filemeta (
-#   dirhash     BIGINT        COMMENT 'first 64 bits of MD5 hash value of directory field',
-#   name        VARCHAR(1000) COMMENT 'directory or file name',
-#   directory   VARCHAR(4096) COMMENT 'full path to parent directory',
-#   meta        BLOB,
+#   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
+#   name        VARCHAR(1000)  COMMENT 'directory or file name',
+#   directory   TEXT           COMMENT 'full path to parent directory',
+#   meta        LONGBLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
+
 enabled = false
 hostname = "localhost"
 port = 3306
@@ -90,8 +99,8 @@ connection_max_open = 100
 [postgres]
 # CREATE TABLE IF NOT EXISTS filemeta (
 #   dirhash     BIGINT,
-#   name        VARCHAR(1000),
-#   directory   VARCHAR(4096),
+#   name        VARCHAR(65535),
+#   directory   VARCHAR(65535),
 #   meta        bytea,
 #   PRIMARY KEY (dirhash, name)
 # );
@@ -134,6 +143,7 @@ addresses = [
     "localhost:30005",
     "localhost:30006",
 ]
+password = ""
 
 `
 
@@ -190,6 +200,7 @@ enabled = false
 # The exchange must have already been created by some other means, like
 # the RabbitMQ management plugin.
 topic_url = "rabbit://myexchange"
+sub_url = "rabbit://myqueue"
 `
 
 	REPLICATION_TOML_EXAMPLE = `
@@ -260,13 +271,20 @@ directory = "/"                # destination directory
 #    /etc/seaweedfs/security.toml
 # this file is read by master, volume server, and filer
 
-# the jwt signing key is read by master and volume server
-# a jwt expires in 10 seconds
+# the jwt signing key is read by master and volume server.
+# a jwt defaults to expire after 10 seconds.
 [jwt.signing]
 key = ""
+expires_after_seconds = 10           # seconds
+
+# jwt for read is only supported with master+volume setup. Filer does not support this mode.
+[jwt.signing.read]
+key = ""
+expires_after_seconds = 10           # seconds
 
 # all grpc tls authentications are mutual
 # the values for the following ca, cert, and key are paths to the PERM files.
+# the host name is not checked, so the PERM files can be shared.
 [grpc]
 ca = ""
 
@@ -299,5 +317,24 @@ cert = ""
 key  = ""
 
 
+`
+
+	MASTER_TOML_EXAMPLE = `
+# Put this file to one of the location, with descending priority
+#    ./master.toml
+#    $HOME/.seaweedfs/master.toml
+#    /etc/seaweedfs/master.toml
+# this file is read by master
+
+[master.maintenance]
+# periodically run these scripts are the same as running them from 'weed shell'
+scripts = """
+  ec.encode -fullPercent=95 -quietFor=1h
+  ec.rebuild -force
+  ec.balance -force
+  volume.balance -force
+"""
+sleep_minutes = 17          # sleep minutes between each script execution
+
 `
 )

weed/command/server.go

@@ -1,9 +1,7 @@
 package command
 
 import (
-	"github.com/chrislusf/raft/protobuf"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/spf13/viper"
+	"fmt"
 	"net/http"
 	"os"
 	"runtime"
@@ -13,6 +11,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/chrislusf/raft/protobuf"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
@@ -28,7 +30,9 @@ type ServerOptions struct {
 
 var (
 	serverOptions ServerOptions
+	masterOptions MasterOptions
 	filerOptions  FilerOptions
+	s3Options     S3Options
 )
 
 func init() {
@@ -37,50 +41,52 @@ func init() {
 
 var cmdServer = &Command{
 	UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
-	Short:     "start a server, including volume server, and automatically elect a master server",
+	Short:     "start a master server, a volume server, and optionally a filer and a S3 gateway",
 	Long: `start both a volume server to provide storage spaces
   and a master server to provide volume=>location mapping service and sequence number of file ids
 
   This is provided as a convenient way to start both volume server and master server.
-  The servers are exactly the same as starting them separately.
+  The servers acts exactly the same as starting them separately.
+  So other volume servers can connect to this master server also.
 
-  So other volume servers can use this embedded master server also.
-
-  Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
-  They run with meta data on disk, not shared. So each filer server is different.
+  Optionally, a filer server can be started.
+  Also optionally, a S3 gateway can be started.
 
   `,
 }
 
 var (
-	serverIp                      = cmdServer.Flag.String("ip", "localhost", "ip or server name")
-	serverBindIp                  = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
-	serverMaxCpu                  = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	serverTimeout                 = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
-	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
-	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
-	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	serverDisableHttp             = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
-	serverPeers                   = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
-	serverGarbageThreshold        = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
-	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
-	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
-	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
-	masterVolumePreallocate       = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
-	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
-	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
-	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
-	pulseSeconds                  = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")
+	serverIp                  = cmdServer.Flag.String("ip", "localhost", "ip or server name")
+	serverBindIp              = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
+	serverTimeout             = cmdServer.Flag.Int("idleTimeout", 30, "connection idle seconds")
+	serverDataCenter          = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
+	serverRack                = cmdServer.Flag.String("rack", "", "current volume server's rack name")
+	serverWhiteListOption     = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
+	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
+	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
+	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
+	pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
+	isStartingFiler           = cmdServer.Flag.Bool("filer", false, "whether to start filer")
+	isStartingS3              = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
 
 	serverWhiteList []string
 )
 
 func init() {
 	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
+
+	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
+	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
+	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
+	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
+	masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "000", "Default replication type if not specified.")
+	masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
+	masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
+	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
+
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
-	filerOptions.grpcPort = cmdServer.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to http port + 10000")
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
 	filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
@@ -90,16 +96,24 @@ func init() {
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
-	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
+	serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
 	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
 
+	s3Options.filerBucketsPath = cmdServer.Flag.String("s3.filer.dir.buckets", "/buckets", "folder on filer to store all buckets")
+	s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
+	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}")
+	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
+	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
+
 }
 
 func runServer(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
+	util.LoadConfiguration("master", false)
 
 	if *serverOptions.cpuprofile != "" {
 		f, err := os.Create(*serverOptions.cpuprofile)
@@ -114,43 +128,53 @@ func runServer(cmd *Command, args []string) bool {
 		*isStartingFiler = true
 	}
 
-	master := *serverIp + ":" + strconv.Itoa(*masterPort)
+	if *isStartingS3 {
+		*isStartingFiler = true
+	}
+
+	master := *serverIp + ":" + strconv.Itoa(*masterOptions.port)
+	masterOptions.ip = serverIp
+	masterOptions.ipBind = serverBindIp
 	filerOptions.masters = &master
-	filerOptions.ip = serverIp
+	filerOptions.ip = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
 	serverOptions.v.masters = &master
 	serverOptions.v.idleConnectionTimeout = serverTimeout
-	serverOptions.v.maxCpu = serverMaxCpu
 	serverOptions.v.dataCenter = serverDataCenter
 	serverOptions.v.rack = serverRack
+
 	serverOptions.v.pulseSeconds = pulseSeconds
+	masterOptions.pulseSeconds = pulseSeconds
+
+	masterOptions.whiteList = serverWhiteListOption
 
 	filerOptions.dataCenter = serverDataCenter
 	filerOptions.disableHttp = serverDisableHttp
+	masterOptions.disableHttp = serverDisableHttp
+
+	filerAddress := fmt.Sprintf("%s:%d", *serverIp, *filerOptions.port)
+	s3Options.filer = &filerAddress
 
 	if *filerOptions.defaultReplicaPlacement == "" {
-		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
+		*filerOptions.defaultReplicaPlacement = *masterOptions.defaultReplication
 	}
 
-	if *serverMaxCpu < 1 {
-		*serverMaxCpu = runtime.NumCPU()
-	}
-	runtime.GOMAXPROCS(*serverMaxCpu)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	folders := strings.Split(*volumeDataFolders, ",")
 
-	if *masterVolumeSizeLimitMB > 30*1000 {
+	if *masterOptions.volumeSizeLimitMB > util.VolumeSizeLimitGB*1000 {
 		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
 	}
 
-	if *masterMetaFolder == "" {
-		*masterMetaFolder = folders[0]
+	if *masterOptions.metaFolder == "" {
+		*masterOptions.metaFolder = folders[0]
 	}
-	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
-		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
+	if err := util.TestFolderWritable(*masterOptions.metaFolder); err != nil {
+		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterOptions.metaFolder, err)
 	}
-	filerOptions.defaultLevelDbDirectory = masterMetaFolder
+	filerOptions.defaultLevelDbDirectory = masterOptions.metaFolder
 
 	if *serverWhiteListOption != "" {
 		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
@@ -165,40 +189,45 @@ func runServer(cmd *Command, args []string) bool {
 		}()
 	}
 
+	if *isStartingS3 {
+		go func() {
+			time.Sleep(2 * time.Second)
+
+			s3Options.startS3Server()
+
+		}()
+	}
+
 	var volumeWait sync.WaitGroup
 
 	volumeWait.Add(1)
 
 	go func() {
 		r := mux.NewRouter()
-		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
-			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
-			*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
-			serverWhiteList, *serverDisableHttp,
-		)
+		ms := weed_server.NewMasterServer(r, masterOptions.toMasterOption(serverWhiteList))
 
-		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
-		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
+		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterOptions.port)
+		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterOptions.port), 0)
 		if e != nil {
 			glog.Fatalf("Master startup error: %v", e)
 		}
 
 		go func() {
 			// start raftServer
-			myMasterAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
+			myMasterAddress, peers := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
 			raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
-				peers, myMasterAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
+				peers, myMasterAddress, *masterOptions.metaFolder, ms.Topo, *masterOptions.pulseSeconds)
 			ms.SetRaftServer(raftServer)
 			r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 
 			// starting grpc server
-			grpcPort := *masterPort + 10000
-			grpcL, err := util.NewListener(*serverIp+":"+strconv.Itoa(grpcPort), 0)
+			grpcPort := *masterOptions.port + 10000
+			grpcL, err := util.NewListener(*serverBindIp+":"+strconv.Itoa(grpcPort), 0)
 			if err != nil {
 				glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
 			}
 			// Create your protocol servers.
-			glog.V(0).Infof("grpc config %+v", viper.Sub("grpc"))
+			glog.V(1).Infof("grpc config %+v", viper.Sub("grpc"))
 			grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
 			master_pb.RegisterSeaweedServer(grpcS, ms)
 			protobuf.RegisterRaftServer(grpcS, raftServer)

weed/command/shell.go

@@ -2,8 +2,8 @@ package command
 
 import (
 	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/shell"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/spf13/viper"
 )
 
@@ -28,9 +28,13 @@ var ()
 
 func runShell(command *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 
+	shellOptions.FilerHost = "localhost"
+	shellOptions.FilerPort = 8888
+	shellOptions.Directory = "/"
+
 	shell.RunShell(shellOptions)
 
 	return true

weed/command/upload.go

@@ -3,12 +3,13 @@ package command
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
 	"os"
 	"path/filepath"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/operation"
 )
 
@@ -37,7 +38,7 @@ func init() {
 	upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name")
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
-	upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
+	upload.maxMB = cmdUpload.Flag.Int("maxMB", 32, "split files larger than the limit")
 }
 
 var cmdUpload = &Command{
@@ -61,7 +62,7 @@ var cmdUpload = &Command{
 
 func runUpload(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
 
 	if len(args) == 0 {

weed/command/volume.go

@@ -1,8 +1,6 @@
 package command
 
 import (
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/spf13/viper"
 	"net/http"
 	"os"
 	"runtime"
@@ -11,6 +9,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
@@ -34,7 +35,6 @@ type VolumeServerOptions struct {
 	masters               *string
 	pulseSeconds          *int
 	idleConnectionTimeout *int
-	maxCpu                *int
 	dataCenter            *string
 	rack                  *string
 	whiteList             []string
@@ -43,6 +43,7 @@ type VolumeServerOptions struct {
 	readRedirect          *bool
 	cpuProfile            *string
 	memProfile            *string
+	compactionMBPerSecond *int
 }
 
 func init() {
@@ -55,14 +56,14 @@ func init() {
 	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
 	v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
 	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
-	v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
-	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
+	v.compactionMBPerSecond = cmdVolume.Flag.Int("compactionMBps", 0, "limit background compaction or copying speed in mega bytes per second")
 }
 
 var cmdVolume = &Command{
@@ -81,12 +82,9 @@ var (
 
 func runVolume(cmd *Command, args []string) bool {
 
-	weed_server.LoadConfiguration("security", false)
+	util.LoadConfiguration("security", false)
 
-	if *v.maxCpu < 1 {
-		*v.maxCpu = runtime.NumCPU()
-	}
-	runtime.GOMAXPROCS(*v.maxCpu)
+	runtime.GOMAXPROCS(runtime.NumCPU())
 	util.SetupProfiling(*v.cpuProfile, *v.memProfile)
 
 	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption)
@@ -142,10 +140,10 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	switch *v.indexType {
 	case "leveldb":
 		volumeNeedleMapKind = storage.NeedleMapLevelDb
-	case "boltdb":
-		volumeNeedleMapKind = storage.NeedleMapBoltDb
-	case "btree":
-		volumeNeedleMapKind = storage.NeedleMapBtree
+	case "leveldbMedium":
+		volumeNeedleMapKind = storage.NeedleMapLevelDbMedium
+	case "leveldbLarge":
+		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
 	}
 
 	masters := *v.masters
@@ -157,6 +155,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		strings.Split(masters, ","), *v.pulseSeconds, *v.dataCenter, *v.rack,
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readRedirect,
+		*v.compactionMBPerSecond,
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)

weed/command/webdav.go

@@ -0,0 +1,109 @@
+package command
+
+import (
+	"fmt"
+	"net/http"
+	"os/user"
+	"strconv"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+)
+
+var (
+	webDavStandaloneOptions WebDavOption
+)
+
+type WebDavOption struct {
+	filer          *string
+	port           *int
+	collection     *string
+	tlsPrivateKey  *string
+	tlsCertificate *string
+}
+
+func init() {
+	cmdWebDav.Run = runWebDav // break init cycle
+	webDavStandaloneOptions.filer = cmdWebDav.Flag.String("filer", "localhost:8888", "filer server address")
+	webDavStandaloneOptions.port = cmdWebDav.Flag.Int("port", 7333, "webdav server http listen port")
+	webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files")
+	webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file")
+	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
+}
+
+var cmdWebDav = &Command{
+	UsageLine: "webdav -port=7333 -filer=<ip:port>",
+	Short:     "<unstable> start a webdav server that is backed by a filer",
+	Long: `start a webdav server that is backed by a filer.
+
+`,
+}
+
+func runWebDav(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+
+	glog.V(0).Infof("Starting Seaweed WebDav Server %s at https port %d", util.VERSION, *webDavStandaloneOptions.port)
+
+	return webDavStandaloneOptions.startWebDav()
+
+}
+
+func (wo *WebDavOption) startWebDav() bool {
+
+	filerGrpcAddress, err := parseFilerGrpcAddress(*wo.filer)
+	if err != nil {
+		glog.Fatal(err)
+		return false
+	}
+
+	// detect current user
+	uid, gid := uint32(0), uint32(0)
+	if u, err := user.Current(); err == nil {
+		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+			uid = uint32(parsedId)
+		}
+		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+			gid = uint32(parsedId)
+		}
+	}
+
+	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
+		Filer:            *wo.filer,
+		FilerGrpcAddress: filerGrpcAddress,
+		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
+		Collection:       *wo.collection,
+		Uid:              uid,
+		Gid:              gid,
+	})
+	if webdavServer_err != nil {
+		glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)
+	}
+
+	httpS := &http.Server{Handler: ws.Handler}
+
+	listenAddress := fmt.Sprintf(":%d", *wo.port)
+	webDavListener, err := util.NewListener(listenAddress, time.Duration(10)*time.Second)
+	if err != nil {
+		glog.Fatalf("WebDav Server listener on %s error: %v", listenAddress, err)
+	}
+
+	if *wo.tlsPrivateKey != "" {
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at https port %d", util.VERSION, *wo.port)
+		if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
+			glog.Fatalf("WebDav Server Fail to serve: %v", err)
+		}
+	} else {
+		glog.V(0).Infof("Start Seaweed WebDav Server %s at http port %d", util.VERSION, *wo.port)
+		if err = httpS.Serve(webDavListener); err != nil {
+			glog.Fatalf("WebDav Server Fail to serve: %v", err)
+		}
+	}
+
+	return true
+
+}

weed/command/weedfuse/README.md

@@ -0,0 +1,84 @@
+Mount the SeaweedFS via FUSE
+
+# Mount by fstab
+
+
+```
+$ # on linux
+$ sudo apt-get install fuse
+$ sudo echo 'user_allow_other' >> /etc/fuse.conf
+$ sudo mv weedfuse /sbin/mount.weedfuse
+
+$ # on Mac
+$ sudo mv weedfuse /sbin/mount_weedfuse
+
+```
+
+On both OS X and Linux, you can add one of the entries to your /etc/fstab file like the following:
+
+```
+# mount the whole SeaweedFS
+localhost:8888/    /home/some/mount/folder  weedfuse
+
+# mount the SeaweedFS sub folder
+localhost:8888/sub/dir    /home/some/mount/folder  weedfuse
+
+# mount the SeaweedFS sub folder with some options
+localhost:8888/sub/dir    /home/some/mount/folder  weedfuse  user
+
+```
+
+To verify it can work, try this command
+```
+$ sudo mount -av
+
+...
+
+/home/some/mount/folder           : successfully mounted
+
+```
+
+If you see `successfully mounted`, try to access the mounted folder and verify everything works.
+
+
+To debug, run these:
+```
+
+$ weedfuse -foreground localhost:8888/ /home/some/mount/folder
+
+```
+
+
+To unmount the folder:
+```
+
+$ sudo umount /home/some/mount/folder
+
+```
+
+<!-- not working yet!
+
+# Mount by autofs
+
+AutoFS can mount a folder if accessed.
+
+```
+# install autofs
+$ sudo apt-get install autofs
+```
+
+Here is an example on how to mount a folder for all users under `/home` directory.
+Assuming there exists corresponding folders under `/home` on both local and SeaweedFS.
+
+Edit `/etc/auto.master` and `/etc/auto.weedfuse` file with these content
+```
+$ cat /etc/auto.master
+/home    /etc/auto.weedfuse
+
+$ cat /etc/auto.weedfuse
+# map /home/<user> to localhost:8888/home/<user>
+* -fstype=weedfuse,rw,allow_other,foreground :localhost\:8888/home/&
+
+```
+
+-->

weed/command/weedfuse/weedfuse.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/command"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/jacobsa/daemonize"
+	"github.com/kardianos/osext"
+)
+
+var (
+	fuseCommand  = flag.NewFlagSet("weedfuse", flag.ContinueOnError)
+	options      = fuseCommand.String("o", "", "comma separated options rw,uid=xxx,gid=xxx")
+	isForeground = fuseCommand.Bool("foreground", false, "starts as a daemon")
+)
+
+func main() {
+
+	err := fuseCommand.Parse(os.Args[1:])
+	if err != nil {
+		glog.Fatal(err)
+	}
+	fmt.Printf("options: %v\n", *options)
+
+	// seems this value is always empty, need to parse it differently
+	optionsString := *options
+	prev := ""
+	for i, arg := range os.Args {
+		fmt.Printf("args[%d]: %v\n", i, arg)
+		if prev == "-o" {
+			optionsString = arg
+		}
+		prev = arg
+	}
+
+	device := fuseCommand.Arg(0)
+	mountPoint := fuseCommand.Arg(1)
+
+	fmt.Printf("source: %v\n", device)
+	fmt.Printf("target: %v\n", mountPoint)
+
+	nouser := true
+	for _, option := range strings.Split(optionsString, ",") {
+		fmt.Printf("option: %v\n", option)
+		switch option {
+		case "user":
+			nouser = false
+		}
+	}
+
+	maybeSetupPath()
+
+	if !*isForeground {
+		startAsDaemon()
+		return
+	}
+
+	parts := strings.SplitN(device, "/", 2)
+	filer, filerPath := parts[0], parts[1]
+
+	command.RunMount(
+		filer, "/"+filerPath, mountPoint, "", "000", "",
+		4, !nouser, 0, 1000000)
+
+}
+
+func maybeSetupPath() {
+	// sudo mount -av may not include PATH in some linux, e.g., Ubuntu
+	hasPathEnv := false
+	for _, e := range os.Environ() {
+		if strings.HasPrefix(e, "PATH=") {
+			hasPathEnv = true
+		}
+		fmt.Println(e)
+	}
+	if !hasPathEnv {
+		os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
+	}
+}
+
+func startAsDaemon() {
+
+	// adapted from gcsfuse
+
+	// Find the executable.
+	var path string
+	path, err := osext.Executable()
+	if err != nil {
+		glog.Fatalf("osext.Executable: %v", err)
+	}
+
+	// Set up arguments. Be sure to use foreground mode.
+	args := append([]string{"-foreground"}, os.Args[1:]...)
+
+	// Pass along PATH so that the daemon can find fusermount on Linux.
+	env := []string{
+		fmt.Sprintf("PATH=%s", os.Getenv("PATH")),
+	}
+
+	err = daemonize.Run(path, args, env, os.Stdout)
+	if err != nil {
+		glog.Fatalf("daemonize.Run: %v", err)
+	}
+
+}

weed/filer2/abstract_sql/abstract_sql_store.go

@@ -26,7 +26,10 @@ type TxOrDB interface {
 }
 
 func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
-	tx, err := store.DB.BeginTx(ctx, nil)
+	tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
+		Isolation: sql.LevelReadCommitted,
+		ReadOnly:  false,
+	})
 	if err != nil {
 		return ctx, err
 	}

weed/filer2/cassandra/cassandra_store.go

@@ -85,7 +85,7 @@ func (store *CassandraStore) FindEntry(ctx context.Context, fullpath filer2.Full
 	}
 
 	if len(data) == 0 {
-		return nil, fmt.Errorf("not found: %s", fullpath)
+		return nil, filer2.ErrNotFound
 	}
 
 	entry = &filer2.Entry{

weed/filer2/entry.go

@@ -52,9 +52,20 @@ func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
 		return nil
 	}
 	return &filer_pb.Entry{
-		Name:        string(entry.FullPath),
+		Name:        entry.FullPath.Name(),
 		IsDirectory: entry.IsDirectory(),
 		Attributes:  EntryAttributeToPb(entry),
 		Chunks:      entry.Chunks,
 	}
 }
+
+func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
+	if entry == nil {
+		return nil
+	}
+	dir, _ := entry.FullPath.DirAndName()
+	return &filer_pb.FullEntry{
+		Dir:   dir,
+		Entry: entry.ToProtoEntry(),
+	}
+}

weed/filer2/filechunks.go

@@ -40,7 +40,7 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file
 		fileIds[interval.fileId] = true
 	}
 	for _, chunk := range chunks {
-		if found := fileIds[chunk.FileId]; found {
+		if _, found := fileIds[chunk.GetFileIdString()]; found {
 			compacted = append(compacted, chunk)
 		} else {
 			garbage = append(garbage, chunk)
@@ -50,15 +50,15 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file
 	return
 }
 
-func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) {
+func MinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
 
 	fileIds := make(map[string]bool)
-	for _, interval := range newChunks {
-		fileIds[interval.FileId] = true
+	for _, interval := range bs {
+		fileIds[interval.GetFileIdString()] = true
 	}
-	for _, chunk := range oldChunks {
-		if found := fileIds[chunk.FileId]; !found {
-			unused = append(unused, chunk)
+	for _, chunk := range as {
+		if _, found := fileIds[chunk.GetFileIdString()]; !found {
+			delta = append(delta, chunk)
 		}
 	}
 
@@ -123,7 +123,7 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.
 	newV := newVisibleInterval(
 		chunk.Offset,
 		chunk.Offset+int64(chunk.Size),
-		chunk.FileId,
+		chunk.GetFileIdString(),
 		chunk.Mtime,
 		true,
 	)

weed/filer2/filer.go

@@ -21,7 +21,7 @@ var (
 )
 
 type Filer struct {
-	store              FilerStore
+	store              *FilerStoreWrapper
 	directoryCache     *ccache.Cache
 	MasterClient       *wdclient.MasterClient
 	fileIdDeletionChan chan string
@@ -42,7 +42,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
 }
 
 func (f *Filer) SetStore(store FilerStore) {
-	f.store = store
+	f.store = NewFilerStoreWrapper(store)
 }
 
 func (f *Filer) DisableDirectoryCache() {
@@ -153,10 +153,12 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 
 	if oldEntry == nil {
 		if err := f.store.InsertEntry(ctx, entry); err != nil {
+			glog.Errorf("insert entry %s: %v", entry.FullPath, err)
 			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
 		}
 	} else {
 		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
+			glog.Errorf("update entry %s: %v", entry.FullPath, err)
 			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
 		}
 	}
@@ -171,9 +173,11 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry) error {
 func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
 	if oldEntry != nil {
 		if oldEntry.IsDirectory() && !entry.IsDirectory() {
+			glog.Errorf("existing %s is a directory", entry.FullPath)
 			return fmt.Errorf("existing %s is a directory", entry.FullPath)
 		}
 		if !oldEntry.IsDirectory() && entry.IsDirectory() {
+			glog.Errorf("existing %s is a file", entry.FullPath)
 			return fmt.Errorf("existing %s is a file", entry.FullPath)
 		}
 	}
@@ -215,35 +219,39 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p FullPath, isRecurs
 		for limit > 0 {
 			entries, err := f.ListDirectoryEntries(ctx, p, lastFileName, includeLastFile, 1024)
 			if err != nil {
+				glog.Errorf("list folder %s: %v", p, err)
 				return fmt.Errorf("list folder %s: %v", p, err)
 			}
+
 			if len(entries) == 0 {
 				break
-			} else {
-				if isRecursive {
-					for _, sub := range entries {
-						lastFileName = sub.Name()
-						f.DeleteEntryMetaAndData(ctx, sub.FullPath, isRecursive, shouldDeleteChunks)
-						limit--
-						if limit <= 0 {
-							break
-						}
+			}
+
+			if isRecursive {
+				for _, sub := range entries {
+					lastFileName = sub.Name()
+					err = f.DeleteEntryMetaAndData(ctx, sub.FullPath, isRecursive, shouldDeleteChunks)
+					if err != nil {
+						return err
 					}
-				} else {
-					if len(entries) > 0 {
-						return fmt.Errorf("folder %s is not empty", p)
+					limit--
+					if limit <= 0 {
+						break
 					}
 				}
-				f.cacheDelDirectory(string(p))
-				if len(entries) < 1024 {
-					break
-				}
+			}
+
+			if len(entries) < 1024 {
+				break
 			}
 		}
+
+		f.cacheDelDirectory(string(p))
+
 	}
 
 	if shouldDeleteChunks {
-		f.DeleteChunks(entry.Chunks)
+		f.DeleteChunks(p, entry.Chunks)
 	}
 
 	if p == "/" {
@@ -264,6 +272,11 @@ func (f *Filer) ListDirectoryEntries(ctx context.Context, p FullPath, startFileN
 }
 
 func (f *Filer) cacheDelDirectory(dirpath string) {
+
+	if dirpath == "/" {
+		return
+	}
+
 	if f.directoryCache == nil {
 		return
 	}
@@ -272,6 +285,7 @@ func (f *Filer) cacheDelDirectory(dirpath string) {
 }
 
 func (f *Filer) cacheGetDirectory(dirpath string) *Entry {
+
 	if f.directoryCache == nil {
 		return nil
 	}

weed/filer2/filer_client_util.go

@@ -0,0 +1,163 @@
+package filer2
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func VolumeId(fileId string) string {
+	lastCommaIndex := strings.LastIndex(fileId, ",")
+	if lastCommaIndex > 0 {
+		return fileId[:lastCommaIndex]
+	}
+	return fileId
+}
+
+type FilerClient interface {
+	WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error
+}
+
+func ReadIntoBuffer(ctx context.Context, filerClient FilerClient, fullFilePath string, buff []byte, chunkViews []*ChunkView, baseOffset int64) (totalRead int64, err error) {
+	var vids []string
+	for _, chunkView := range chunkViews {
+		vids = append(vids, VolumeId(chunkView.FileId))
+	}
+
+	vid2Locations := make(map[string]*filer_pb.Locations)
+
+	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+
+		glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+			VolumeIds: vids,
+		})
+		if err != nil {
+			return err
+		}
+
+		vid2Locations = resp.LocationsMap
+
+		return nil
+	})
+
+	if err != nil {
+		return 0, fmt.Errorf("failed to lookup volume ids %v: %v", vids, err)
+	}
+
+	var wg sync.WaitGroup
+	for _, chunkView := range chunkViews {
+		wg.Add(1)
+		go func(chunkView *ChunkView) {
+			defer wg.Done()
+
+			glog.V(4).Infof("read fh reading chunk: %+v", chunkView)
+
+			locations := vid2Locations[VolumeId(chunkView.FileId)]
+			if locations == nil || len(locations.Locations) == 0 {
+				glog.V(0).Infof("failed to locate %s", chunkView.FileId)
+				err = fmt.Errorf("failed to locate %s", chunkView.FileId)
+				return
+			}
+
+			var n int64
+			n, err = util.ReadUrl(
+				fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
+				chunkView.Offset,
+				int(chunkView.Size),
+				buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)],
+				!chunkView.IsFullChunk)
+
+			if err != nil {
+
+				glog.V(0).Infof("%v read http://%s/%v %v bytes: %v", fullFilePath, locations.Locations[0].Url, chunkView.FileId, n, err)
+
+				err = fmt.Errorf("failed to read http://%s/%s: %v",
+					locations.Locations[0].Url, chunkView.FileId, err)
+				return
+			}
+
+			glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView)
+			totalRead += n
+
+		}(chunkView)
+	}
+	wg.Wait()
+	return
+}
+
+func GetEntry(ctx context.Context, filerClient FilerClient, fullFilePath string) (entry *filer_pb.Entry, err error) {
+
+	dir, name := FullPath(fullFilePath).DirAndName()
+
+	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		}
+
+		glog.V(3).Infof("read %s request: %v", fullFilePath, request)
+		resp, err := client.LookupDirectoryEntry(ctx, request)
+		if err != nil {
+			if err == ErrNotFound || strings.Contains(err.Error(), ErrNotFound.Error()) {
+				return nil
+			}
+			glog.V(3).Infof("read %s attr %v: %v", fullFilePath, request, err)
+			return err
+		}
+
+		if resp.Entry != nil {
+			entry = resp.Entry
+		}
+
+		return nil
+	})
+
+	return
+}
+
+func ReadDirAllEntries(ctx context.Context, filerClient FilerClient, fullDirPath string, fn func(entry *filer_pb.Entry)) (err error) {
+
+	err = filerClient.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+
+		paginationLimit := 1024
+
+		lastEntryName := ""
+
+		for {
+
+			request := &filer_pb.ListEntriesRequest{
+				Directory:         fullDirPath,
+				StartFromFileName: lastEntryName,
+				Limit:             uint32(paginationLimit),
+			}
+
+			glog.V(3).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
+			if err != nil {
+				return fmt.Errorf("list %s: %v", fullDirPath, err)
+			}
+
+			for _, entry := range resp.Entries {
+				fn(entry)
+				lastEntryName = entry.Name
+			}
+
+			if len(resp.Entries) < paginationLimit {
+				break
+			}
+
+		}
+
+		return nil
+
+	})
+
+	return
+}

weed/filer2/filer_deletion.go

@@ -51,12 +51,15 @@ func (f *Filer) loopProcessingDeletion() {
 	}
 }
 
-func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
+func (f *Filer) DeleteChunks(fullpath FullPath, chunks []*filer_pb.FileChunk) {
 	for _, chunk := range chunks {
-		f.fileIdDeletionChan <- chunk.FileId
+		glog.V(3).Infof("deleting %s chunk %s", fullpath, chunk.String())
+		f.fileIdDeletionChan <- chunk.GetFileIdString()
 	}
 }
 
+// DeleteFileByFileId direct delete by file id.
+// Only used when the fileId is not being managed by snapshots.
 func (f *Filer) DeleteFileByFileId(fileId string) {
 	f.fileIdDeletionChan <- fileId
 }
@@ -67,22 +70,19 @@ func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
 		return
 	}
 	if newEntry == nil {
-		f.DeleteChunks(oldEntry.Chunks)
+		f.DeleteChunks(oldEntry.FullPath, oldEntry.Chunks)
 	}
 
 	var toDelete []*filer_pb.FileChunk
+	newChunkIds := make(map[string]bool)
+	for _, newChunk := range newEntry.Chunks {
+		newChunkIds[newChunk.GetFileIdString()] = true
+	}
 
 	for _, oldChunk := range oldEntry.Chunks {
-		found := false
-		for _, newChunk := range newEntry.Chunks {
-			if oldChunk.FileId == newChunk.FileId {
-				found = true
-				break
-			}
-		}
-		if !found {
+		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
 			toDelete = append(toDelete, oldChunk)
 		}
 	}
-	f.DeleteChunks(toDelete)
+	f.DeleteChunks(oldEntry.FullPath, toDelete)
 }

weed/filer2/filer_notify.go

@@ -20,12 +20,18 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool)
 
 		glog.V(3).Infof("notifying entry update %v", key)
 
+		newParentPath := ""
+		if newEntry != nil {
+			newParentPath, _ = newEntry.FullPath.DirAndName()
+		}
+
 		notification.Queue.SendMessage(
 			key,
 			&filer_pb.EventNotification{
-				OldEntry:     oldEntry.ToProtoEntry(),
-				NewEntry:     newEntry.ToProtoEntry(),
-				DeleteChunks: deleteChunks,
+				OldEntry:      oldEntry.ToProtoEntry(),
+				NewEntry:      newEntry.ToProtoEntry(),
+				DeleteChunks:  deleteChunks,
+				NewParentPath: newParentPath,
 			},
 		)
 

weed/filer2/filerstore.go

@@ -3,6 +3,10 @@ package filer2
 import (
 	"context"
 	"errors"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/stats"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -24,3 +28,97 @@ type FilerStore interface {
 }
 
 var ErrNotFound = errors.New("filer: no entry is found in filer store")
+
+type FilerStoreWrapper struct {
+	actualStore FilerStore
+}
+
+func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
+	return &FilerStoreWrapper{
+		actualStore: store,
+	}
+}
+
+func (fsw *FilerStoreWrapper) GetName() string {
+	return fsw.actualStore.GetName()
+}
+
+func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration) error {
+	return fsw.actualStore.Initialize(configuration)
+}
+
+func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {
+	stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "insert").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "insert").Observe(time.Since(start).Seconds())
+	}()
+
+	filer_pb.BeforeEntrySerialization(entry.Chunks)
+	return fsw.actualStore.InsertEntry(ctx, entry)
+}
+
+func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
+	stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "update").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "update").Observe(time.Since(start).Seconds())
+	}()
+
+	filer_pb.BeforeEntrySerialization(entry.Chunks)
+	return fsw.actualStore.UpdateEntry(ctx, entry)
+}
+
+func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp FullPath) (entry *Entry, err error) {
+	stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "find").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "find").Observe(time.Since(start).Seconds())
+	}()
+
+	entry, err = fsw.actualStore.FindEntry(ctx, fp)
+	if err != nil {
+		return nil, err
+	}
+	filer_pb.AfterEntryDeserialization(entry.Chunks)
+	return
+}
+
+func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp FullPath) (err error) {
+	stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "delete").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "delete").Observe(time.Since(start).Seconds())
+	}()
+
+	return fsw.actualStore.DeleteEntry(ctx, fp)
+}
+
+func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) {
+	stats.FilerStoreCounter.WithLabelValues(fsw.actualStore.GetName(), "list").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(fsw.actualStore.GetName(), "list").Observe(time.Since(start).Seconds())
+	}()
+
+	entries, err := fsw.actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range entries {
+		filer_pb.AfterEntryDeserialization(entry.Chunks)
+	}
+	return entries, err
+}
+
+func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return fsw.actualStore.BeginTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) CommitTransaction(ctx context.Context) error {
+	return fsw.actualStore.CommitTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) RollbackTransaction(ctx context.Context) error {
+	return fsw.actualStore.RollbackTransaction(ctx)
+}

weed/filer2/leveldb/leveldb_store.go

@@ -9,6 +9,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	weed_util "github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -39,7 +40,13 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}
 
-	if store.db, err = leveldb.OpenFile(dir, nil); err != nil {
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+	}
+
+	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
 		glog.Infof("filer store open dir %s: %v", dir, err)
 		return
 	}

weed/filer2/leveldb2/leveldb2_store.go

@@ -0,0 +1,208 @@
+package leveldb
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	weed_util "github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func init() {
+	filer2.Stores = append(filer2.Stores, &LevelDB2Store{})
+}
+
+type LevelDB2Store struct {
+	dbs []*leveldb.DB
+	dbCount int
+}
+
+func (store *LevelDB2Store) GetName() string {
+	return "leveldb2"
+}
+
+func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration) (err error) {
+	dir := configuration.GetString("dir")
+	return store.initialize(dir, 8)
+}
+
+func (store *LevelDB2Store) initialize(dir string, dbCount int) (err error) {
+	glog.Infof("filer store leveldb2 dir: %s", dir)
+	if err := weed_util.TestFolderWritable(dir); err != nil {
+		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
+	}
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 4,
+	}
+
+	for d := 0 ; d < dbCount; d++ {
+		dbFolder := fmt.Sprintf("%s/%02d", dir, d)
+		os.MkdirAll(dbFolder, 0755)
+		db, dbErr := leveldb.OpenFile(dbFolder, opts)
+		if dbErr != nil {
+			glog.Errorf("filer store open dir %s: %v", dbFolder, dbErr)
+			return
+		}
+		store.dbs = append(store.dbs, db)
+	}
+	store.dbCount = dbCount
+
+	return
+}
+
+func (store *LevelDB2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *LevelDB2Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+	dir, name := entry.DirAndName()
+	key, partitionId := genKey(dir, name, store.dbCount)
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	err = store.dbs[partitionId].Put(key, value, nil)
+
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	// println("saved", entry.FullPath, "chunks", len(entry.Chunks))
+
+	return nil
+}
+
+func (store *LevelDB2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+	dir, name := fullpath.DirAndName()
+	key, partitionId := genKey(dir, name, store.dbCount)
+
+	data, err := store.dbs[partitionId].Get(key, nil)
+
+	if err == leveldb.ErrNotFound {
+		return nil, filer2.ErrNotFound
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)
+	}
+
+	entry = &filer2.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(data)
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))
+
+	return entry, nil
+}
+
+func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
+	dir, name := fullpath.DirAndName()
+	key, partitionId := genKey(dir, name, store.dbCount)
+
+	err = store.dbs[partitionId].Delete(key, nil)
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
+	limit int) (entries []*filer2.Entry, err error) {
+
+	directoryPrefix, partitionId := genDirectoryKeyPrefix(fullpath, "", store.dbCount)
+	lastFileStart, _ := genDirectoryKeyPrefix(fullpath, startFileName, store.dbCount)
+
+	iter := store.dbs[partitionId].NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)
+	for iter.Next() {
+		key := iter.Key()
+		if !bytes.HasPrefix(key, directoryPrefix) {
+			break
+		}
+		fileName := getNameFromKey(key)
+		if fileName == "" {
+			continue
+		}
+		if fileName == startFileName && !inclusive {
+			continue
+		}
+		limit--
+		if limit < 0 {
+			break
+		}
+		entry := &filer2.Entry{
+			FullPath: filer2.NewFullPath(string(fullpath), fileName),
+		}
+
+		// println("list", entry.FullPath, "chunks", len(entry.Chunks))
+
+		if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+			break
+		}
+		entries = append(entries, entry)
+	}
+	iter.Release()
+
+	return entries, err
+}
+
+func genKey(dirPath, fileName string, dbCount int) (key []byte, partitionId int) {
+	key, partitionId = hashToBytes(dirPath, dbCount)
+	key = append(key, []byte(fileName)...)
+	return key, partitionId
+}
+
+func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string, dbCount int) (keyPrefix []byte, partitionId int) {
+	keyPrefix, partitionId = hashToBytes(string(fullpath), dbCount)
+	if len(startFileName) > 0 {
+		keyPrefix = append(keyPrefix, []byte(startFileName)...)
+	}
+	return keyPrefix, partitionId
+}
+
+func getNameFromKey(key []byte) string {
+
+	return string(key[md5.Size:])
+
+}
+
+// hash directory, and use last byte for partitioning
+func hashToBytes(dir string, dbCount int) ([]byte, int) {
+	h := md5.New()
+	io.WriteString(h, dir)
+
+	b := h.Sum(nil)
+
+	x := b[len(b)-1]
+
+	return b, int(x)%dbCount
+}

weed/filer2/leveldb2/leveldb2_store_test.go

@@ -0,0 +1,88 @@
+package leveldb
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestCreateAndFind(t *testing.T) {
+	filer := filer2.NewFiler(nil, nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	defer os.RemoveAll(dir)
+	store := &LevelDB2Store{}
+	store.initialize(dir,2)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
+
+	ctx := context.Background()
+
+	entry1 := &filer2.Entry{
+		FullPath: fullpath,
+		Attr: filer2.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+
+	if err := filer.CreateEntry(ctx, entry1); err != nil {
+		t.Errorf("create entry %v: %v", entry1.FullPath, err)
+		return
+	}
+
+	entry, err := filer.FindEntry(ctx, fullpath)
+
+	if err != nil {
+		t.Errorf("find entry: %v", err)
+		return
+	}
+
+	if entry.FullPath != entry1.FullPath {
+		t.Errorf("find wrong entry: %v", entry.FullPath)
+		return
+	}
+
+	// checking one upper directory
+	entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+	// checking one upper directory
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}
+
+func TestEmptyRoot(t *testing.T) {
+	filer := filer2.NewFiler(nil, nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	defer os.RemoveAll(dir)
+	store := &LevelDB2Store{}
+	store.initialize(dir,2)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	ctx := context.Background()
+
+	// checking one upper directory
+	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
+	if err != nil {
+		t.Errorf("list entries: %v", err)
+		return
+	}
+	if len(entries) != 0 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}

weed/filer2/memdb/memdb_store.go

@@ -7,6 +7,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/google/btree"
 	"strings"
+	"sync"
 )
 
 func init() {
@@ -14,7 +15,8 @@ func init() {
 }
 
 type MemDbStore struct {
-	tree *btree.BTree
+	tree     *btree.BTree
+	treeLock sync.Mutex
 }
 
 type entryItem struct {
@@ -46,7 +48,9 @@ func (store *MemDbStore) RollbackTransaction(ctx context.Context) error {
 
 func (store *MemDbStore) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
 	// println("inserting", entry.FullPath)
+	store.treeLock.Lock()
 	store.tree.ReplaceOrInsert(entryItem{entry})
+	store.treeLock.Unlock()
 	return nil
 }
 
@@ -54,7 +58,9 @@ func (store *MemDbStore) UpdateEntry(ctx context.Context, entry *filer2.Entry) (
 	if _, err = store.FindEntry(ctx, entry.FullPath); err != nil {
 		return fmt.Errorf("no such file %s : %v", entry.FullPath, err)
 	}
+	store.treeLock.Lock()
 	store.tree.ReplaceOrInsert(entryItem{entry})
+	store.treeLock.Unlock()
 	return nil
 }
 
@@ -68,7 +74,9 @@ func (store *MemDbStore) FindEntry(ctx context.Context, fullpath filer2.FullPath
 }
 
 func (store *MemDbStore) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
+	store.treeLock.Lock()
 	store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}})
+	store.treeLock.Unlock()
 	return nil
 }
 

weed/filer2/postgres/README.txt

@@ -9,8 +9,8 @@ $PGHOME/bin/psql --username=postgres --password seaweedfs
 
 CREATE TABLE IF NOT EXISTS filemeta (
   dirhash     BIGINT,
-  name        VARCHAR(1000),
-  directory   VARCHAR(4096),
+  name        VARCHAR(65535),
+  directory   VARCHAR(65535),
   meta        bytea,
   PRIMARY KEY (dirhash, name)
 );

weed/filer2/redis/redis_cluster_store.go

@@ -21,12 +21,14 @@ func (store *RedisClusterStore) GetName() string {
 func (store *RedisClusterStore) Initialize(configuration util.Configuration) (err error) {
 	return store.initialize(
 		configuration.GetStringSlice("addresses"),
+		configuration.GetString("password"),
 	)
 }
 
-func (store *RedisClusterStore) initialize(addresses []string) (err error) {
+func (store *RedisClusterStore) initialize(addresses []string, password string) (err error) {
 	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
-		Addrs: addresses,
+		Addrs:    addresses,
+		Password: password,
 	})
 	return
 }

weed/filer2/stream.go

@@ -0,0 +1,41 @@
+package filer2
+
+import (
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/wdclient"
+)
+
+func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*filer_pb.FileChunk, offset int64, size int) error {
+
+	chunkViews := ViewFromChunks(chunks, offset, size)
+
+	fileId2Url := make(map[string]string)
+
+	for _, chunkView := range chunkViews {
+
+		urlString, err := masterClient.LookupFileId(chunkView.FileId)
+		if err != nil {
+			glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
+			return err
+		}
+		fileId2Url[chunkView.FileId] = urlString
+	}
+
+	for _, chunkView := range chunkViews {
+		urlString := fileId2Url[chunkView.FileId]
+		_, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) {
+			w.Write(data)
+		})
+		if err != nil {
+			glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err)
+			return err
+		}
+	}
+
+	return nil
+
+}

weed/filesys/dir.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"os"
 	"path"
-	"path/filepath"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
@@ -35,9 +34,7 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 	attr.Valid = time.Second
 
 	if dir.Path == dir.wfs.option.FilerMountRootPath {
-		attr.Uid = dir.wfs.option.MountUid
-		attr.Gid = dir.wfs.option.MountGid
-		attr.Mode = dir.wfs.option.MountMode
+		dir.setRootDirAttributes(attr)
 		return nil
 	}
 
@@ -54,40 +51,14 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 		return nil
 	}
 
-	parent, name := filepath.Split(dir.Path)
-
-	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
-
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: parent,
-			Name:      name,
-		}
-
-		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
-		if err != nil {
-			if err == filer2.ErrNotFound {
-				return nil
-			}
-			glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
-			return err
-		}
-
-		if resp.Entry != nil {
-			dir.attributes = resp.Entry.Attributes
-		}
-
-		// dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
-
-		return nil
-	})
-
+	entry, err := filer2.GetEntry(ctx, dir.wfs, dir.Path)
 	if err != nil {
+		glog.V(2).Infof("read dir %s attr: %v, error: %v", dir.Path, dir.attributes, err)
 		return err
 	}
+	dir.attributes = entry.Attributes
 
-	// glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
-	// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))
+	glog.V(2).Infof("dir %s: %v perm: %v", dir.Path, dir.attributes, os.FileMode(dir.attributes.FileMode))
 
 	attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir
 
@@ -99,6 +70,16 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 	return nil
 }
 
+func (dir *Dir) setRootDirAttributes(attr *fuse.Attr) {
+	attr.Uid = dir.wfs.option.MountUid
+	attr.Gid = dir.wfs.option.MountGid
+	attr.Mode = dir.wfs.option.MountMode
+	attr.Crtime = dir.wfs.option.MountCtime
+	attr.Ctime = dir.wfs.option.MountCtime
+	attr.Mtime = dir.wfs.option.MountMtime
+	attr.Atime = dir.wfs.option.MountMtime
+}
+
 func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
 	return &File{
 		Name:           name,
@@ -132,7 +113,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	glog.V(1).Infof("create: %v", request)
 
 	if request.Entry.IsDirectory {
-		if err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+		if err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 			if _, err := client.CreateEntry(ctx, request); err != nil {
 				glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
 				return fuse.EIO
@@ -155,7 +136,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 
 func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
 
-	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.CreateEntryRequest{
 			Directory: dir.Path,
@@ -192,33 +173,18 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err
 func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
 
 	var entry *filer_pb.Entry
+	fullFilePath := path.Join(dir.Path, req.Name)
 
-	item := dir.wfs.listDirectoryEntriesCache.Get(path.Join(dir.Path, req.Name))
+	item := dir.wfs.listDirectoryEntriesCache.Get(fullFilePath)
 	if item != nil && !item.Expired() {
 		entry = item.Value().(*filer_pb.Entry)
 	}
 
 	if entry == nil {
-		err = dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
-
-			request := &filer_pb.LookupDirectoryEntryRequest{
-				Directory: dir.Path,
-				Name:      req.Name,
-			}
-
-			glog.V(4).Infof("lookup directory entry: %v", request)
-			resp, err := client.LookupDirectoryEntry(ctx, request)
-			if err != nil {
-				// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
-				return fuse.ENOENT
-			}
-
-			entry = resp.Entry
-
-			// dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
-
-			return nil
-		})
+		entry, err = filer2.GetEntry(ctx, dir.wfs, fullFilePath)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if entry != nil {
@@ -243,7 +209,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 
 func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
-	err = dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err = dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		paginationLimit := 1024
 		remaining := dir.wfs.option.DirListingLimit
@@ -305,33 +271,14 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 
 func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
 
-	var entry *filer_pb.Entry
-	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
-
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: dir.Path,
-			Name:      req.Name,
-		}
-
-		glog.V(4).Infof("lookup to-be-removed entry: %v", request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
-		if err != nil {
-			// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
-			return fuse.ENOENT
-		}
-
-		entry = resp.Entry
-
-		return nil
-	})
-
+	entry, err := filer2.GetEntry(ctx, dir.wfs, path.Join(dir.Path, req.Name))
 	if err != nil {
 		return err
 	}
 
 	dir.wfs.deleteFileChunks(ctx, entry.Chunks)
 
-	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -355,7 +302,7 @@ func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) erro
 
 func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
 
-	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
@@ -401,7 +348,7 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
 	}
 
 	parentDir, name := filer2.FullPath(dir.Path).DirAndName()
-	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: parentDir,

weed/filesys/dir_link.go

@@ -35,7 +35,7 @@ func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node,
 		},
 	}
 
-	err := dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	err := dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			glog.V(0).Infof("symlink %s/%s: %v", dir.Path, req.NewName, err)
 			return fuse.EIO

weed/filesys/dir_rename.go

@@ -12,7 +12,7 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 
 	newDir := newDirectory.(*Dir)
 
-	return dir.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return dir.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AtomicRenameEntryRequest{
 			OldDirectory: dir.Path,

weed/filesys/dirty_page.go

@@ -110,7 +110,7 @@ func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int6
 	// flush existing
 	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
 		if chunk != nil {
-			glog.V(4).Infof("%s/%s flush existing [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
+			glog.V(4).Infof("%s/%s flush existing [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
 			chunks = append(chunks, chunk)
 		}
 	} else {
@@ -123,7 +123,7 @@ func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int6
 	// flush the new page
 	if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil {
 		if chunk != nil {
-			glog.V(4).Infof("%s/%s flush big request [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
+			glog.V(4).Infof("%s/%s flush big request [%d,%d) to %s", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId)
 			chunks = append(chunks, chunk)
 		}
 	} else {
@@ -167,7 +167,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 	var fileId, host string
 	var auth security.EncodedJwt
 
-	if err := pages.f.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pages.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.AssignVolumeRequest{
 			Count:       1,

weed/filesys/file.go

@@ -109,7 +109,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		return nil
 	}
 
-	return file.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return file.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: file.dir.Path,
@@ -144,7 +144,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 			file.setEntry(entry)
 			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
 		} else {
-			err := file.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+			err := file.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 				request := &filer_pb.LookupDirectoryEntryRequest{
 					Name:      file.Name,
@@ -194,6 +194,8 @@ func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
 		newVisibles = t
 	}
 
+	glog.V(3).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(file.entry.Chunks), len(chunks))
+
 	file.entry.Chunks = append(file.entry.Chunks, chunks...)
 }
 

weed/filesys/filehandle.go

@@ -3,20 +3,16 @@ package filesys
 import (
 	"context"
 	"fmt"
+	"mime"
+	"path"
+	"time"
+
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gabriel-vasile/mimetype"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
-	"google.golang.org/grpc"
-	"mime"
-	"path"
-	"strings"
-	"sync"
-	"time"
 )
 
 type FileHandle struct {
@@ -68,75 +64,14 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 
 	chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)
 
-	var vids []string
-	for _, chunkView := range chunkViews {
-		vids = append(vids, volumeId(chunkView.FileId))
-	}
-
-	vid2Locations := make(map[string]*filer_pb.Locations)
-
-	err := fh.f.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
-
-		glog.V(4).Infof("read fh lookup volume id locations: %v", vids)
-		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
-			VolumeIds: vids,
-		})
-		if err != nil {
-			return err
-		}
-
-		vid2Locations = resp.LocationsMap
-
-		return nil
-	})
-
-	if err != nil {
-		glog.V(4).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err)
-		return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err)
-	}
-
-	var totalRead int64
-	var wg sync.WaitGroup
-	for _, chunkView := range chunkViews {
-		wg.Add(1)
-		go func(chunkView *filer2.ChunkView) {
-			defer wg.Done()
-
-			glog.V(4).Infof("read fh reading chunk: %+v", chunkView)
-
-			locations := vid2Locations[volumeId(chunkView.FileId)]
-			if locations == nil || len(locations.Locations) == 0 {
-				glog.V(0).Infof("failed to locate %s", chunkView.FileId)
-				err = fmt.Errorf("failed to locate %s", chunkView.FileId)
-				return
-			}
-
-			var n int64
-			n, err = util.ReadUrl(
-				fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
-				chunkView.Offset,
-				int(chunkView.Size),
-				buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)],
-				!chunkView.IsFullChunk)
-
-			if err != nil {
-
-				glog.V(0).Infof("%v/%v read http://%s/%v %v bytes: %v", fh.f.dir.Path, fh.f.Name, locations.Locations[0].Url, chunkView.FileId, n, err)
-
-				err = fmt.Errorf("failed to read http://%s/%s: %v",
-					locations.Locations[0].Url, chunkView.FileId, err)
-				return
-			}
-
-			glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView)
-			totalRead += n
-
-		}(chunkView)
-	}
-	wg.Wait()
+	totalRead, err := filer2.ReadIntoBuffer(ctx, fh.f.wfs, fh.f.fullpath(), buff, chunkViews, req.Offset)
 
 	resp.Data = buff[:totalRead]
 
+	if err != nil {
+		glog.Errorf("file handle read %s: %v", fh.f.fullpath(), err)
+	}
+
 	return err
 }
 
@@ -205,7 +140,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		return nil
 	}
 
-	return fh.f.wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	return fh.f.wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		if fh.f.entry.Attributes != nil {
 			fh.f.entry.Attributes.Mime = fh.contentType
@@ -221,70 +156,25 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 			Entry:     fh.f.entry,
 		}
 
-		//glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
-		//for i, chunk := range fh.f.entry.Chunks {
-		//	glog.V(4).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
-		//}
+		glog.V(3).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
+		for i, chunk := range fh.f.entry.Chunks {
+			glog.V(3).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+		}
 
 		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
 		fh.f.entry.Chunks = chunks
 		// fh.f.entryViewCache = nil
-		fh.f.wfs.deleteFileChunks(ctx, garbages)
 
 		if _, err := client.CreateEntry(ctx, request); err != nil {
+			glog.Errorf("update fh: %v", err)
 			return fmt.Errorf("update fh: %v", err)
 		}
 
+		fh.f.wfs.deleteFileChunks(ctx, garbages)
+		for i, chunk := range garbages {
+			glog.V(3).Infof("garbage %s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+		}
+
 		return nil
 	})
 }
-
-func deleteFileIds(ctx context.Context, grpcDialOption grpc.DialOption, client filer_pb.SeaweedFilerClient, fileIds []string) error {
-
-	var vids []string
-	for _, fileId := range fileIds {
-		vids = append(vids, volumeId(fileId))
-	}
-
-	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
-
-		m := make(map[string]operation.LookupResult)
-
-		glog.V(4).Infof("remove file lookup volume id locations: %v", vids)
-		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
-			VolumeIds: vids,
-		})
-		if err != nil {
-			return m, err
-		}
-
-		for _, vid := range vids {
-			lr := operation.LookupResult{
-				VolumeId:  vid,
-				Locations: nil,
-			}
-			locations := resp.LocationsMap[vid]
-			for _, loc := range locations.Locations {
-				lr.Locations = append(lr.Locations, operation.Location{
-					Url:       loc.Url,
-					PublicUrl: loc.PublicUrl,
-				})
-			}
-			m[vid] = lr
-		}
-
-		return m, err
-	}
-
-	_, err := operation.DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
-
-	return err
-}
-
-func volumeId(fileId string) string {
-	lastCommaIndex := strings.LastIndex(fileId, ",")
-	if lastCommaIndex > 0 {
-		return fileId[:lastCommaIndex]
-	}
-	return fileId
-}

weed/filesys/wfs.go

@@ -29,9 +29,11 @@ type Option struct {
 	DirListingLimit    int
 	EntryCacheTtl      time.Duration
 
-	MountUid  uint32
-	MountGid  uint32
-	MountMode os.FileMode
+	MountUid   uint32
+	MountGid   uint32
+	MountMode  os.FileMode
+	MountCtime time.Time
+	MountMtime time.Time
 }
 
 var _ = fs.FS(&WFS{})
@@ -73,7 +75,7 @@ func (wfs *WFS) Root() (fs.Node, error) {
 	return &Dir{Path: wfs.option.FilerMountRootPath, wfs: wfs}, nil
 }
 
-func (wfs *WFS) withFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
+func (wfs *WFS) WithFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
 	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
@@ -133,7 +135,7 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
 
 	if wfs.stats.lastChecked < time.Now().Unix()-20 {
 
-		err := wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+		err := wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 			request := &filer_pb.StatisticsRequest{
 				Collection:  wfs.option.Collection,

weed/filesys/wfs_deletion.go

@@ -2,7 +2,12 @@ package filesys
 
 import (
 	"context"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
 )
 
 func (wfs *WFS) deleteFileChunks(ctx context.Context, chunks []*filer_pb.FileChunk) {
@@ -12,11 +17,53 @@ func (wfs *WFS) deleteFileChunks(ctx context.Context, chunks []*filer_pb.FileChu
 
 	var fileIds []string
 	for _, chunk := range chunks {
-		fileIds = append(fileIds, chunk.FileId)
+		fileIds = append(fileIds, chunk.GetFileIdString())
 	}
 
-	wfs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+	wfs.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 		deleteFileIds(ctx, wfs.option.GrpcDialOption, client, fileIds)
 		return nil
 	})
 }
+
+func deleteFileIds(ctx context.Context, grpcDialOption grpc.DialOption, client filer_pb.SeaweedFilerClient, fileIds []string) error {
+
+	var vids []string
+	for _, fileId := range fileIds {
+		vids = append(vids, filer2.VolumeId(fileId))
+	}
+
+	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
+
+		m := make(map[string]operation.LookupResult)
+
+		glog.V(4).Infof("remove file lookup volume id locations: %v", vids)
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+			VolumeIds: vids,
+		})
+		if err != nil {
+			return m, err
+		}
+
+		for _, vid := range vids {
+			lr := operation.LookupResult{
+				VolumeId:  vid,
+				Locations: nil,
+			}
+			locations := resp.LocationsMap[vid]
+			for _, loc := range locations.Locations {
+				lr.Locations = append(lr.Locations, operation.Location{
+					Url:       loc.Url,
+					PublicUrl: loc.PublicUrl,
+				})
+			}
+			m[vid] = lr
+		}
+
+		return m, err
+	}
+
+	_, err := operation.DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
+
+	return err
+}

weed/glide.yaml

@@ -1,14 +1,18 @@
 package: github.com/chrislusf/seaweedfs/weed
 import:
 - package: cloud.google.com/go
+  version: ^0.40.0
   subpackages:
   - pubsub
   - storage
 - package: github.com/Azure/azure-storage-blob-go
+  version: ^0.7.0
   subpackages:
   - azblob
 - package: github.com/Shopify/sarama
+  version: ^1.22.1
 - package: github.com/aws/aws-sdk-go
+  version: ^1.20.12
   subpackages:
   - aws
   - aws/awserr
@@ -17,45 +21,75 @@ import:
   - service/s3
   - service/s3/s3iface
   - service/sqs
-- package: github.com/boltdb/bolt
 - package: github.com/chrislusf/raft
   subpackages:
   - protobuf
 - package: github.com/dgrijalva/jwt-go
+  version: ^3.2.0
 - package: github.com/disintegration/imaging
+  version: ^1.6.0
 - package: github.com/dustin/go-humanize
+  version: ^1.0.0
+- package: github.com/gabriel-vasile/mimetype
+  version: ^0.3.14
 - package: github.com/go-redis/redis
+  version: ^6.15.3
 - package: github.com/go-sql-driver/mysql
+  version: ^1.4.1
 - package: github.com/gocql/gocql
 - package: github.com/golang/protobuf
+  version: ^1.3.1
   subpackages:
   - proto
 - package: github.com/google/btree
+  version: ^1.0.0
 - package: github.com/gorilla/mux
+  version: ^1.7.3
+- package: github.com/jacobsa/daemonize
+- package: github.com/kardianos/osext
 - package: github.com/karlseguin/ccache
+  version: ^2.0.3
 - package: github.com/klauspost/crc32
+  version: ^1.2.0
+- package: github.com/klauspost/reedsolomon
+  version: ^1.9.2
 - package: github.com/kurin/blazer
+  version: ^0.5.3
   subpackages:
   - b2
 - package: github.com/lib/pq
+  version: ^1.1.1
 - package: github.com/peterh/liner
+  version: ^1.1.0
+- package: github.com/prometheus/client_golang
+  version: ^1.0.0
+  subpackages:
+  - prometheus
+  - prometheus/push
 - package: github.com/rakyll/statik
+  version: ^0.1.6
   subpackages:
   - fs
 - package: github.com/rwcarlsen/goexif
   subpackages:
   - exif
 - package: github.com/satori/go.uuid
+  version: ^1.2.0
 - package: github.com/seaweedfs/fuse
   subpackages:
   - fs
 - package: github.com/spf13/viper
+  version: ^1.4.0
 - package: github.com/syndtr/goleveldb
+  version: ^1.0.0
   subpackages:
   - leveldb
+  - leveldb/opt
   - leveldb/util
 - package: github.com/willf/bloom
+  version: ^2.0.3
 - package: gocloud.dev
+  version: ^0.15.0
   subpackages:
   - pubsub
   - pubsub/awssnssqs
@@ -66,13 +100,16 @@ import:
 - package: golang.org/x/net
   subpackages:
   - context
+  - webdav
 - package: golang.org/x/tools
   subpackages:
   - godoc/util
 - package: google.golang.org/api
+  version: ^0.7.0
   subpackages:
   - option
 - package: google.golang.org/grpc
+  version: ^1.21.1
   subpackages:
   - credentials
   - keepalive

weed/operation/assign_file_id.go

@@ -43,7 +43,7 @@ func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *Volum
 			continue
 		}
 
-		lastError = withMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+		lastError = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 			req := &master_pb.AssignRequest{
 				Count:       primaryRequest.Count,

weed/operation/chunked_file.go

@@ -4,11 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"google.golang.org/grpc"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"sort"
 
+	"google.golang.org/grpc"
+
 	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -54,7 +56,7 @@ func (s ChunkList) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 func LoadChunkManifest(buffer []byte, isGzipped bool) (*ChunkManifest, error) {
 	if isGzipped {
 		var err error
-		if buffer, err = UnGzipData(buffer); err != nil {
+		if buffer, err = util.UnGzipData(buffer); err != nil {
 			return nil, err
 		}
 	}
@@ -103,7 +105,10 @@ func readChunkNeedle(fileUrl string, w io.Writer, offset int64) (written int64,
 	if err != nil {
 		return written, err
 	}
-	defer resp.Body.Close()
+	defer func() {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}()
 
 	switch resp.StatusCode {
 	case http.StatusRequestedRangeNotSatisfiable:

weed/operation/delete_content.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"google.golang.org/grpc"
 	"net/http"
@@ -84,8 +85,8 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str
 		}
 	}
 
+	resultChan := make(chan []*volume_server_pb.DeleteResult, len(server_to_fileIds))
 	var wg sync.WaitGroup
-
 	for server, fidList := range server_to_fileIds {
 		wg.Add(1)
 		go func(server string, fidList []string) {
@@ -94,12 +95,19 @@ func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []str
 			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList); deleteErr != nil {
 				err = deleteErr
 			} else {
-				ret = append(ret, deleteResults...)
+				resultChan <- deleteResults
 			}
 
 		}(server, fidList)
 	}
 	wg.Wait()
+	close(resultChan)
+
+	for result := range resultChan {
+		ret = append(ret, result...)
+	}
+
+	glog.V(0).Infof("deleted %d items", len(ret))
 
 	return ret, err
 }

weed/operation/grpc_client.go

@@ -38,13 +38,13 @@ func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err err
 	return fmt.Sprintf("%s:%d", volumeServer[0:sepIndex], port+10000), nil
 }
 
-func withMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
+func WithMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
 
 	ctx := context.Background()
 
 	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer)
 	if parseErr != nil {
-		return fmt.Errorf("failed to parse master grpc %v", masterServer)
+		return fmt.Errorf("failed to parse master grpc %v: %v", masterServer, parseErr)
 	}
 
 	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {

weed/operation/lookup.go

@@ -99,7 +99,7 @@ func LookupVolumeIds(server string, grpcDialOption grpc.DialOption, vids []strin
 
 	//only query unknown_vids
 
-	err := withMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+	err := WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 		req := &master_pb.LookupVolumeRequest{
 			VolumeIds: unknown_vids,

weed/operation/stats.go

@@ -9,7 +9,7 @@ import (
 
 func Statistics(server string, grpcDialOption grpc.DialOption, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
 
-	err = withMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+	err = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 		grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req)
 		if grpcErr != nil {

weed/operation/tail_volume.go

@@ -0,0 +1,82 @@
+package operation
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+	"google.golang.org/grpc"
+)
+
+func TailVolume(master string, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, timeoutSeconds int, fn func(n *needle.Needle) error) error {
+	// find volume location, replication, ttl info
+	lookup, err := Lookup(master, vid.String())
+	if err != nil {
+		return fmt.Errorf("look up volume %d: %v", vid, err)
+	}
+	if len(lookup.Locations) == 0 {
+		return fmt.Errorf("unable to locate volume %d", vid)
+	}
+
+	volumeServer := lookup.Locations[0].Url
+
+	return TailVolumeFromSource(volumeServer, grpcDialOption, vid, sinceNs, timeoutSeconds, fn)
+}
+
+func TailVolumeFromSource(volumeServer string, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, idleTimeoutSeconds int, fn func(n *needle.Needle) error) error {
+	return WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+
+		stream, err := client.VolumeTailSender(context.Background(), &volume_server_pb.VolumeTailSenderRequest{
+			VolumeId:           uint32(vid),
+			SinceNs:            sinceNs,
+			IdleTimeoutSeconds: uint32(idleTimeoutSeconds),
+		})
+		if err != nil {
+			return err
+		}
+
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			needleHeader := resp.NeedleHeader
+			needleBody := resp.NeedleBody
+
+			if len(needleHeader) == 0 {
+				continue
+			}
+
+			for !resp.IsLastChunk {
+				resp, recvErr = stream.Recv()
+				if recvErr != nil {
+					if recvErr == io.EOF {
+						break
+					} else {
+						return recvErr
+					}
+				}
+				needleBody = append(needleBody, resp.NeedleBody...)
+			}
+
+			n := new(needle.Needle)
+			n.ParseNeedleHeader(needleHeader)
+			n.ReadNeedleBodyBytes(needleBody, needle.CurrentVersion)
+
+			err = fn(n)
+
+			if err != nil {
+				return err
+			}
+
+		}
+		return nil
+	})
+}

weed/operation/upload_content.go

@@ -2,6 +2,8 @@ package operation
 
 import (
 	"bytes"
+	"compress/flate"
+	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -16,6 +18,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type UploadResult struct {
@@ -37,13 +40,43 @@ func init() {
 
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
-// Upload sends a POST request to a volume server to upload the content
-func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
-	return upload_content(uploadUrl, func(w io.Writer) (err error) {
-		_, err = io.Copy(w, reader)
-		return
-	}, filename, isGzipped, mtype, pairMap, jwt)
+// Upload sends a POST request to a volume server to upload the content with adjustable compression level
+func UploadWithLocalCompressionLevel(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt, compressionLevel int) (*UploadResult, error) {
+	if compressionLevel < 1 {
+		compressionLevel = 1
+	}
+	if compressionLevel > 9 {
+		compressionLevel = 9
+	}
+	return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, compressionLevel, jwt)
 }
+
+// Upload sends a POST request to a volume server to upload the content with fast compression
+func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
+	return doUpload(uploadUrl, filename, reader, isGzipped, mtype, pairMap, flate.BestSpeed, jwt)
+}
+
+func doUpload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, compression int, jwt security.EncodedJwt) (*UploadResult, error) {
+	contentIsGzipped := isGzipped
+	shouldGzipNow := false
+	if !isGzipped {
+		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
+			shouldGzipNow = true
+			contentIsGzipped = true
+		}
+	}
+	return upload_content(uploadUrl, func(w io.Writer) (err error) {
+		if shouldGzipNow {
+			gzWriter, _ := gzip.NewWriterLevel(w, compression)
+			_, err = io.Copy(gzWriter, reader)
+			gzWriter.Close()
+		} else {
+			_, err = io.Copy(w, reader)
+		}
+		return
+	}, filename, contentIsGzipped, mtype, pairMap, jwt)
+}
+
 func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error, filename string, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	body_buf := bytes.NewBufferString("")
 	body_writer := multipart.NewWriter(body_buf)

weed/pb/filer.proto

@@ -39,6 +39,9 @@ service SeaweedFiler {
     rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
     }
 
+    rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
+    }
+
 }
 
 //////////////////////////////////////////////////
@@ -72,19 +75,33 @@ message Entry {
     map<string, bytes> extended = 5;
 }
 
+message FullEntry {
+    string dir = 1;
+    Entry entry = 2;
+}
+
 message EventNotification {
     Entry old_entry = 1;
     Entry new_entry = 2;
     bool delete_chunks = 3;
+    string new_parent_path = 4;
 }
 
 message FileChunk {
-    string file_id = 1;
+    string file_id = 1; // to be deprecated
     int64 offset = 2;
     uint64 size = 3;
     int64 mtime = 4;
     string e_tag = 5;
-    string source_file_id = 6;
+    string source_file_id = 6; // to be deprecated
+    FileId fid = 7;
+    FileId source_fid = 8;
+}
+
+message FileId {
+    uint32 volume_id = 1;
+    uint64 file_key = 2;
+    fixed32 cookie = 3;
 }
 
 message FuseAttributes {
@@ -191,3 +208,12 @@ message StatisticsResponse {
     uint64 used_size = 5;
     uint64 file_count = 6;
 }
+
+message GetFilerConfigurationRequest {
+}
+message GetFilerConfigurationResponse {
+    repeated string masters = 1;
+    string replication = 2;
+    string collection = 3;
+    uint32 max_mb = 4;
+}

weed/pb/filer_pb/filer.pb.go

@@ -14,8 +14,10 @@ It has these top-level messages:
 	ListEntriesRequest
 	ListEntriesResponse
 	Entry
+	FullEntry
 	EventNotification
 	FileChunk
+	FileId
 	FuseAttributes
 	CreateEntryRequest
 	CreateEntryResponse
@@ -35,6 +37,8 @@ It has these top-level messages:
 	DeleteCollectionResponse
 	StatisticsRequest
 	StatisticsResponse
+	GetFilerConfigurationRequest
+	GetFilerConfigurationResponse
 */
 package filer_pb
 
@@ -210,16 +214,41 @@ func (m *Entry) GetExtended() map[string][]byte {
 	return nil
 }
 
+type FullEntry struct {
+	Dir   string `protobuf:"bytes,1,opt,name=dir" json:"dir,omitempty"`
+	Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
+}
+
+func (m *FullEntry) Reset()                    { *m = FullEntry{} }
+func (m *FullEntry) String() string            { return proto.CompactTextString(m) }
+func (*FullEntry) ProtoMessage()               {}
+func (*FullEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+func (m *FullEntry) GetDir() string {
+	if m != nil {
+		return m.Dir
+	}
+	return ""
+}
+
+func (m *FullEntry) GetEntry() *Entry {
+	if m != nil {
+		return m.Entry
+	}
+	return nil
+}
+
 type EventNotification struct {
-	OldEntry     *Entry `protobuf:"bytes,1,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"`
-	NewEntry     *Entry `protobuf:"bytes,2,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"`
-	DeleteChunks bool   `protobuf:"varint,3,opt,name=delete_chunks,json=deleteChunks" json:"delete_chunks,omitempty"`
+	OldEntry      *Entry `protobuf:"bytes,1,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"`
+	NewEntry      *Entry `protobuf:"bytes,2,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"`
+	DeleteChunks  bool   `protobuf:"varint,3,opt,name=delete_chunks,json=deleteChunks" json:"delete_chunks,omitempty"`
+	NewParentPath string `protobuf:"bytes,4,opt,name=new_parent_path,json=newParentPath" json:"new_parent_path,omitempty"`
 }
 
 func (m *EventNotification) Reset()                    { *m = EventNotification{} }
 func (m *EventNotification) String() string            { return proto.CompactTextString(m) }
 func (*EventNotification) ProtoMessage()               {}
-func (*EventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+func (*EventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
 
 func (m *EventNotification) GetOldEntry() *Entry {
 	if m != nil {
@@ -242,19 +271,28 @@ func (m *EventNotification) GetDeleteChunks() bool {
 	return false
 }
 
+func (m *EventNotification) GetNewParentPath() string {
+	if m != nil {
+		return m.NewParentPath
+	}
+	return ""
+}
+
 type FileChunk struct {
-	FileId       string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
-	Offset       int64  `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
-	Size         uint64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
-	Mtime        int64  `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
-	ETag         string `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
-	SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
+	FileId       string  `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
+	Offset       int64   `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
+	Size         uint64  `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
+	Mtime        int64   `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
+	ETag         string  `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
+	SourceFileId string  `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
+	Fid          *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"`
+	SourceFid    *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"`
 }
 
 func (m *FileChunk) Reset()                    { *m = FileChunk{} }
 func (m *FileChunk) String() string            { return proto.CompactTextString(m) }
 func (*FileChunk) ProtoMessage()               {}
-func (*FileChunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+func (*FileChunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
 
 func (m *FileChunk) GetFileId() string {
 	if m != nil {
@@ -298,6 +336,52 @@ func (m *FileChunk) GetSourceFileId() string {
 	return ""
 }
 
+func (m *FileChunk) GetFid() *FileId {
+	if m != nil {
+		return m.Fid
+	}
+	return nil
+}
+
+func (m *FileChunk) GetSourceFid() *FileId {
+	if m != nil {
+		return m.SourceFid
+	}
+	return nil
+}
+
+type FileId struct {
+	VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+	FileKey  uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"`
+	Cookie   uint32 `protobuf:"fixed32,3,opt,name=cookie" json:"cookie,omitempty"`
+}
+
+func (m *FileId) Reset()                    { *m = FileId{} }
+func (m *FileId) String() string            { return proto.CompactTextString(m) }
+func (*FileId) ProtoMessage()               {}
+func (*FileId) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *FileId) GetVolumeId() uint32 {
+	if m != nil {
+		return m.VolumeId
+	}
+	return 0
+}
+
+func (m *FileId) GetFileKey() uint64 {
+	if m != nil {
+		return m.FileKey
+	}
+	return 0
+}
+
+func (m *FileId) GetCookie() uint32 {
+	if m != nil {
+		return m.Cookie
+	}
+	return 0
+}
+
 type FuseAttributes struct {
 	FileSize      uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
 	Mtime         int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
@@ -317,7 +401,7 @@ type FuseAttributes struct {
 func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
 func (m *FuseAttributes) String() string            { return proto.CompactTextString(m) }
 func (*FuseAttributes) ProtoMessage()               {}
-func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
 
 func (m *FuseAttributes) GetFileSize() uint64 {
 	if m != nil {
@@ -418,7 +502,7 @@ type CreateEntryRequest struct {
 func (m *CreateEntryRequest) Reset()                    { *m = CreateEntryRequest{} }
 func (m *CreateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryRequest) ProtoMessage()               {}
-func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
 
 func (m *CreateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -440,7 +524,7 @@ type CreateEntryResponse struct {
 func (m *CreateEntryResponse) Reset()                    { *m = CreateEntryResponse{} }
 func (m *CreateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryResponse) ProtoMessage()               {}
-func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
 
 type UpdateEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@@ -450,7 +534,7 @@ type UpdateEntryRequest struct {
 func (m *UpdateEntryRequest) Reset()                    { *m = UpdateEntryRequest{} }
 func (m *UpdateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryRequest) ProtoMessage()               {}
-func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
 
 func (m *UpdateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -472,7 +556,7 @@ type UpdateEntryResponse struct {
 func (m *UpdateEntryResponse) Reset()                    { *m = UpdateEntryResponse{} }
 func (m *UpdateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryResponse) ProtoMessage()               {}
-func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
 
 type DeleteEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@@ -485,7 +569,7 @@ type DeleteEntryRequest struct {
 func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
 func (m *DeleteEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryRequest) ProtoMessage()               {}
-func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
 
 func (m *DeleteEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -521,7 +605,7 @@ type DeleteEntryResponse struct {
 func (m *DeleteEntryResponse) Reset()                    { *m = DeleteEntryResponse{} }
 func (m *DeleteEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryResponse) ProtoMessage()               {}
-func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
 
 type AtomicRenameEntryRequest struct {
 	OldDirectory string `protobuf:"bytes,1,opt,name=old_directory,json=oldDirectory" json:"old_directory,omitempty"`
@@ -533,7 +617,7 @@ type AtomicRenameEntryRequest struct {
 func (m *AtomicRenameEntryRequest) Reset()                    { *m = AtomicRenameEntryRequest{} }
 func (m *AtomicRenameEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*AtomicRenameEntryRequest) ProtoMessage()               {}
-func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
 
 func (m *AtomicRenameEntryRequest) GetOldDirectory() string {
 	if m != nil {
@@ -569,7 +653,7 @@ type AtomicRenameEntryResponse struct {
 func (m *AtomicRenameEntryResponse) Reset()                    { *m = AtomicRenameEntryResponse{} }
 func (m *AtomicRenameEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*AtomicRenameEntryResponse) ProtoMessage()               {}
-func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
 
 type AssignVolumeRequest struct {
 	Count       int32  `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
@@ -582,7 +666,7 @@ type AssignVolumeRequest struct {
 func (m *AssignVolumeRequest) Reset()                    { *m = AssignVolumeRequest{} }
 func (m *AssignVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeRequest) ProtoMessage()               {}
-func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
 
 func (m *AssignVolumeRequest) GetCount() int32 {
 	if m != nil {
@@ -630,7 +714,7 @@ type AssignVolumeResponse struct {
 func (m *AssignVolumeResponse) Reset()                    { *m = AssignVolumeResponse{} }
 func (m *AssignVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeResponse) ProtoMessage()               {}
-func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
 
 func (m *AssignVolumeResponse) GetFileId() string {
 	if m != nil {
@@ -674,7 +758,7 @@ type LookupVolumeRequest struct {
 func (m *LookupVolumeRequest) Reset()                    { *m = LookupVolumeRequest{} }
 func (m *LookupVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeRequest) ProtoMessage()               {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
 
 func (m *LookupVolumeRequest) GetVolumeIds() []string {
 	if m != nil {
@@ -690,7 +774,7 @@ type Locations struct {
 func (m *Locations) Reset()                    { *m = Locations{} }
 func (m *Locations) String() string            { return proto.CompactTextString(m) }
 func (*Locations) ProtoMessage()               {}
-func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
 
 func (m *Locations) GetLocations() []*Location {
 	if m != nil {
@@ -707,7 +791,7 @@ type Location struct {
 func (m *Location) Reset()                    { *m = Location{} }
 func (m *Location) String() string            { return proto.CompactTextString(m) }
 func (*Location) ProtoMessage()               {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
 
 func (m *Location) GetUrl() string {
 	if m != nil {
@@ -730,7 +814,7 @@ type LookupVolumeResponse struct {
 func (m *LookupVolumeResponse) Reset()                    { *m = LookupVolumeResponse{} }
 func (m *LookupVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse) ProtoMessage()               {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
 
 func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations {
 	if m != nil {
@@ -746,7 +830,7 @@ type DeleteCollectionRequest struct {
 func (m *DeleteCollectionRequest) Reset()                    { *m = DeleteCollectionRequest{} }
 func (m *DeleteCollectionRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionRequest) ProtoMessage()               {}
-func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
 
 func (m *DeleteCollectionRequest) GetCollection() string {
 	if m != nil {
@@ -761,7 +845,7 @@ type DeleteCollectionResponse struct {
 func (m *DeleteCollectionResponse) Reset()                    { *m = DeleteCollectionResponse{} }
 func (m *DeleteCollectionResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionResponse) ProtoMessage()               {}
-func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
 type StatisticsRequest struct {
 	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
@@ -772,7 +856,7 @@ type StatisticsRequest struct {
 func (m *StatisticsRequest) Reset()                    { *m = StatisticsRequest{} }
 func (m *StatisticsRequest) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsRequest) ProtoMessage()               {}
-func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
 
 func (m *StatisticsRequest) GetReplication() string {
 	if m != nil {
@@ -807,7 +891,7 @@ type StatisticsResponse struct {
 func (m *StatisticsResponse) Reset()                    { *m = StatisticsResponse{} }
 func (m *StatisticsResponse) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsResponse) ProtoMessage()               {}
-func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
 
 func (m *StatisticsResponse) GetReplication() string {
 	if m != nil {
@@ -851,14 +935,64 @@ func (m *StatisticsResponse) GetFileCount() uint64 {
 	return 0
 }
 
+type GetFilerConfigurationRequest struct {
+}
+
+func (m *GetFilerConfigurationRequest) Reset()                    { *m = GetFilerConfigurationRequest{} }
+func (m *GetFilerConfigurationRequest) String() string            { return proto.CompactTextString(m) }
+func (*GetFilerConfigurationRequest) ProtoMessage()               {}
+func (*GetFilerConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+
+type GetFilerConfigurationResponse struct {
+	Masters     []string `protobuf:"bytes,1,rep,name=masters" json:"masters,omitempty"`
+	Replication string   `protobuf:"bytes,2,opt,name=replication" json:"replication,omitempty"`
+	Collection  string   `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
+	MaxMb       uint32   `protobuf:"varint,4,opt,name=max_mb,json=maxMb" json:"max_mb,omitempty"`
+}
+
+func (m *GetFilerConfigurationResponse) Reset()                    { *m = GetFilerConfigurationResponse{} }
+func (m *GetFilerConfigurationResponse) String() string            { return proto.CompactTextString(m) }
+func (*GetFilerConfigurationResponse) ProtoMessage()               {}
+func (*GetFilerConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+
+func (m *GetFilerConfigurationResponse) GetMasters() []string {
+	if m != nil {
+		return m.Masters
+	}
+	return nil
+}
+
+func (m *GetFilerConfigurationResponse) GetReplication() string {
+	if m != nil {
+		return m.Replication
+	}
+	return ""
+}
+
+func (m *GetFilerConfigurationResponse) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+func (m *GetFilerConfigurationResponse) GetMaxMb() uint32 {
+	if m != nil {
+		return m.MaxMb
+	}
+	return 0
+}
+
 func init() {
 	proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest")
 	proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse")
 	proto.RegisterType((*ListEntriesRequest)(nil), "filer_pb.ListEntriesRequest")
 	proto.RegisterType((*ListEntriesResponse)(nil), "filer_pb.ListEntriesResponse")
 	proto.RegisterType((*Entry)(nil), "filer_pb.Entry")
+	proto.RegisterType((*FullEntry)(nil), "filer_pb.FullEntry")
 	proto.RegisterType((*EventNotification)(nil), "filer_pb.EventNotification")
 	proto.RegisterType((*FileChunk)(nil), "filer_pb.FileChunk")
+	proto.RegisterType((*FileId)(nil), "filer_pb.FileId")
 	proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes")
 	proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest")
 	proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse")
@@ -878,6 +1012,8 @@ func init() {
 	proto.RegisterType((*DeleteCollectionResponse)(nil), "filer_pb.DeleteCollectionResponse")
 	proto.RegisterType((*StatisticsRequest)(nil), "filer_pb.StatisticsRequest")
 	proto.RegisterType((*StatisticsResponse)(nil), "filer_pb.StatisticsResponse")
+	proto.RegisterType((*GetFilerConfigurationRequest)(nil), "filer_pb.GetFilerConfigurationRequest")
+	proto.RegisterType((*GetFilerConfigurationResponse)(nil), "filer_pb.GetFilerConfigurationResponse")
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -901,6 +1037,7 @@ type SeaweedFilerClient interface {
 	LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error)
 	DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
 	Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
+	GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error)
 }
 
 type seaweedFilerClient struct {
@@ -1001,6 +1138,15 @@ func (c *seaweedFilerClient) Statistics(ctx context.Context, in *StatisticsReque
 	return out, nil
 }
 
+func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) {
+	out := new(GetFilerConfigurationResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFilerConfiguration", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for SeaweedFiler service
 
 type SeaweedFilerServer interface {
@@ -1014,6 +1160,7 @@ type SeaweedFilerServer interface {
 	LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error)
 	DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
 	Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
+	GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error)
 }
 
 func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) {
@@ -1200,6 +1347,24 @@ func _SeaweedFiler_Statistics_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetFilerConfigurationRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/GetFilerConfiguration",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).GetFilerConfiguration(ctx, req.(*GetFilerConfigurationRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "filer_pb.SeaweedFiler",
 	HandlerType: (*SeaweedFilerServer)(nil),
@@ -1244,6 +1409,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			MethodName: "Statistics",
 			Handler:    _SeaweedFiler_Statistics_Handler,
 		},
+		{
+			MethodName: "GetFilerConfiguration",
+			Handler:    _SeaweedFiler_GetFilerConfiguration_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "filer.proto",
@@ -1252,92 +1421,104 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1391 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xdd, 0x6e, 0xdc, 0x44,
-	0x14, 0xae, 0xf7, 0x2f, 0xf1, 0xd9, 0xdd, 0x92, 0x4c, 0x02, 0x75, 0x37, 0x49, 0xd9, 0x3a, 0x14,
-	0xa5, 0xa2, 0x8a, 0xaa, 0xc2, 0x45, 0x4b, 0x85, 0x44, 0x9b, 0xa6, 0x52, 0xa5, 0xb4, 0x45, 0x4e,
-	0x8b, 0x84, 0x90, 0xb0, 0x1c, 0x7b, 0xb2, 0x1d, 0xc5, 0x6b, 0x2f, 0x9e, 0x71, 0xd2, 0xf2, 0x08,
-	0xdc, 0x70, 0xc3, 0x15, 0x12, 0x17, 0x5c, 0xf1, 0x16, 0xdc, 0xf0, 0x14, 0xbc, 0x04, 0xcf, 0x80,
-	0xce, 0xcc, 0xd8, 0x3b, 0x5e, 0x3b, 0x29, 0x08, 0xf5, 0x6e, 0xe6, 0xfc, 0x7e, 0xe7, 0xcc, 0xf9,
-	0xb1, 0xa1, 0x7f, 0xcc, 0x62, 0x9a, 0xed, 0xce, 0xb2, 0x54, 0xa4, 0x64, 0x59, 0x5e, 0xfc, 0xd9,
-	0x91, 0xfb, 0x1c, 0x36, 0x0e, 0xd2, 0xf4, 0x24, 0x9f, 0x3d, 0x62, 0x19, 0x0d, 0x45, 0x9a, 0xbd,
-	0xd9, 0x4f, 0x44, 0xf6, 0xc6, 0xa3, 0xdf, 0xe7, 0x94, 0x0b, 0xb2, 0x09, 0x76, 0x54, 0x30, 0x1c,
-	0x6b, 0x6c, 0xed, 0xd8, 0xde, 0x9c, 0x40, 0x08, 0x74, 0x92, 0x60, 0x4a, 0x9d, 0x96, 0x64, 0xc8,
-	0xb3, 0xbb, 0x0f, 0x9b, 0xcd, 0x06, 0xf9, 0x2c, 0x4d, 0x38, 0x25, 0x37, 0xa0, 0x4b, 0x91, 0x20,
-	0xad, 0xf5, 0xef, 0xbc, 0xb7, 0x5b, 0x40, 0xd9, 0x55, 0x72, 0x8a, 0xeb, 0xfe, 0x61, 0x01, 0x39,
-	0x60, 0x5c, 0x20, 0x91, 0x51, 0xfe, 0xef, 0xf0, 0x7c, 0x00, 0xbd, 0x59, 0x46, 0x8f, 0xd9, 0x6b,
-	0x8d, 0x48, 0xdf, 0xc8, 0x2d, 0x58, 0xe5, 0x22, 0xc8, 0xc4, 0xe3, 0x2c, 0x9d, 0x3e, 0x66, 0x31,
-	0x7d, 0x86, 0xa0, 0xdb, 0x52, 0xa4, 0xce, 0x20, 0xbb, 0x40, 0x58, 0x12, 0xc6, 0x39, 0x67, 0xa7,
-	0xf4, 0xb0, 0xe0, 0x3a, 0x9d, 0xb1, 0xb5, 0xb3, 0xec, 0x35, 0x70, 0xc8, 0x3a, 0x74, 0x63, 0x36,
-	0x65, 0xc2, 0xe9, 0x8e, 0xad, 0x9d, 0xa1, 0xa7, 0x2e, 0xee, 0x97, 0xb0, 0x56, 0xc1, 0xaf, 0xc3,
-	0xbf, 0x09, 0x4b, 0x54, 0x91, 0x1c, 0x6b, 0xdc, 0x6e, 0x4a, 0x40, 0xc1, 0x77, 0x7f, 0x6d, 0x41,
-	0x57, 0x92, 0xca, 0x3c, 0x5b, 0xf3, 0x3c, 0x93, 0xeb, 0x30, 0x60, 0xdc, 0x9f, 0x27, 0xa3, 0x25,
-	0xf1, 0xf5, 0x19, 0x2f, 0xf3, 0x4e, 0x3e, 0x81, 0x5e, 0xf8, 0x2a, 0x4f, 0x4e, 0xb8, 0xd3, 0x96,
-	0xae, 0xd6, 0xe6, 0xae, 0x30, 0xd8, 0x3d, 0xe4, 0x79, 0x5a, 0x84, 0xdc, 0x05, 0x08, 0x84, 0xc8,
-	0xd8, 0x51, 0x2e, 0x28, 0x97, 0xd1, 0xf6, 0xef, 0x38, 0x86, 0x42, 0xce, 0xe9, 0x83, 0x92, 0xef,
-	0x19, 0xb2, 0xe4, 0x1e, 0x2c, 0xd3, 0xd7, 0x82, 0x26, 0x11, 0x8d, 0x9c, 0xae, 0x74, 0xb4, 0xb5,
-	0x10, 0xd3, 0xee, 0xbe, 0xe6, 0xab, 0x08, 0x4b, 0xf1, 0xd1, 0x7d, 0x18, 0x56, 0x58, 0x64, 0x05,
-	0xda, 0x27, 0xb4, 0x78, 0x59, 0x3c, 0x62, 0x76, 0x4f, 0x83, 0x38, 0x57, 0x45, 0x36, 0xf0, 0xd4,
-	0xe5, 0xf3, 0xd6, 0x5d, 0xcb, 0xfd, 0xd9, 0x82, 0xd5, 0xfd, 0x53, 0x9a, 0x88, 0x67, 0xa9, 0x60,
-	0xc7, 0x2c, 0x0c, 0x04, 0x4b, 0x13, 0x72, 0x0b, 0xec, 0x34, 0x8e, 0xfc, 0x0b, 0x6b, 0x6c, 0x39,
-	0x8d, 0xb5, 0xbf, 0x5b, 0x60, 0x27, 0xf4, 0x4c, 0x4b, 0xb7, 0xce, 0x91, 0x4e, 0xe8, 0x99, 0x92,
-	0xde, 0x86, 0x61, 0x44, 0x63, 0x2a, 0xa8, 0x5f, 0xe6, 0x15, 0x93, 0x3e, 0x50, 0x44, 0x99, 0x4f,
-	0xee, 0xfe, 0x66, 0x81, 0x5d, 0xa6, 0x97, 0x5c, 0x81, 0x25, 0x34, 0xe7, 0xb3, 0x48, 0x07, 0xd5,
-	0xc3, 0xeb, 0x93, 0x08, 0x6b, 0x35, 0x3d, 0x3e, 0xe6, 0x54, 0x48, 0xb7, 0x6d, 0x4f, 0xdf, 0xf0,
-	0xad, 0x39, 0xfb, 0x41, 0x95, 0x67, 0xc7, 0x93, 0x67, 0xcc, 0xc1, 0x54, 0xb0, 0x29, 0x95, 0xcf,
-	0xd2, 0xf6, 0xd4, 0x85, 0xac, 0x41, 0x97, 0xfa, 0x22, 0x98, 0xc8, 0xba, 0xb3, 0xbd, 0x0e, 0x7d,
-	0x11, 0x4c, 0xc8, 0x47, 0x70, 0x99, 0xa7, 0x79, 0x16, 0x52, 0xbf, 0x70, 0xdb, 0x93, 0xdc, 0x81,
-	0xa2, 0x3e, 0x96, 0xce, 0xdd, 0xbf, 0x5b, 0x70, 0xb9, 0xfa, 0xa2, 0x64, 0x03, 0x6c, 0xa9, 0x21,
-	0x9d, 0x5b, 0xd2, 0xb9, 0x9c, 0x12, 0x87, 0x15, 0x00, 0x2d, 0x13, 0x40, 0xa1, 0x32, 0x4d, 0x23,
-	0x85, 0x77, 0xa8, 0x54, 0x9e, 0xa6, 0x11, 0xc5, 0x97, 0xcc, 0x59, 0x24, 0x11, 0x0f, 0x3d, 0x3c,
-	0x22, 0x65, 0xc2, 0x22, 0xdd, 0x25, 0x78, 0xc4, 0x1c, 0x84, 0x99, 0xb4, 0xdb, 0x53, 0x39, 0x50,
-	0x37, 0xcc, 0xc1, 0x14, 0xa9, 0x4b, 0x2a, 0x30, 0x3c, 0x93, 0x31, 0xf4, 0x33, 0x3a, 0x8b, 0xf5,
-	0x33, 0x3b, 0xcb, 0x92, 0x65, 0x92, 0xc8, 0x35, 0x80, 0x30, 0x8d, 0x63, 0x1a, 0x4a, 0x01, 0x5b,
-	0x0a, 0x18, 0x14, 0x7c, 0x0a, 0x21, 0x62, 0x9f, 0xd3, 0xd0, 0x81, 0xb1, 0xb5, 0xd3, 0xf5, 0x7a,
-	0x42, 0xc4, 0x87, 0x34, 0xc4, 0x38, 0x72, 0x4e, 0x33, 0x5f, 0xf6, 0x58, 0x5f, 0xea, 0x2d, 0x23,
-	0x41, 0x4e, 0x83, 0x2d, 0x80, 0x49, 0x96, 0xe6, 0x33, 0xc5, 0x1d, 0x8c, 0xdb, 0x38, 0x72, 0x24,
-	0x45, 0xb2, 0x6f, 0xc0, 0x65, 0xfe, 0x66, 0x1a, 0xb3, 0xe4, 0xc4, 0x17, 0x41, 0x36, 0xa1, 0xc2,
-	0x19, 0x4a, 0x03, 0x43, 0x4d, 0x7d, 0x21, 0x89, 0xee, 0x37, 0x40, 0xf6, 0x32, 0x1a, 0x08, 0xfa,
-	0x1f, 0xa6, 0x6b, 0x39, 0x29, 0x5b, 0x17, 0x4e, 0xca, 0xf7, 0x61, 0xad, 0x62, 0x5a, 0x0d, 0x1a,
-	0xf4, 0xf8, 0x72, 0x16, 0xbd, 0x2b, 0x8f, 0x15, 0xd3, 0xda, 0xe3, 0x4f, 0x16, 0x90, 0x47, 0xb2,
-	0x13, 0xfe, 0xdf, 0x0a, 0xc1, 0x1a, 0xc6, 0xd1, 0xa6, 0x3a, 0x2d, 0x0a, 0x44, 0xa0, 0x87, 0xef,
-	0x80, 0x71, 0x65, 0xff, 0x51, 0x20, 0x02, 0x3d, 0x00, 0x33, 0x1a, 0xe6, 0x19, 0xce, 0x63, 0x59,
-	0x57, 0x72, 0x00, 0x7a, 0x05, 0x09, 0x81, 0x56, 0x00, 0x69, 0xa0, 0xbf, 0x58, 0xe0, 0x3c, 0x10,
-	0xe9, 0x94, 0x85, 0x1e, 0x45, 0x87, 0x15, 0xb8, 0xdb, 0x30, 0xc4, 0xf9, 0xb1, 0x08, 0x79, 0x90,
-	0xc6, 0xd1, 0x7c, 0xb2, 0x5e, 0x05, 0x1c, 0x21, 0xbe, 0x81, 0x7c, 0x29, 0x8d, 0x23, 0x59, 0x10,
-	0xdb, 0x30, 0xc4, 0x89, 0x32, 0xd7, 0x57, 0x7b, 0x66, 0x90, 0xd0, 0xb3, 0x8a, 0x3e, 0x0a, 0x49,
-	0xfd, 0x8e, 0xd2, 0x4f, 0xe8, 0x19, 0xea, 0xbb, 0x1b, 0x70, 0xb5, 0x01, 0x9b, 0x46, 0xfe, 0xbb,
-	0x05, 0x6b, 0x0f, 0x38, 0x67, 0x93, 0xe4, 0xeb, 0x34, 0xce, 0xa7, 0xb4, 0x00, 0xbd, 0x0e, 0xdd,
-	0x30, 0xcd, 0x13, 0x21, 0xc1, 0x76, 0x3d, 0x75, 0x59, 0x68, 0x88, 0x56, 0xad, 0x21, 0x16, 0x5a,
-	0xaa, 0x5d, 0x6f, 0x29, 0xa3, 0x65, 0x3a, 0x95, 0x96, 0xf9, 0x10, 0xfa, 0xf8, 0x30, 0x7e, 0x48,
-	0x13, 0x41, 0x33, 0x3d, 0x81, 0x00, 0x49, 0x7b, 0x92, 0xe2, 0xfe, 0x68, 0xc1, 0x7a, 0x15, 0xa9,
-	0x5e, 0x80, 0xe7, 0x0e, 0x44, 0x1c, 0x18, 0x59, 0xac, 0x61, 0xe2, 0x11, 0x5b, 0x6f, 0x96, 0x1f,
-	0xc5, 0x2c, 0xf4, 0x91, 0xa1, 0xe0, 0xd9, 0x8a, 0xf2, 0x32, 0x8b, 0xe7, 0x41, 0x77, 0xcc, 0xa0,
-	0x09, 0x74, 0x82, 0x5c, 0xbc, 0x2a, 0x86, 0x22, 0x9e, 0xdd, 0xcf, 0x60, 0x4d, 0x7d, 0x93, 0x54,
-	0xb3, 0xb6, 0x05, 0x70, 0x2a, 0x09, 0x3e, 0x8b, 0xd4, 0x3a, 0xb6, 0x3d, 0x5b, 0x51, 0x9e, 0x44,
-	0xdc, 0xfd, 0x02, 0xec, 0x83, 0x54, 0x25, 0x82, 0x93, 0xdb, 0x60, 0xc7, 0xc5, 0x45, 0x6f, 0x6e,
-	0x32, 0x6f, 0x8f, 0x42, 0xce, 0x9b, 0x0b, 0xb9, 0xf7, 0x61, 0xb9, 0x20, 0x17, 0xb1, 0x59, 0xe7,
-	0xc5, 0xd6, 0x5a, 0x88, 0xcd, 0xfd, 0xd3, 0x82, 0xf5, 0x2a, 0x64, 0x9d, 0xbe, 0x97, 0x30, 0x2c,
-	0x5d, 0xf8, 0xd3, 0x60, 0xa6, 0xb1, 0xdc, 0x36, 0xb1, 0xd4, 0xd5, 0x4a, 0x80, 0xfc, 0x69, 0x30,
-	0x53, 0x25, 0x35, 0x88, 0x0d, 0xd2, 0xe8, 0x05, 0xac, 0xd6, 0x44, 0x1a, 0x96, 0xf1, 0x4d, 0x73,
-	0x19, 0x57, 0x3e, 0x28, 0x4a, 0x6d, 0x73, 0x43, 0xdf, 0x83, 0x2b, 0xaa, 0xff, 0xf6, 0xca, 0xa2,
-	0x2b, 0x72, 0x5f, 0xad, 0x4d, 0x6b, 0xb1, 0x36, 0xdd, 0x11, 0x38, 0x75, 0x55, 0xdd, 0x05, 0x13,
-	0x58, 0x3d, 0x14, 0x81, 0x60, 0x5c, 0xb0, 0xb0, 0xfc, 0x32, 0x5c, 0x28, 0x66, 0xeb, 0x6d, 0xfb,
-	0xa1, 0xde, 0x0e, 0x2b, 0xd0, 0x16, 0xa2, 0xa8, 0x33, 0x3c, 0xe2, 0x2b, 0x10, 0xd3, 0x93, 0x7e,
-	0x83, 0x77, 0xe0, 0x0a, 0xeb, 0x41, 0xa4, 0x22, 0x88, 0xd5, 0xfe, 0xed, 0xc8, 0xfd, 0x6b, 0x4b,
-	0x8a, 0x5c, 0xc0, 0x6a, 0x45, 0x45, 0x8a, 0xdb, 0x55, 0xdb, 0x19, 0x09, 0x92, 0xb9, 0x05, 0x20,
-	0x5b, 0x4a, 0x75, 0x43, 0x4f, 0xe9, 0x22, 0x65, 0x0f, 0x09, 0x77, 0xfe, 0xea, 0xc1, 0xe0, 0x90,
-	0x06, 0x67, 0x94, 0x46, 0xb8, 0xfe, 0x33, 0x32, 0x29, 0x6a, 0xab, 0xfa, 0x89, 0x4e, 0x6e, 0x2c,
-	0x16, 0x51, 0xe3, 0x3f, 0xc1, 0xe8, 0xe3, 0xb7, 0x89, 0xe9, 0x67, 0xba, 0x44, 0x0e, 0xa0, 0x6f,
-	0x7c, 0x03, 0x93, 0x4d, 0x43, 0xb1, 0xf6, 0x69, 0x3f, 0xda, 0x3a, 0x87, 0x6b, 0x5a, 0x33, 0x16,
-	0x9d, 0x69, 0xad, 0xbe, 0x5a, 0x4d, 0x6b, 0x4d, 0xdb, 0x51, 0x5a, 0x33, 0x96, 0x98, 0x69, 0xad,
-	0xbe, 0x36, 0x4d, 0x6b, 0x4d, 0x9b, 0x4f, 0x5a, 0x33, 0x36, 0x8d, 0x69, 0xad, 0xbe, 0x11, 0x4d,
-	0x6b, 0x4d, 0xeb, 0xe9, 0x12, 0xf9, 0x0e, 0x56, 0x6b, 0x3b, 0x80, 0xb8, 0x73, 0xad, 0xf3, 0x96,
-	0xd7, 0x68, 0xfb, 0x42, 0x99, 0xd2, 0xfe, 0x73, 0x18, 0x98, 0xb3, 0x99, 0x18, 0x80, 0x1a, 0xb6,
-	0xcb, 0xe8, 0xda, 0x79, 0x6c, 0xd3, 0xa0, 0x39, 0x76, 0x4c, 0x83, 0x0d, 0x83, 0xd7, 0x34, 0xd8,
-	0x34, 0xad, 0xdc, 0x4b, 0xe4, 0x5b, 0x58, 0x59, 0x6c, 0x7f, 0x72, 0x7d, 0x31, 0x6d, 0xb5, 0xa9,
-	0x32, 0x72, 0x2f, 0x12, 0x29, 0x8d, 0x3f, 0x01, 0x98, 0x77, 0x35, 0xd9, 0x98, 0xeb, 0xd4, 0xa6,
-	0xca, 0x68, 0xb3, 0x99, 0x59, 0x98, 0x7a, 0x78, 0x0d, 0x56, 0xb8, 0x6a, 0xad, 0x63, 0xbe, 0x1b,
-	0xc6, 0x8c, 0x26, 0xe2, 0x21, 0xc8, 0x2e, 0xfb, 0x0a, 0x7f, 0xb4, 0x8f, 0x7a, 0xf2, 0x7f, 0xfb,
-	0xd3, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x11, 0xc9, 0xda, 0x7e, 0x0f, 0x00, 0x00,
+	// 1583 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xdb, 0x6f, 0xdc, 0x44,
+	0x17, 0xaf, 0xf7, 0xee, 0xb3, 0xbb, 0x6d, 0x32, 0x49, 0xbf, 0xba, 0x9b, 0xcb, 0x97, 0x3a, 0x5f,
+	0xfb, 0xa5, 0xa2, 0x0a, 0x55, 0xe1, 0xa1, 0xa5, 0x42, 0xa2, 0xcd, 0x05, 0x45, 0xa4, 0x17, 0x39,
+	0x2d, 0x02, 0x21, 0x61, 0x39, 0xf6, 0xec, 0x66, 0x88, 0xed, 0x59, 0xec, 0x71, 0x92, 0xf2, 0x27,
+	0xf0, 0x82, 0xc4, 0x23, 0x12, 0xcf, 0xfc, 0x13, 0x88, 0x17, 0xc4, 0xbf, 0xc3, 0x23, 0xcf, 0x68,
+	0x2e, 0xf6, 0x8e, 0xd7, 0x9b, 0xa4, 0x08, 0xf5, 0xcd, 0x73, 0xae, 0xbf, 0x73, 0xe6, 0x5c, 0x66,
+	0x17, 0xba, 0x43, 0x12, 0xe2, 0x64, 0x73, 0x9c, 0x50, 0x46, 0x51, 0x47, 0x1c, 0xdc, 0xf1, 0xa1,
+	0xfd, 0x02, 0x96, 0xf6, 0x29, 0x3d, 0xce, 0xc6, 0xdb, 0x24, 0xc1, 0x3e, 0xa3, 0xc9, 0x9b, 0x9d,
+	0x98, 0x25, 0x6f, 0x1c, 0xfc, 0x6d, 0x86, 0x53, 0x86, 0x96, 0xc1, 0x0c, 0x72, 0x86, 0x65, 0xac,
+	0x19, 0x1b, 0xa6, 0x33, 0x21, 0x20, 0x04, 0x8d, 0xd8, 0x8b, 0xb0, 0x55, 0x13, 0x0c, 0xf1, 0x6d,
+	0xef, 0xc0, 0xf2, 0x6c, 0x83, 0xe9, 0x98, 0xc6, 0x29, 0x46, 0xb7, 0xa1, 0x89, 0x39, 0x41, 0x58,
+	0xeb, 0x3e, 0xb8, 0xb6, 0x99, 0x43, 0xd9, 0x94, 0x72, 0x92, 0x6b, 0xff, 0x66, 0x00, 0xda, 0x27,
+	0x29, 0xe3, 0x44, 0x82, 0xd3, 0xb7, 0xc3, 0xf3, 0x1f, 0x68, 0x8d, 0x13, 0x3c, 0x24, 0x67, 0x0a,
+	0x91, 0x3a, 0xa1, 0x7b, 0x30, 0x9f, 0x32, 0x2f, 0x61, 0xbb, 0x09, 0x8d, 0x76, 0x49, 0x88, 0x9f,
+	0x73, 0xd0, 0x75, 0x21, 0x52, 0x65, 0xa0, 0x4d, 0x40, 0x24, 0xf6, 0xc3, 0x2c, 0x25, 0x27, 0xf8,
+	0x20, 0xe7, 0x5a, 0x8d, 0x35, 0x63, 0xa3, 0xe3, 0xcc, 0xe0, 0xa0, 0x45, 0x68, 0x86, 0x24, 0x22,
+	0xcc, 0x6a, 0xae, 0x19, 0x1b, 0x7d, 0x47, 0x1e, 0xec, 0x4f, 0x60, 0xa1, 0x84, 0x5f, 0x85, 0x7f,
+	0x17, 0xda, 0x58, 0x92, 0x2c, 0x63, 0xad, 0x3e, 0x2b, 0x01, 0x39, 0xdf, 0xfe, 0xb9, 0x06, 0x4d,
+	0x41, 0x2a, 0xf2, 0x6c, 0x4c, 0xf2, 0x8c, 0x6e, 0x41, 0x8f, 0xa4, 0xee, 0x24, 0x19, 0x35, 0x81,
+	0xaf, 0x4b, 0xd2, 0x22, 0xef, 0xe8, 0x3d, 0x68, 0xf9, 0x47, 0x59, 0x7c, 0x9c, 0x5a, 0x75, 0xe1,
+	0x6a, 0x61, 0xe2, 0x8a, 0x07, 0xbb, 0xc5, 0x79, 0x8e, 0x12, 0x41, 0x0f, 0x01, 0x3c, 0xc6, 0x12,
+	0x72, 0x98, 0x31, 0x9c, 0x8a, 0x68, 0xbb, 0x0f, 0x2c, 0x4d, 0x21, 0x4b, 0xf1, 0x93, 0x82, 0xef,
+	0x68, 0xb2, 0xe8, 0x11, 0x74, 0xf0, 0x19, 0xc3, 0x71, 0x80, 0x03, 0xab, 0x29, 0x1c, 0xad, 0x4c,
+	0xc5, 0xb4, 0xb9, 0xa3, 0xf8, 0x32, 0xc2, 0x42, 0x7c, 0xf0, 0x18, 0xfa, 0x25, 0x16, 0x9a, 0x83,
+	0xfa, 0x31, 0xce, 0x6f, 0x96, 0x7f, 0xf2, 0xec, 0x9e, 0x78, 0x61, 0x26, 0x8b, 0xac, 0xe7, 0xc8,
+	0xc3, 0x47, 0xb5, 0x87, 0x86, 0xbd, 0x0d, 0xe6, 0x6e, 0x16, 0x86, 0x85, 0x62, 0x40, 0x92, 0x5c,
+	0x31, 0x20, 0xc9, 0xa4, 0xd0, 0x6a, 0x17, 0x16, 0xda, 0xaf, 0x06, 0xcc, 0xef, 0x9c, 0xe0, 0x98,
+	0x3d, 0xa7, 0x8c, 0x0c, 0x89, 0xef, 0x31, 0x42, 0x63, 0x74, 0x0f, 0x4c, 0x1a, 0x06, 0xee, 0x85,
+	0x95, 0xda, 0xa1, 0xa1, 0x42, 0x7d, 0x0f, 0xcc, 0x18, 0x9f, 0xba, 0x17, 0xba, 0xeb, 0xc4, 0xf8,
+	0x54, 0x4a, 0xaf, 0x43, 0x3f, 0xc0, 0x21, 0x66, 0xd8, 0x2d, 0x6e, 0x87, 0x5f, 0x5d, 0x4f, 0x12,
+	0xb7, 0xe4, 0x75, 0xdc, 0x81, 0x6b, 0xdc, 0xe4, 0xd8, 0x4b, 0x70, 0xcc, 0xdc, 0xb1, 0xc7, 0x8e,
+	0xc4, 0x9d, 0x98, 0x4e, 0x3f, 0xc6, 0xa7, 0x2f, 0x05, 0xf5, 0xa5, 0xc7, 0x8e, 0xec, 0xbf, 0x0c,
+	0x30, 0x8b, 0xcb, 0x44, 0x37, 0xa0, 0xcd, 0xdd, 0xba, 0x24, 0x50, 0x99, 0x68, 0xf1, 0xe3, 0x5e,
+	0xc0, 0x3b, 0x83, 0x0e, 0x87, 0x29, 0x66, 0x02, 0x5e, 0xdd, 0x51, 0x27, 0x5e, 0x59, 0x29, 0xf9,
+	0x4e, 0x36, 0x43, 0xc3, 0x11, 0xdf, 0x3c, 0xe3, 0x11, 0x23, 0x11, 0x16, 0x0e, 0xeb, 0x8e, 0x3c,
+	0xa0, 0x05, 0x68, 0x62, 0x97, 0x79, 0x23, 0x51, 0xe5, 0xa6, 0xd3, 0xc0, 0xaf, 0xbc, 0x11, 0xfa,
+	0x1f, 0x5c, 0x4d, 0x69, 0x96, 0xf8, 0xd8, 0xcd, 0xdd, 0xb6, 0x04, 0xb7, 0x27, 0xa9, 0xbb, 0xd2,
+	0xb9, 0x0d, 0xf5, 0x21, 0x09, 0xac, 0xb6, 0x48, 0xcc, 0x5c, 0xb9, 0x08, 0xf7, 0x02, 0x87, 0x33,
+	0xd1, 0xfb, 0x00, 0x85, 0xa5, 0xc0, 0xea, 0x9c, 0x23, 0x6a, 0xe6, 0x76, 0x03, 0xfb, 0x0b, 0x68,
+	0x29, 0xf3, 0x4b, 0x60, 0x9e, 0xd0, 0x30, 0x8b, 0x8a, 0xb0, 0xfb, 0x4e, 0x47, 0x12, 0xf6, 0x02,
+	0x74, 0x13, 0xc4, 0xac, 0x73, 0x79, 0x55, 0xd5, 0x44, 0x90, 0x22, 0x43, 0x9f, 0x61, 0x31, 0x2d,
+	0x7c, 0x4a, 0x8f, 0x89, 0x8c, 0xbe, 0xed, 0xa8, 0x93, 0xfd, 0x67, 0x0d, 0xae, 0x96, 0xcb, 0x9d,
+	0xbb, 0x10, 0x56, 0x44, 0xae, 0x0c, 0x61, 0x46, 0x98, 0x3d, 0x28, 0xe5, 0xab, 0xa6, 0xe7, 0x2b,
+	0x57, 0x89, 0x68, 0x20, 0x1d, 0xf4, 0xa5, 0xca, 0x33, 0x1a, 0x60, 0x5e, 0xad, 0x19, 0x09, 0x44,
+	0x82, 0xfb, 0x0e, 0xff, 0xe4, 0x94, 0x11, 0x09, 0xd4, 0x08, 0xe1, 0x9f, 0x02, 0x5e, 0x22, 0xec,
+	0xb6, 0xe4, 0x95, 0xc9, 0x13, 0xbf, 0xb2, 0x88, 0x53, 0xdb, 0xf2, 0x1e, 0xf8, 0x37, 0x5a, 0x83,
+	0x6e, 0x82, 0xc7, 0xa1, 0xaa, 0x5e, 0x91, 0x3e, 0xd3, 0xd1, 0x49, 0x68, 0x15, 0xc0, 0xa7, 0x61,
+	0x88, 0x7d, 0x21, 0x60, 0x0a, 0x01, 0x8d, 0xc2, 0x2b, 0x87, 0xb1, 0xd0, 0x4d, 0xb1, 0x6f, 0xc1,
+	0x9a, 0xb1, 0xd1, 0x74, 0x5a, 0x8c, 0x85, 0x07, 0xd8, 0xe7, 0x71, 0x64, 0x29, 0x4e, 0x5c, 0x31,
+	0x80, 0xba, 0x42, 0xaf, 0xc3, 0x09, 0x62, 0x54, 0xae, 0x00, 0x8c, 0x12, 0x9a, 0x8d, 0x25, 0xb7,
+	0xb7, 0x56, 0xe7, 0xf3, 0x58, 0x50, 0x04, 0xfb, 0x36, 0x5c, 0x4d, 0xdf, 0x44, 0x21, 0x89, 0x8f,
+	0x5d, 0xe6, 0x25, 0x23, 0xcc, 0xac, 0xbe, 0xac, 0x61, 0x45, 0x7d, 0x25, 0x88, 0xf6, 0x97, 0x80,
+	0xb6, 0x12, 0xec, 0x31, 0xfc, 0x0f, 0x56, 0xcf, 0x5b, 0x76, 0xf7, 0x75, 0x58, 0x28, 0x99, 0x96,
+	0x53, 0x98, 0x7b, 0x7c, 0x3d, 0x0e, 0xde, 0x95, 0xc7, 0x92, 0x69, 0xe5, 0xf1, 0x07, 0x03, 0xd0,
+	0xb6, 0x68, 0xf0, 0x7f, 0xb7, 0x5f, 0x79, 0xcb, 0xf1, 0xb9, 0x2f, 0x07, 0x48, 0xe0, 0x31, 0x4f,
+	0x6d, 0xa6, 0x1e, 0x49, 0xa5, 0xfd, 0x6d, 0x8f, 0x79, 0x6a, 0x3b, 0x24, 0xd8, 0xcf, 0x12, 0xbe,
+	0xac, 0x44, 0x5d, 0x89, 0xed, 0xe0, 0xe4, 0x24, 0x0e, 0xb4, 0x04, 0x48, 0x01, 0xfd, 0xc9, 0x00,
+	0xeb, 0x09, 0xa3, 0x11, 0xf1, 0x1d, 0xcc, 0x1d, 0x96, 0xe0, 0xae, 0x43, 0x9f, 0x8f, 0xc5, 0x69,
+	0xc8, 0x3d, 0x1a, 0x06, 0x93, 0xb5, 0x73, 0x13, 0xf8, 0x64, 0x74, 0x35, 0xe4, 0x6d, 0x1a, 0x06,
+	0xa2, 0x20, 0xd6, 0x81, 0x8f, 0x2f, 0x4d, 0x5f, 0x2e, 0xe1, 0x5e, 0x8c, 0x4f, 0x4b, 0xfa, 0x5c,
+	0x48, 0xe8, 0xcb, 0x99, 0xd7, 0x8e, 0xf1, 0x29, 0xd7, 0xb7, 0x97, 0xe0, 0xe6, 0x0c, 0x6c, 0x0a,
+	0xf9, 0x2f, 0x06, 0x2c, 0x3c, 0x49, 0x53, 0x32, 0x8a, 0x3f, 0x17, 0xdd, 0x9f, 0x83, 0x5e, 0x84,
+	0xa6, 0x4f, 0xb3, 0x98, 0x09, 0xb0, 0x4d, 0x47, 0x1e, 0xa6, 0x1a, 0xa2, 0x56, 0x69, 0x88, 0xa9,
+	0x96, 0xaa, 0x57, 0x5b, 0x4a, 0x6b, 0x99, 0x46, 0xa9, 0x65, 0xfe, 0x0b, 0x5d, 0x7e, 0x31, 0xae,
+	0x8f, 0x63, 0x86, 0x13, 0x35, 0x30, 0x81, 0x93, 0xb6, 0x04, 0xc5, 0xfe, 0xde, 0x80, 0xc5, 0x32,
+	0x52, 0xf5, 0x3a, 0x38, 0x77, 0x7e, 0xf3, 0x81, 0x91, 0x84, 0x0a, 0x26, 0xff, 0xe4, 0xad, 0x37,
+	0xce, 0x0e, 0x43, 0xe2, 0xbb, 0x9c, 0x21, 0xe1, 0x99, 0x92, 0xf2, 0x3a, 0x09, 0x27, 0x41, 0x37,
+	0xf4, 0xa0, 0x11, 0x34, 0xbc, 0x8c, 0x1d, 0xe5, 0x33, 0x9c, 0x7f, 0xdb, 0x1f, 0xc2, 0x82, 0x7c,
+	0xb0, 0x95, 0xb3, 0xb6, 0x02, 0x50, 0x4c, 0x55, 0xf9, 0x56, 0x31, 0x1d, 0x33, 0x1f, 0xab, 0xa9,
+	0xfd, 0x31, 0x98, 0xfb, 0x54, 0x26, 0x22, 0x45, 0xf7, 0xc1, 0x0c, 0xf3, 0x83, 0x7a, 0xd6, 0xa0,
+	0x49, 0x7b, 0xe4, 0x72, 0xce, 0x44, 0xc8, 0x7e, 0x0c, 0x9d, 0x9c, 0x9c, 0xc7, 0x66, 0x9c, 0x17,
+	0x5b, 0x6d, 0x2a, 0x36, 0xfb, 0x77, 0x03, 0x16, 0xcb, 0x90, 0x55, 0xfa, 0x5e, 0x43, 0xbf, 0x70,
+	0xe1, 0x46, 0xde, 0x58, 0x61, 0xb9, 0xaf, 0x63, 0xa9, 0xaa, 0x15, 0x00, 0xd3, 0x67, 0xde, 0x58,
+	0x96, 0x54, 0x2f, 0xd4, 0x48, 0x83, 0x57, 0x30, 0x5f, 0x11, 0x99, 0xf1, 0x52, 0xb9, 0xab, 0xbf,
+	0x54, 0x4a, 0xaf, 0xad, 0x42, 0x5b, 0x7f, 0xbe, 0x3c, 0x82, 0x1b, 0xb2, 0xff, 0xb6, 0x8a, 0xa2,
+	0xcb, 0x73, 0x5f, 0xae, 0x4d, 0x63, 0xba, 0x36, 0xed, 0x01, 0x58, 0x55, 0x55, 0xd5, 0x05, 0x23,
+	0x98, 0x3f, 0x60, 0x1e, 0x23, 0x29, 0x23, 0x7e, 0xf1, 0x6c, 0x9e, 0x2a, 0x66, 0xe3, 0xb2, 0xfd,
+	0x50, 0x6d, 0x87, 0x39, 0xa8, 0x33, 0x96, 0xd7, 0x19, 0xff, 0xe4, 0xb7, 0x80, 0x74, 0x4f, 0xea,
+	0x0e, 0xde, 0x81, 0x2b, 0x5e, 0x0f, 0x8c, 0x32, 0x2f, 0x94, 0xfb, 0xb7, 0x21, 0xf6, 0xaf, 0x29,
+	0x28, 0x62, 0x01, 0xcb, 0x15, 0x15, 0x48, 0x6e, 0x53, 0x6e, 0x67, 0x4e, 0x10, 0xcc, 0x15, 0x00,
+	0xd1, 0x52, 0xb2, 0x1b, 0x5a, 0x52, 0x97, 0x53, 0xb6, 0x38, 0xc1, 0x5e, 0x85, 0xe5, 0x4f, 0x31,
+	0xe3, 0x2f, 0x89, 0x64, 0x8b, 0xc6, 0x43, 0x32, 0xca, 0x12, 0x4f, 0xbb, 0x0a, 0xfb, 0x47, 0x03,
+	0x56, 0xce, 0x11, 0x50, 0x01, 0x5b, 0xd0, 0x8e, 0xbc, 0x94, 0xe1, 0x24, 0xef, 0x92, 0xfc, 0x38,
+	0x9d, 0x8a, 0xda, 0x65, 0xa9, 0xa8, 0x57, 0x52, 0x71, 0x1d, 0x5a, 0x91, 0x77, 0xe6, 0x46, 0x87,
+	0xea, 0xa9, 0xd0, 0x8c, 0xbc, 0xb3, 0x67, 0x87, 0x0f, 0xfe, 0x68, 0x43, 0xef, 0x00, 0x7b, 0xa7,
+	0x18, 0x07, 0x02, 0x18, 0x1a, 0xe5, 0x0d, 0x51, 0xfe, 0xd1, 0x85, 0x6e, 0x4f, 0x57, 0xfe, 0xcc,
+	0x5f, 0x79, 0x83, 0x3b, 0x97, 0x89, 0xa9, 0xda, 0xba, 0x82, 0xf6, 0xa1, 0xab, 0xfd, 0xaa, 0x41,
+	0xcb, 0x9a, 0x62, 0xe5, 0xc7, 0xda, 0x60, 0xe5, 0x1c, 0xae, 0x6e, 0x4d, 0xdb, 0xce, 0xba, 0xb5,
+	0xea, 0x7b, 0x40, 0xb7, 0x36, 0x6b, 0xa5, 0x0b, 0x6b, 0xda, 0xe6, 0xd5, 0xad, 0x55, 0x77, 0xbd,
+	0x6e, 0x6d, 0xd6, 0xba, 0x16, 0xd6, 0xb4, 0xf5, 0xa8, 0x5b, 0xab, 0xae, 0x71, 0xdd, 0xda, 0xac,
+	0x9d, 0x7a, 0x05, 0x7d, 0x0d, 0xf3, 0x95, 0xc5, 0x85, 0xec, 0x89, 0xd6, 0x79, 0x1b, 0x77, 0xb0,
+	0x7e, 0xa1, 0x4c, 0x61, 0xff, 0x05, 0xf4, 0xf4, 0x85, 0x82, 0x34, 0x40, 0x33, 0x56, 0xe2, 0x60,
+	0xf5, 0x3c, 0xb6, 0x6e, 0x50, 0x9f, 0x95, 0xba, 0xc1, 0x19, 0xdb, 0x42, 0x37, 0x38, 0x6b, 0xc4,
+	0xda, 0x57, 0xd0, 0x57, 0x30, 0x37, 0x3d, 0xb3, 0xd0, 0xad, 0xe9, 0xb4, 0x55, 0x46, 0xe1, 0xc0,
+	0xbe, 0x48, 0xa4, 0x30, 0xbe, 0x07, 0x30, 0x19, 0x45, 0x68, 0x69, 0xa2, 0x53, 0x19, 0x85, 0x83,
+	0xe5, 0xd9, 0xcc, 0xc2, 0xd4, 0x37, 0x70, 0x7d, 0x66, 0xbf, 0x23, 0xad, 0x49, 0x2e, 0x9a, 0x18,
+	0x83, 0xff, 0x5f, 0x2a, 0x97, 0xfb, 0x7a, 0xba, 0x0a, 0x73, 0xa9, 0x6c, 0xe3, 0x61, 0xba, 0xe9,
+	0x87, 0x04, 0xc7, 0xec, 0x29, 0x08, 0x8d, 0x97, 0x09, 0x65, 0xf4, 0xb0, 0x25, 0xfe, 0xad, 0xf9,
+	0xe0, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2d, 0xec, 0xb0, 0x56, 0xbc, 0x11, 0x00, 0x00,
 }

weed/pb/filer_pb/filer_pb_helper.go

@@ -0,0 +1,69 @@
+package filer_pb
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+)
+
+func toFileIdObject(fileIdStr string) (*FileId, error) {
+	t, err := needle.ParseFileIdFromString(fileIdStr)
+	if err != nil {
+		return nil, err
+	}
+	return &FileId{
+		VolumeId: uint32(t.VolumeId),
+		Cookie:   uint32(t.Cookie),
+		FileKey:  uint64(t.Key),
+	}, nil
+
+}
+
+func (fid *FileId) toFileIdString() string {
+	return needle.NewFileId(needle.VolumeId(fid.VolumeId), fid.FileKey, fid.Cookie).String()
+}
+
+func (c *FileChunk) GetFileIdString() string {
+	if c.FileId != "" {
+		return c.FileId
+	}
+	if c.Fid != nil {
+		c.FileId = c.Fid.toFileIdString()
+		return c.FileId
+	}
+	return ""
+}
+
+func BeforeEntrySerialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.FileId != "" {
+			if fid, err := toFileIdObject(chunk.FileId); err == nil {
+				chunk.Fid = fid
+				chunk.FileId = ""
+			}
+		}
+
+		if chunk.SourceFileId != "" {
+			if fid, err := toFileIdObject(chunk.SourceFileId); err == nil {
+				chunk.SourceFid = fid
+				chunk.SourceFileId = ""
+			}
+		}
+
+	}
+}
+
+func AfterEntryDeserialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.Fid != nil && chunk.FileId == "" {
+			chunk.FileId = chunk.Fid.toFileIdString()
+		}
+
+		if chunk.SourceFid != nil && chunk.SourceFileId == "" {
+			chunk.SourceFileId = chunk.SourceFid.toFileIdString()
+		}
+
+	}
+}

weed/pb/filer_pb/filer_pb_helper_test.go

@@ -0,0 +1,17 @@
+package filer_pb
+
+import (
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+)
+
+func TestFileIdSize(t *testing.T) {
+	fileIdStr := "11745,0293434534cbb9892b"
+
+	fid, _ := toFileIdObject(fileIdStr)
+	bytes, _ := proto.Marshal(fid)
+
+	println(len(fileIdStr))
+	println(len(bytes))
+}

weed/pb/master.proto

@@ -21,6 +21,10 @@ service Seaweed {
     }
     rpc VolumeList (VolumeListRequest) returns (VolumeListResponse) {
     }
+    rpc LookupEcVolume (LookupEcVolumeRequest) returns (LookupEcVolumeResponse) {
+    }
+    rpc GetMasterConfiguration (GetMasterConfigurationRequest) returns (GetMasterConfigurationResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -35,14 +39,25 @@ message Heartbeat {
     string rack = 7;
     uint32 admin_port = 8;
     repeated VolumeInformationMessage volumes = 9;
-    // delta volume ids
-    repeated uint32 new_vids = 10;
-    repeated uint32 deleted_vids = 11;
+    // delta volumes
+    repeated VolumeShortInformationMessage new_volumes = 10;
+    repeated VolumeShortInformationMessage deleted_volumes = 11;
+    bool has_no_volumes = 12;
+
+    // erasure coding
+    repeated VolumeEcShardInformationMessage ec_shards = 16;
+    // delta erasure coding shards
+    repeated VolumeEcShardInformationMessage new_ec_shards = 17;
+    repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
+    bool has_no_ec_shards = 19;
+
 }
 
 message HeartbeatResponse {
-    uint64 volumeSizeLimit = 1;
-    string leader = 3;
+    uint64 volume_size_limit = 1;
+    string leader = 2;
+    string metrics_address = 3;
+    uint32 metrics_interval_seconds = 4;
 }
 
 message VolumeInformationMessage {
@@ -57,6 +72,21 @@ message VolumeInformationMessage {
     uint32 version = 9;
     uint32 ttl = 10;
     uint32 compact_revision = 11;
+    int64 modified_at_second = 12;
+}
+
+message VolumeShortInformationMessage {
+    uint32 id = 1;
+    string collection = 3;
+    uint32 replica_placement = 8;
+    uint32 version = 9;
+    uint32 ttl = 10;
+}
+
+message VolumeEcShardInformationMessage {
+    uint32 id = 1;
+    string collection = 2;
+    uint32 ec_index_bits = 3;
 }
 
 message Empty {
@@ -144,6 +174,8 @@ message Collection {
     string name = 1;
 }
 message CollectionListRequest {
+    bool include_normal_volumes = 1;
+    bool include_ec_volumes = 2;
 }
 message CollectionListResponse {
     repeated Collection collections = 1;
@@ -165,6 +197,7 @@ message DataNodeInfo {
     uint64 free_volume_count = 4;
     uint64 active_volume_count = 5;
     repeated VolumeInformationMessage volume_infos = 6;
+    repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
 }
 message RackInfo {
     string id = 1;
@@ -194,4 +227,24 @@ message VolumeListRequest {
 }
 message VolumeListResponse {
     TopologyInfo topology_info = 1;
+    uint64 volume_size_limit_mb = 2;
+}
+
+message LookupEcVolumeRequest {
+    uint32 volume_id = 1;
+}
+message LookupEcVolumeResponse {
+    uint32 volume_id = 1;
+    message EcShardIdLocation {
+        uint32 shard_id = 1;
+        repeated Location locations = 2;
+    }
+    repeated EcShardIdLocation shard_id_locations = 2;
+}
+
+message GetMasterConfigurationRequest {
+}
+message GetMasterConfigurationResponse {
+    string metrics_address = 1;
+    uint32 metrics_interval_seconds = 2;
 }

weed/pb/master_pb/master.pb.go

@@ -12,6 +12,8 @@ It has these top-level messages:
 	Heartbeat
 	HeartbeatResponse
 	VolumeInformationMessage
+	VolumeShortInformationMessage
+	VolumeEcShardInformationMessage
 	Empty
 	SuperBlockExtra
 	ClientListenRequest
@@ -35,6 +37,10 @@ It has these top-level messages:
 	TopologyInfo
 	VolumeListRequest
 	VolumeListResponse
+	LookupEcVolumeRequest
+	LookupEcVolumeResponse
+	GetMasterConfigurationRequest
+	GetMasterConfigurationResponse
 */
 package master_pb
 
@@ -68,9 +74,16 @@ type Heartbeat struct {
 	Rack           string                      `protobuf:"bytes,7,opt,name=rack" json:"rack,omitempty"`
 	AdminPort      uint32                      `protobuf:"varint,8,opt,name=admin_port,json=adminPort" json:"admin_port,omitempty"`
 	Volumes        []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"`
-	// delta volume ids
-	NewVids     []uint32 `protobuf:"varint,10,rep,packed,name=new_vids,json=newVids" json:"new_vids,omitempty"`
-	DeletedVids []uint32 `protobuf:"varint,11,rep,packed,name=deleted_vids,json=deletedVids" json:"deleted_vids,omitempty"`
+	// delta volumes
+	NewVolumes     []*VolumeShortInformationMessage `protobuf:"bytes,10,rep,name=new_volumes,json=newVolumes" json:"new_volumes,omitempty"`
+	DeletedVolumes []*VolumeShortInformationMessage `protobuf:"bytes,11,rep,name=deleted_volumes,json=deletedVolumes" json:"deleted_volumes,omitempty"`
+	HasNoVolumes   bool                             `protobuf:"varint,12,opt,name=has_no_volumes,json=hasNoVolumes" json:"has_no_volumes,omitempty"`
+	// erasure coding
+	EcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,16,rep,name=ec_shards,json=ecShards" json:"ec_shards,omitempty"`
+	// delta erasure coding shards
+	NewEcShards     []*VolumeEcShardInformationMessage `protobuf:"bytes,17,rep,name=new_ec_shards,json=newEcShards" json:"new_ec_shards,omitempty"`
+	DeletedEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,18,rep,name=deleted_ec_shards,json=deletedEcShards" json:"deleted_ec_shards,omitempty"`
+	HasNoEcShards   bool                               `protobuf:"varint,19,opt,name=has_no_ec_shards,json=hasNoEcShards" json:"has_no_ec_shards,omitempty"`
 }
 
 func (m *Heartbeat) Reset()                    { *m = Heartbeat{} }
@@ -141,23 +154,60 @@ func (m *Heartbeat) GetVolumes() []*VolumeInformationMessage {
 	return nil
 }
 
-func (m *Heartbeat) GetNewVids() []uint32 {
+func (m *Heartbeat) GetNewVolumes() []*VolumeShortInformationMessage {
 	if m != nil {
-		return m.NewVids
+		return m.NewVolumes
 	}
 	return nil
 }
 
-func (m *Heartbeat) GetDeletedVids() []uint32 {
+func (m *Heartbeat) GetDeletedVolumes() []*VolumeShortInformationMessage {
 	if m != nil {
-		return m.DeletedVids
+		return m.DeletedVolumes
 	}
 	return nil
 }
 
+func (m *Heartbeat) GetHasNoVolumes() bool {
+	if m != nil {
+		return m.HasNoVolumes
+	}
+	return false
+}
+
+func (m *Heartbeat) GetEcShards() []*VolumeEcShardInformationMessage {
+	if m != nil {
+		return m.EcShards
+	}
+	return nil
+}
+
+func (m *Heartbeat) GetNewEcShards() []*VolumeEcShardInformationMessage {
+	if m != nil {
+		return m.NewEcShards
+	}
+	return nil
+}
+
+func (m *Heartbeat) GetDeletedEcShards() []*VolumeEcShardInformationMessage {
+	if m != nil {
+		return m.DeletedEcShards
+	}
+	return nil
+}
+
+func (m *Heartbeat) GetHasNoEcShards() bool {
+	if m != nil {
+		return m.HasNoEcShards
+	}
+	return false
+}
+
 type HeartbeatResponse struct {
-	VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"`
-	Leader          string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"`
+	VolumeSizeLimit        uint64 `protobuf:"varint,1,opt,name=volume_size_limit,json=volumeSizeLimit" json:"volume_size_limit,omitempty"`
+	Leader                 string `protobuf:"bytes,2,opt,name=leader" json:"leader,omitempty"`
+	MetricsAddress         string `protobuf:"bytes,3,opt,name=metrics_address,json=metricsAddress" json:"metrics_address,omitempty"`
+	MetricsIntervalSeconds uint32 `protobuf:"varint,4,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds" json:"metrics_interval_seconds,omitempty"`
 }
 
 func (m *HeartbeatResponse) Reset()                    { *m = HeartbeatResponse{} }
@@ -179,6 +229,20 @@ func (m *HeartbeatResponse) GetLeader() string {
 	return ""
 }
 
+func (m *HeartbeatResponse) GetMetricsAddress() string {
+	if m != nil {
+		return m.MetricsAddress
+	}
+	return ""
+}
+
+func (m *HeartbeatResponse) GetMetricsIntervalSeconds() uint32 {
+	if m != nil {
+		return m.MetricsIntervalSeconds
+	}
+	return 0
+}
+
 type VolumeInformationMessage struct {
 	Id               uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
 	Size             uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
@@ -191,6 +255,7 @@ type VolumeInformationMessage struct {
 	Version          uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"`
 	Ttl              uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
 	CompactRevision  uint32 `protobuf:"varint,11,opt,name=compact_revision,json=compactRevision" json:"compact_revision,omitempty"`
+	ModifiedAtSecond int64  `protobuf:"varint,12,opt,name=modified_at_second,json=modifiedAtSecond" json:"modified_at_second,omitempty"`
 }
 
 func (m *VolumeInformationMessage) Reset()                    { *m = VolumeInformationMessage{} }
@@ -275,13 +340,100 @@ func (m *VolumeInformationMessage) GetCompactRevision() uint32 {
 	return 0
 }
 
+func (m *VolumeInformationMessage) GetModifiedAtSecond() int64 {
+	if m != nil {
+		return m.ModifiedAtSecond
+	}
+	return 0
+}
+
+type VolumeShortInformationMessage struct {
+	Id               uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
+	Collection       string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
+	ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"`
+	Version          uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"`
+	Ttl              uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
+}
+
+func (m *VolumeShortInformationMessage) Reset()                    { *m = VolumeShortInformationMessage{} }
+func (m *VolumeShortInformationMessage) String() string            { return proto.CompactTextString(m) }
+func (*VolumeShortInformationMessage) ProtoMessage()               {}
+func (*VolumeShortInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+func (m *VolumeShortInformationMessage) GetId() uint32 {
+	if m != nil {
+		return m.Id
+	}
+	return 0
+}
+
+func (m *VolumeShortInformationMessage) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+func (m *VolumeShortInformationMessage) GetReplicaPlacement() uint32 {
+	if m != nil {
+		return m.ReplicaPlacement
+	}
+	return 0
+}
+
+func (m *VolumeShortInformationMessage) GetVersion() uint32 {
+	if m != nil {
+		return m.Version
+	}
+	return 0
+}
+
+func (m *VolumeShortInformationMessage) GetTtl() uint32 {
+	if m != nil {
+		return m.Ttl
+	}
+	return 0
+}
+
+type VolumeEcShardInformationMessage struct {
+	Id          uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
+	Collection  string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
+	EcIndexBits uint32 `protobuf:"varint,3,opt,name=ec_index_bits,json=ecIndexBits" json:"ec_index_bits,omitempty"`
+}
+
+func (m *VolumeEcShardInformationMessage) Reset()                    { *m = VolumeEcShardInformationMessage{} }
+func (m *VolumeEcShardInformationMessage) String() string            { return proto.CompactTextString(m) }
+func (*VolumeEcShardInformationMessage) ProtoMessage()               {}
+func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *VolumeEcShardInformationMessage) GetId() uint32 {
+	if m != nil {
+		return m.Id
+	}
+	return 0
+}
+
+func (m *VolumeEcShardInformationMessage) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+func (m *VolumeEcShardInformationMessage) GetEcIndexBits() uint32 {
+	if m != nil {
+		return m.EcIndexBits
+	}
+	return 0
+}
+
 type Empty struct {
 }
 
 func (m *Empty) Reset()                    { *m = Empty{} }
 func (m *Empty) String() string            { return proto.CompactTextString(m) }
 func (*Empty) ProtoMessage()               {}
-func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
 
 type SuperBlockExtra struct {
 	ErasureCoding *SuperBlockExtra_ErasureCoding `protobuf:"bytes,1,opt,name=erasure_coding,json=erasureCoding" json:"erasure_coding,omitempty"`
@@ -290,7 +442,7 @@ type SuperBlockExtra struct {
 func (m *SuperBlockExtra) Reset()                    { *m = SuperBlockExtra{} }
 func (m *SuperBlockExtra) String() string            { return proto.CompactTextString(m) }
 func (*SuperBlockExtra) ProtoMessage()               {}
-func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
 
 func (m *SuperBlockExtra) GetErasureCoding() *SuperBlockExtra_ErasureCoding {
 	if m != nil {
@@ -309,7 +461,7 @@ func (m *SuperBlockExtra_ErasureCoding) Reset()         { *m = SuperBlockExtra_E
 func (m *SuperBlockExtra_ErasureCoding) String() string { return proto.CompactTextString(m) }
 func (*SuperBlockExtra_ErasureCoding) ProtoMessage()    {}
 func (*SuperBlockExtra_ErasureCoding) Descriptor() ([]byte, []int) {
-	return fileDescriptor0, []int{4, 0}
+	return fileDescriptor0, []int{6, 0}
 }
 
 func (m *SuperBlockExtra_ErasureCoding) GetData() uint32 {
@@ -340,7 +492,7 @@ type ClientListenRequest struct {
 func (m *ClientListenRequest) Reset()                    { *m = ClientListenRequest{} }
 func (m *ClientListenRequest) String() string            { return proto.CompactTextString(m) }
 func (*ClientListenRequest) ProtoMessage()               {}
-func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
 
 func (m *ClientListenRequest) GetName() string {
 	if m != nil {
@@ -359,7 +511,7 @@ type VolumeLocation struct {
 func (m *VolumeLocation) Reset()                    { *m = VolumeLocation{} }
 func (m *VolumeLocation) String() string            { return proto.CompactTextString(m) }
 func (*VolumeLocation) ProtoMessage()               {}
-func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
 
 func (m *VolumeLocation) GetUrl() string {
 	if m != nil {
@@ -397,7 +549,7 @@ type LookupVolumeRequest struct {
 func (m *LookupVolumeRequest) Reset()                    { *m = LookupVolumeRequest{} }
 func (m *LookupVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeRequest) ProtoMessage()               {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
 
 func (m *LookupVolumeRequest) GetVolumeIds() []string {
 	if m != nil {
@@ -420,7 +572,7 @@ type LookupVolumeResponse struct {
 func (m *LookupVolumeResponse) Reset()                    { *m = LookupVolumeResponse{} }
 func (m *LookupVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse) ProtoMessage()               {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
 
 func (m *LookupVolumeResponse) GetVolumeIdLocations() []*LookupVolumeResponse_VolumeIdLocation {
 	if m != nil {
@@ -439,7 +591,7 @@ func (m *LookupVolumeResponse_VolumeIdLocation) Reset()         { *m = LookupVol
 func (m *LookupVolumeResponse_VolumeIdLocation) String() string { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage()    {}
 func (*LookupVolumeResponse_VolumeIdLocation) Descriptor() ([]byte, []int) {
-	return fileDescriptor0, []int{8, 0}
+	return fileDescriptor0, []int{10, 0}
 }
 
 func (m *LookupVolumeResponse_VolumeIdLocation) GetVolumeId() string {
@@ -471,7 +623,7 @@ type Location struct {
 func (m *Location) Reset()                    { *m = Location{} }
 func (m *Location) String() string            { return proto.CompactTextString(m) }
 func (*Location) ProtoMessage()               {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
 
 func (m *Location) GetUrl() string {
 	if m != nil {
@@ -500,7 +652,7 @@ type AssignRequest struct {
 func (m *AssignRequest) Reset()                    { *m = AssignRequest{} }
 func (m *AssignRequest) String() string            { return proto.CompactTextString(m) }
 func (*AssignRequest) ProtoMessage()               {}
-func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
 
 func (m *AssignRequest) GetCount() uint64 {
 	if m != nil {
@@ -563,7 +715,7 @@ type AssignResponse struct {
 func (m *AssignResponse) Reset()                    { *m = AssignResponse{} }
 func (m *AssignResponse) String() string            { return proto.CompactTextString(m) }
 func (*AssignResponse) ProtoMessage()               {}
-func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
 
 func (m *AssignResponse) GetFid() string {
 	if m != nil {
@@ -616,7 +768,7 @@ type StatisticsRequest struct {
 func (m *StatisticsRequest) Reset()                    { *m = StatisticsRequest{} }
 func (m *StatisticsRequest) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsRequest) ProtoMessage()               {}
-func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
 
 func (m *StatisticsRequest) GetReplication() string {
 	if m != nil {
@@ -651,7 +803,7 @@ type StatisticsResponse struct {
 func (m *StatisticsResponse) Reset()                    { *m = StatisticsResponse{} }
 func (m *StatisticsResponse) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsResponse) ProtoMessage()               {}
-func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
 
 func (m *StatisticsResponse) GetReplication() string {
 	if m != nil {
@@ -703,7 +855,7 @@ type StorageType struct {
 func (m *StorageType) Reset()                    { *m = StorageType{} }
 func (m *StorageType) String() string            { return proto.CompactTextString(m) }
 func (*StorageType) ProtoMessage()               {}
-func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
 
 func (m *StorageType) GetReplication() string {
 	if m != nil {
@@ -726,7 +878,7 @@ type Collection struct {
 func (m *Collection) Reset()                    { *m = Collection{} }
 func (m *Collection) String() string            { return proto.CompactTextString(m) }
 func (*Collection) ProtoMessage()               {}
-func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
 
 func (m *Collection) GetName() string {
 	if m != nil {
@@ -736,12 +888,28 @@ func (m *Collection) GetName() string {
 }
 
 type CollectionListRequest struct {
+	IncludeNormalVolumes bool `protobuf:"varint,1,opt,name=include_normal_volumes,json=includeNormalVolumes" json:"include_normal_volumes,omitempty"`
+	IncludeEcVolumes     bool `protobuf:"varint,2,opt,name=include_ec_volumes,json=includeEcVolumes" json:"include_ec_volumes,omitempty"`
 }
 
 func (m *CollectionListRequest) Reset()                    { *m = CollectionListRequest{} }
 func (m *CollectionListRequest) String() string            { return proto.CompactTextString(m) }
 func (*CollectionListRequest) ProtoMessage()               {}
-func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+
+func (m *CollectionListRequest) GetIncludeNormalVolumes() bool {
+	if m != nil {
+		return m.IncludeNormalVolumes
+	}
+	return false
+}
+
+func (m *CollectionListRequest) GetIncludeEcVolumes() bool {
+	if m != nil {
+		return m.IncludeEcVolumes
+	}
+	return false
+}
 
 type CollectionListResponse struct {
 	Collections []*Collection `protobuf:"bytes,1,rep,name=collections" json:"collections,omitempty"`
@@ -750,7 +918,7 @@ type CollectionListResponse struct {
 func (m *CollectionListResponse) Reset()                    { *m = CollectionListResponse{} }
 func (m *CollectionListResponse) String() string            { return proto.CompactTextString(m) }
 func (*CollectionListResponse) ProtoMessage()               {}
-func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
 
 func (m *CollectionListResponse) GetCollections() []*Collection {
 	if m != nil {
@@ -766,7 +934,7 @@ type CollectionDeleteRequest struct {
 func (m *CollectionDeleteRequest) Reset()                    { *m = CollectionDeleteRequest{} }
 func (m *CollectionDeleteRequest) String() string            { return proto.CompactTextString(m) }
 func (*CollectionDeleteRequest) ProtoMessage()               {}
-func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
 
 func (m *CollectionDeleteRequest) GetName() string {
 	if m != nil {
@@ -781,24 +949,25 @@ type CollectionDeleteResponse struct {
 func (m *CollectionDeleteResponse) Reset()                    { *m = CollectionDeleteResponse{} }
 func (m *CollectionDeleteResponse) String() string            { return proto.CompactTextString(m) }
 func (*CollectionDeleteResponse) ProtoMessage()               {}
-func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
 
 //
 // volume related
 //
 type DataNodeInfo struct {
-	Id                string                      `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
-	VolumeCount       uint64                      `protobuf:"varint,2,opt,name=volume_count,json=volumeCount" json:"volume_count,omitempty"`
-	MaxVolumeCount    uint64                      `protobuf:"varint,3,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
-	FreeVolumeCount   uint64                      `protobuf:"varint,4,opt,name=free_volume_count,json=freeVolumeCount" json:"free_volume_count,omitempty"`
-	ActiveVolumeCount uint64                      `protobuf:"varint,5,opt,name=active_volume_count,json=activeVolumeCount" json:"active_volume_count,omitempty"`
-	VolumeInfos       []*VolumeInformationMessage `protobuf:"bytes,6,rep,name=volume_infos,json=volumeInfos" json:"volume_infos,omitempty"`
+	Id                string                             `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
+	VolumeCount       uint64                             `protobuf:"varint,2,opt,name=volume_count,json=volumeCount" json:"volume_count,omitempty"`
+	MaxVolumeCount    uint64                             `protobuf:"varint,3,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
+	FreeVolumeCount   uint64                             `protobuf:"varint,4,opt,name=free_volume_count,json=freeVolumeCount" json:"free_volume_count,omitempty"`
+	ActiveVolumeCount uint64                             `protobuf:"varint,5,opt,name=active_volume_count,json=activeVolumeCount" json:"active_volume_count,omitempty"`
+	VolumeInfos       []*VolumeInformationMessage        `protobuf:"bytes,6,rep,name=volume_infos,json=volumeInfos" json:"volume_infos,omitempty"`
+	EcShardInfos      []*VolumeEcShardInformationMessage `protobuf:"bytes,7,rep,name=ec_shard_infos,json=ecShardInfos" json:"ec_shard_infos,omitempty"`
 }
 
 func (m *DataNodeInfo) Reset()                    { *m = DataNodeInfo{} }
 func (m *DataNodeInfo) String() string            { return proto.CompactTextString(m) }
 func (*DataNodeInfo) ProtoMessage()               {}
-func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
 
 func (m *DataNodeInfo) GetId() string {
 	if m != nil {
@@ -842,6 +1011,13 @@ func (m *DataNodeInfo) GetVolumeInfos() []*VolumeInformationMessage {
 	return nil
 }
 
+func (m *DataNodeInfo) GetEcShardInfos() []*VolumeEcShardInformationMessage {
+	if m != nil {
+		return m.EcShardInfos
+	}
+	return nil
+}
+
 type RackInfo struct {
 	Id                string          `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"`
 	VolumeCount       uint64          `protobuf:"varint,2,opt,name=volume_count,json=volumeCount" json:"volume_count,omitempty"`
@@ -854,7 +1030,7 @@ type RackInfo struct {
 func (m *RackInfo) Reset()                    { *m = RackInfo{} }
 func (m *RackInfo) String() string            { return proto.CompactTextString(m) }
 func (*RackInfo) ProtoMessage()               {}
-func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
 
 func (m *RackInfo) GetId() string {
 	if m != nil {
@@ -910,7 +1086,7 @@ type DataCenterInfo struct {
 func (m *DataCenterInfo) Reset()                    { *m = DataCenterInfo{} }
 func (m *DataCenterInfo) String() string            { return proto.CompactTextString(m) }
 func (*DataCenterInfo) ProtoMessage()               {}
-func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
 
 func (m *DataCenterInfo) GetId() string {
 	if m != nil {
@@ -966,7 +1142,7 @@ type TopologyInfo struct {
 func (m *TopologyInfo) Reset()                    { *m = TopologyInfo{} }
 func (m *TopologyInfo) String() string            { return proto.CompactTextString(m) }
 func (*TopologyInfo) ProtoMessage()               {}
-func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
 func (m *TopologyInfo) GetId() string {
 	if m != nil {
@@ -1016,16 +1192,17 @@ type VolumeListRequest struct {
 func (m *VolumeListRequest) Reset()                    { *m = VolumeListRequest{} }
 func (m *VolumeListRequest) String() string            { return proto.CompactTextString(m) }
 func (*VolumeListRequest) ProtoMessage()               {}
-func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
 
 type VolumeListResponse struct {
-	TopologyInfo *TopologyInfo `protobuf:"bytes,1,opt,name=topology_info,json=topologyInfo" json:"topology_info,omitempty"`
+	TopologyInfo      *TopologyInfo `protobuf:"bytes,1,opt,name=topology_info,json=topologyInfo" json:"topology_info,omitempty"`
+	VolumeSizeLimitMb uint64        `protobuf:"varint,2,opt,name=volume_size_limit_mb,json=volumeSizeLimitMb" json:"volume_size_limit_mb,omitempty"`
 }
 
 func (m *VolumeListResponse) Reset()                    { *m = VolumeListResponse{} }
 func (m *VolumeListResponse) String() string            { return proto.CompactTextString(m) }
 func (*VolumeListResponse) ProtoMessage()               {}
-func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
 
 func (m *VolumeListResponse) GetTopologyInfo() *TopologyInfo {
 	if m != nil {
@@ -1034,10 +1211,119 @@ func (m *VolumeListResponse) GetTopologyInfo() *TopologyInfo {
 	return nil
 }
 
+func (m *VolumeListResponse) GetVolumeSizeLimitMb() uint64 {
+	if m != nil {
+		return m.VolumeSizeLimitMb
+	}
+	return 0
+}
+
+type LookupEcVolumeRequest struct {
+	VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+}
+
+func (m *LookupEcVolumeRequest) Reset()                    { *m = LookupEcVolumeRequest{} }
+func (m *LookupEcVolumeRequest) String() string            { return proto.CompactTextString(m) }
+func (*LookupEcVolumeRequest) ProtoMessage()               {}
+func (*LookupEcVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+
+func (m *LookupEcVolumeRequest) GetVolumeId() uint32 {
+	if m != nil {
+		return m.VolumeId
+	}
+	return 0
+}
+
+type LookupEcVolumeResponse struct {
+	VolumeId         uint32                                      `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+	ShardIdLocations []*LookupEcVolumeResponse_EcShardIdLocation `protobuf:"bytes,2,rep,name=shard_id_locations,json=shardIdLocations" json:"shard_id_locations,omitempty"`
+}
+
+func (m *LookupEcVolumeResponse) Reset()                    { *m = LookupEcVolumeResponse{} }
+func (m *LookupEcVolumeResponse) String() string            { return proto.CompactTextString(m) }
+func (*LookupEcVolumeResponse) ProtoMessage()               {}
+func (*LookupEcVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+
+func (m *LookupEcVolumeResponse) GetVolumeId() uint32 {
+	if m != nil {
+		return m.VolumeId
+	}
+	return 0
+}
+
+func (m *LookupEcVolumeResponse) GetShardIdLocations() []*LookupEcVolumeResponse_EcShardIdLocation {
+	if m != nil {
+		return m.ShardIdLocations
+	}
+	return nil
+}
+
+type LookupEcVolumeResponse_EcShardIdLocation struct {
+	ShardId   uint32      `protobuf:"varint,1,opt,name=shard_id,json=shardId" json:"shard_id,omitempty"`
+	Locations []*Location `protobuf:"bytes,2,rep,name=locations" json:"locations,omitempty"`
+}
+
+func (m *LookupEcVolumeResponse_EcShardIdLocation) Reset() {
+	*m = LookupEcVolumeResponse_EcShardIdLocation{}
+}
+func (m *LookupEcVolumeResponse_EcShardIdLocation) String() string { return proto.CompactTextString(m) }
+func (*LookupEcVolumeResponse_EcShardIdLocation) ProtoMessage()    {}
+func (*LookupEcVolumeResponse_EcShardIdLocation) Descriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{29, 0}
+}
+
+func (m *LookupEcVolumeResponse_EcShardIdLocation) GetShardId() uint32 {
+	if m != nil {
+		return m.ShardId
+	}
+	return 0
+}
+
+func (m *LookupEcVolumeResponse_EcShardIdLocation) GetLocations() []*Location {
+	if m != nil {
+		return m.Locations
+	}
+	return nil
+}
+
+type GetMasterConfigurationRequest struct {
+}
+
+func (m *GetMasterConfigurationRequest) Reset()                    { *m = GetMasterConfigurationRequest{} }
+func (m *GetMasterConfigurationRequest) String() string            { return proto.CompactTextString(m) }
+func (*GetMasterConfigurationRequest) ProtoMessage()               {}
+func (*GetMasterConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
+
+type GetMasterConfigurationResponse struct {
+	MetricsAddress         string `protobuf:"bytes,1,opt,name=metrics_address,json=metricsAddress" json:"metrics_address,omitempty"`
+	MetricsIntervalSeconds uint32 `protobuf:"varint,2,opt,name=metrics_interval_seconds,json=metricsIntervalSeconds" json:"metrics_interval_seconds,omitempty"`
+}
+
+func (m *GetMasterConfigurationResponse) Reset()                    { *m = GetMasterConfigurationResponse{} }
+func (m *GetMasterConfigurationResponse) String() string            { return proto.CompactTextString(m) }
+func (*GetMasterConfigurationResponse) ProtoMessage()               {}
+func (*GetMasterConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
+
+func (m *GetMasterConfigurationResponse) GetMetricsAddress() string {
+	if m != nil {
+		return m.MetricsAddress
+	}
+	return ""
+}
+
+func (m *GetMasterConfigurationResponse) GetMetricsIntervalSeconds() uint32 {
+	if m != nil {
+		return m.MetricsIntervalSeconds
+	}
+	return 0
+}
+
 func init() {
 	proto.RegisterType((*Heartbeat)(nil), "master_pb.Heartbeat")
 	proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse")
 	proto.RegisterType((*VolumeInformationMessage)(nil), "master_pb.VolumeInformationMessage")
+	proto.RegisterType((*VolumeShortInformationMessage)(nil), "master_pb.VolumeShortInformationMessage")
+	proto.RegisterType((*VolumeEcShardInformationMessage)(nil), "master_pb.VolumeEcShardInformationMessage")
 	proto.RegisterType((*Empty)(nil), "master_pb.Empty")
 	proto.RegisterType((*SuperBlockExtra)(nil), "master_pb.SuperBlockExtra")
 	proto.RegisterType((*SuperBlockExtra_ErasureCoding)(nil), "master_pb.SuperBlockExtra.ErasureCoding")
@@ -1063,6 +1349,11 @@ func init() {
 	proto.RegisterType((*TopologyInfo)(nil), "master_pb.TopologyInfo")
 	proto.RegisterType((*VolumeListRequest)(nil), "master_pb.VolumeListRequest")
 	proto.RegisterType((*VolumeListResponse)(nil), "master_pb.VolumeListResponse")
+	proto.RegisterType((*LookupEcVolumeRequest)(nil), "master_pb.LookupEcVolumeRequest")
+	proto.RegisterType((*LookupEcVolumeResponse)(nil), "master_pb.LookupEcVolumeResponse")
+	proto.RegisterType((*LookupEcVolumeResponse_EcShardIdLocation)(nil), "master_pb.LookupEcVolumeResponse.EcShardIdLocation")
+	proto.RegisterType((*GetMasterConfigurationRequest)(nil), "master_pb.GetMasterConfigurationRequest")
+	proto.RegisterType((*GetMasterConfigurationResponse)(nil), "master_pb.GetMasterConfigurationResponse")
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -1084,6 +1375,8 @@ type SeaweedClient interface {
 	CollectionList(ctx context.Context, in *CollectionListRequest, opts ...grpc.CallOption) (*CollectionListResponse, error)
 	CollectionDelete(ctx context.Context, in *CollectionDeleteRequest, opts ...grpc.CallOption) (*CollectionDeleteResponse, error)
 	VolumeList(ctx context.Context, in *VolumeListRequest, opts ...grpc.CallOption) (*VolumeListResponse, error)
+	LookupEcVolume(ctx context.Context, in *LookupEcVolumeRequest, opts ...grpc.CallOption) (*LookupEcVolumeResponse, error)
+	GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error)
 }
 
 type seaweedClient struct {
@@ -1210,6 +1503,24 @@ func (c *seaweedClient) VolumeList(ctx context.Context, in *VolumeListRequest, o
 	return out, nil
 }
 
+func (c *seaweedClient) LookupEcVolume(ctx context.Context, in *LookupEcVolumeRequest, opts ...grpc.CallOption) (*LookupEcVolumeResponse, error) {
+	out := new(LookupEcVolumeResponse)
+	err := grpc.Invoke(ctx, "/master_pb.Seaweed/LookupEcVolume", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedClient) GetMasterConfiguration(ctx context.Context, in *GetMasterConfigurationRequest, opts ...grpc.CallOption) (*GetMasterConfigurationResponse, error) {
+	out := new(GetMasterConfigurationResponse)
+	err := grpc.Invoke(ctx, "/master_pb.Seaweed/GetMasterConfiguration", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for Seaweed service
 
 type SeaweedServer interface {
@@ -1221,6 +1532,8 @@ type SeaweedServer interface {
 	CollectionList(context.Context, *CollectionListRequest) (*CollectionListResponse, error)
 	CollectionDelete(context.Context, *CollectionDeleteRequest) (*CollectionDeleteResponse, error)
 	VolumeList(context.Context, *VolumeListRequest) (*VolumeListResponse, error)
+	LookupEcVolume(context.Context, *LookupEcVolumeRequest) (*LookupEcVolumeResponse, error)
+	GetMasterConfiguration(context.Context, *GetMasterConfigurationRequest) (*GetMasterConfigurationResponse, error)
 }
 
 func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) {
@@ -1387,6 +1700,42 @@ func _Seaweed_VolumeList_Handler(srv interface{}, ctx context.Context, dec func(
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Seaweed_LookupEcVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(LookupEcVolumeRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedServer).LookupEcVolume(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/master_pb.Seaweed/LookupEcVolume",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedServer).LookupEcVolume(ctx, req.(*LookupEcVolumeRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Seaweed_GetMasterConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetMasterConfigurationRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedServer).GetMasterConfiguration(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/master_pb.Seaweed/GetMasterConfiguration",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedServer).GetMasterConfiguration(ctx, req.(*GetMasterConfigurationRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _Seaweed_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "master_pb.Seaweed",
 	HandlerType: (*SeaweedServer)(nil),
@@ -1415,6 +1764,14 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 			MethodName: "VolumeList",
 			Handler:    _Seaweed_VolumeList_Handler,
 		},
+		{
+			MethodName: "LookupEcVolume",
+			Handler:    _Seaweed_LookupEcVolume_Handler,
+		},
+		{
+			MethodName: "GetMasterConfiguration",
+			Handler:    _Seaweed_GetMasterConfiguration_Handler,
+		},
 	},
 	Streams: []grpc.StreamDesc{
 		{
@@ -1436,94 +1793,123 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("master.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1416 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6f, 0xdc, 0x44,
-	0x14, 0x8f, 0xbd, 0x1f, 0x59, 0xbf, 0xfd, 0xc8, 0xee, 0x24, 0x6d, 0xdd, 0x2d, 0x6d, 0xb7, 0xee,
-	0x65, 0xcb, 0x47, 0x54, 0xc2, 0x81, 0x03, 0xa0, 0xaa, 0x4d, 0x53, 0x51, 0x35, 0xd0, 0xd6, 0x69,
-	0x8b, 0x84, 0x84, 0xcc, 0xc4, 0x7e, 0x09, 0x56, 0xbc, 0xb6, 0xb1, 0x67, 0xb7, 0xd9, 0x5e, 0xb8,
-	0x70, 0x44, 0x70, 0xe0, 0xff, 0xe1, 0xc2, 0x91, 0x3f, 0x85, 0x03, 0x12, 0x77, 0x24, 0x34, 0xe3,
-	0xb1, 0x77, 0xec, 0xdd, 0xa4, 0x15, 0x12, 0x87, 0xde, 0x3c, 0xbf, 0xf7, 0x66, 0xe6, 0xcd, 0xef,
-	0x7d, 0xee, 0x42, 0x67, 0x42, 0x53, 0x86, 0xc9, 0x76, 0x9c, 0x44, 0x2c, 0x22, 0x46, 0xb6, 0x72,
-	0xe2, 0x43, 0xeb, 0x4f, 0x1d, 0x8c, 0xcf, 0x91, 0x26, 0xec, 0x10, 0x29, 0x23, 0x3d, 0xd0, 0xfd,
-	0xd8, 0xd4, 0x46, 0xda, 0xd8, 0xb0, 0x75, 0x3f, 0x26, 0x04, 0xea, 0x71, 0x94, 0x30, 0x53, 0x1f,
-	0x69, 0xe3, 0xae, 0x2d, 0xbe, 0xc9, 0x55, 0x80, 0x78, 0x7a, 0x18, 0xf8, 0xae, 0x33, 0x4d, 0x02,
-	0xb3, 0x26, 0x74, 0x8d, 0x0c, 0x79, 0x9e, 0x04, 0x64, 0x0c, 0xfd, 0x09, 0x3d, 0x75, 0x66, 0x51,
-	0x30, 0x9d, 0xa0, 0xe3, 0x46, 0xd3, 0x90, 0x99, 0x75, 0xb1, 0xbd, 0x37, 0xa1, 0xa7, 0x2f, 0x04,
-	0xbc, 0xcb, 0x51, 0x32, 0xe2, 0x56, 0x9d, 0x3a, 0x47, 0x7e, 0x80, 0xce, 0x09, 0xce, 0xcd, 0xc6,
-	0x48, 0x1b, 0xd7, 0x6d, 0x98, 0xd0, 0xd3, 0x07, 0x7e, 0x80, 0x8f, 0x70, 0x4e, 0xae, 0x43, 0xdb,
-	0xa3, 0x8c, 0x3a, 0x2e, 0x86, 0x0c, 0x13, 0xb3, 0x29, 0xee, 0x02, 0x0e, 0xed, 0x0a, 0x84, 0xdb,
-	0x97, 0x50, 0xf7, 0xc4, 0x5c, 0x17, 0x12, 0xf1, 0xcd, 0xed, 0xa3, 0xde, 0xc4, 0x0f, 0x1d, 0x61,
-	0x79, 0x4b, 0x5c, 0x6d, 0x08, 0xe4, 0x09, 0x37, 0xff, 0x33, 0x58, 0xcf, 0x6c, 0x4b, 0x4d, 0x63,
-	0x54, 0x1b, 0xb7, 0x77, 0x6e, 0x6e, 0x17, 0x6c, 0x6c, 0x67, 0xe6, 0x3d, 0x0c, 0x8f, 0xa2, 0x64,
-	0x42, 0x99, 0x1f, 0x85, 0x5f, 0x60, 0x9a, 0xd2, 0x63, 0xb4, 0xf3, 0x3d, 0xe4, 0x32, 0xb4, 0x42,
-	0x7c, 0xe9, 0xcc, 0x7c, 0x2f, 0x35, 0x61, 0x54, 0x1b, 0x77, 0xed, 0xf5, 0x10, 0x5f, 0xbe, 0xf0,
-	0xbd, 0x94, 0xdc, 0x80, 0x8e, 0x87, 0x01, 0x32, 0xf4, 0x32, 0x71, 0x5b, 0x88, 0xdb, 0x12, 0xe3,
-	0x2a, 0xd6, 0x73, 0x18, 0x14, 0x64, 0xdb, 0x98, 0xc6, 0x51, 0x98, 0x22, 0x19, 0xc3, 0x46, 0x76,
-	0xfa, 0x81, 0xff, 0x0a, 0xf7, 0xfd, 0x89, 0xcf, 0x84, 0x07, 0xea, 0x76, 0x15, 0x26, 0x17, 0xa1,
-	0x19, 0x20, 0xf5, 0x30, 0x91, 0xb4, 0xcb, 0x95, 0xf5, 0x97, 0x0e, 0xe6, 0x59, 0xa6, 0x0b, 0x9f,
-	0x7a, 0xe2, 0xc4, 0xae, 0xad, 0xfb, 0x1e, 0xe7, 0x2c, 0xf5, 0x5f, 0xa1, 0xf0, 0x69, 0xdd, 0x16,
-	0xdf, 0xe4, 0x1a, 0x80, 0x1b, 0x05, 0x01, 0xba, 0x7c, 0xa3, 0x3c, 0x5c, 0x41, 0x38, 0xa7, 0xc2,
-	0x4d, 0x0b, 0x77, 0xd6, 0x6d, 0x83, 0x23, 0x99, 0x27, 0x8b, 0x97, 0x4b, 0x85, 0xcc, 0x93, 0xf2,
-	0xe5, 0x99, 0xca, 0xfb, 0x40, 0x72, 0x72, 0x0e, 0xe7, 0x85, 0x62, 0x53, 0x28, 0xf6, 0xa5, 0xe4,
-	0xde, 0x3c, 0xd7, 0xbe, 0x02, 0x46, 0x82, 0xd4, 0x73, 0xa2, 0x30, 0x98, 0x0b, 0xe7, 0xb6, 0xec,
-	0x16, 0x07, 0x1e, 0x87, 0xc1, 0x9c, 0xbc, 0x07, 0x83, 0x04, 0xe3, 0xc0, 0x77, 0xa9, 0x13, 0x07,
-	0xd4, 0xc5, 0x09, 0x86, 0xb9, 0x9f, 0xfb, 0x52, 0xf0, 0x24, 0xc7, 0x89, 0x09, 0xeb, 0x33, 0x4c,
-	0x52, 0xfe, 0x2c, 0x43, 0xa8, 0xe4, 0x4b, 0xd2, 0x87, 0x1a, 0x63, 0x81, 0x09, 0x02, 0xe5, 0x9f,
-	0xe4, 0x16, 0xf4, 0xdd, 0x68, 0x12, 0x53, 0x97, 0x39, 0x09, 0xce, 0x7c, 0xb1, 0xa9, 0x2d, 0xc4,
-	0x1b, 0x12, 0xb7, 0x25, 0x6c, 0xad, 0x43, 0x63, 0x6f, 0x12, 0xb3, 0xb9, 0xf5, 0x9b, 0x06, 0x1b,
-	0x07, 0xd3, 0x18, 0x93, 0x7b, 0x41, 0xe4, 0x9e, 0xec, 0x9d, 0xb2, 0x84, 0x92, 0xc7, 0xd0, 0xc3,
-	0x84, 0xa6, 0xd3, 0x84, 0x3f, 0xd3, 0xf3, 0xc3, 0x63, 0xc1, 0x7e, 0x7b, 0x67, 0xac, 0x44, 0x5a,
-	0x65, 0xcf, 0xf6, 0x5e, 0xb6, 0x61, 0x57, 0xe8, 0xdb, 0x5d, 0x54, 0x97, 0xc3, 0xaf, 0xa1, 0x5b,
-	0x92, 0x73, 0x1f, 0xf2, 0x2c, 0x90, 0x5e, 0x15, 0xdf, 0x3c, 0x38, 0x62, 0x9a, 0xf8, 0x6c, 0x2e,
-	0xb3, 0x55, 0xae, 0xb8, 0xef, 0x64, 0x32, 0xf2, 0xa0, 0xac, 0x89, 0xa0, 0x34, 0x32, 0xe4, 0xa1,
-	0x97, 0x5a, 0xb7, 0x60, 0x73, 0x37, 0xf0, 0x31, 0x64, 0xfb, 0x7e, 0xca, 0x30, 0xb4, 0xf1, 0xfb,
-	0x29, 0xa6, 0x8c, 0xdf, 0x10, 0xd2, 0x09, 0xca, 0x5a, 0x20, 0xbe, 0xad, 0x1f, 0xa0, 0x97, 0x45,
-	0xd9, 0x7e, 0xe4, 0x8a, 0x10, 0xe3, 0x1c, 0xf2, 0x22, 0x90, 0x29, 0xf1, 0xcf, 0x4a, 0x75, 0xd0,
-	0xab, 0xd5, 0x41, 0x4d, 0x9f, 0xda, 0xf9, 0xe9, 0x53, 0x5f, 0x4e, 0x9f, 0x67, 0xb0, 0xb9, 0x1f,
-	0x45, 0x27, 0xd3, 0x38, 0x33, 0x23, 0xb7, 0xb5, 0xfc, 0x42, 0x6d, 0x54, 0xe3, 0x77, 0x16, 0x2f,
-	0xac, 0x04, 0xb7, 0x5e, 0x0d, 0x6e, 0xeb, 0x6f, 0x0d, 0xb6, 0xca, 0xc7, 0xca, 0xc4, 0xfc, 0x16,
-	0x36, 0x8b, 0x73, 0x9d, 0x40, 0xbe, 0x39, 0xbb, 0xa0, 0xbd, 0x73, 0x5b, 0x71, 0xe6, 0xaa, 0xdd,
-	0x79, 0x2d, 0xf1, 0x72, 0xb2, 0xec, 0xc1, 0xac, 0x82, 0xa4, 0xc3, 0x53, 0xe8, 0x57, 0xd5, 0x78,
-	0xec, 0x17, 0xb7, 0x4a, 0x66, 0x5b, 0xf9, 0x4e, 0xf2, 0x21, 0x18, 0x0b, 0x43, 0x74, 0x61, 0xc8,
-	0x66, 0xc9, 0x10, 0x79, 0xd7, 0x42, 0x8b, 0x6c, 0x41, 0x03, 0x93, 0x24, 0xca, 0x6b, 0x46, 0xb6,
-	0xb0, 0x3e, 0x81, 0xd6, 0x7f, 0xf6, 0xa2, 0xf5, 0x87, 0x06, 0xdd, 0xbb, 0x69, 0xea, 0x1f, 0x17,
-	0xe1, 0xb2, 0x05, 0x8d, 0x2c, 0xa3, 0xb3, 0xca, 0x95, 0x2d, 0xc8, 0x08, 0xda, 0x32, 0x21, 0x15,
-	0xea, 0x55, 0xe8, 0xb5, 0x85, 0x47, 0x26, 0x69, 0x3d, 0x33, 0x8d, 0x27, 0x69, 0xa5, 0x27, 0x34,
-	0xce, 0xec, 0x09, 0x4d, 0xa5, 0x27, 0x5c, 0x01, 0x43, 0x6c, 0x0a, 0x23, 0x0f, 0x65, 0xb3, 0x68,
-	0x71, 0xe0, 0xcb, 0xc8, 0x43, 0xeb, 0x57, 0x0d, 0x7a, 0xf9, 0x6b, 0xa4, 0xe7, 0xfb, 0x50, 0x3b,
-	0x2a, 0xd8, 0xe7, 0x9f, 0x39, 0x47, 0xfa, 0x59, 0x1c, 0x2d, 0xf5, 0xc1, 0x82, 0x91, 0xba, 0xca,
-	0x48, 0xe1, 0x8c, 0x86, 0xe2, 0x0c, 0x6e, 0x32, 0x9d, 0xb2, 0xef, 0x72, 0x93, 0xf9, 0xb7, 0x75,
-	0x0c, 0x83, 0x03, 0x46, 0x99, 0x9f, 0x32, 0xdf, 0x4d, 0x73, 0x9a, 0x2b, 0x84, 0x6a, 0xaf, 0x23,
-	0x54, 0x3f, 0x8b, 0xd0, 0x5a, 0x41, 0xa8, 0xf5, 0xbb, 0x06, 0x44, 0xbd, 0x49, 0x52, 0xf0, 0x3f,
-	0x5c, 0xc5, 0x29, 0x63, 0x11, 0xa3, 0x81, 0x23, 0x1a, 0x90, 0x6c, 0x23, 0x02, 0xe1, 0x3d, 0x8e,
-	0x7b, 0x69, 0x9a, 0xa2, 0x97, 0x49, 0xb3, 0x1e, 0xd2, 0xe2, 0x80, 0x10, 0x96, 0x5b, 0x50, 0xb3,
-	0xd2, 0x82, 0xac, 0xbb, 0xd0, 0x3e, 0x60, 0x51, 0x42, 0x8f, 0xf1, 0xd9, 0x3c, 0x7e, 0x13, 0xeb,
-	0xa5, 0x75, 0xfa, 0x82, 0x88, 0x11, 0xc0, 0xee, 0xc2, 0xfa, 0x55, 0x05, 0xf0, 0x12, 0x5c, 0x58,
-	0x68, 0xf0, 0x7a, 0x29, 0xfd, 0x62, 0x3d, 0x85, 0x8b, 0x55, 0x81, 0xa4, 0xf1, 0x63, 0x68, 0x2f,
-	0x28, 0xc9, 0x6b, 0xc7, 0x05, 0x25, 0x65, 0x17, 0xfb, 0x6c, 0x55, 0xd3, 0xfa, 0x00, 0x2e, 0x2d,
-	0x44, 0xf7, 0x45, 0x11, 0x3c, 0xaf, 0x36, 0x0f, 0xc1, 0x5c, 0x56, 0xcf, 0x6c, 0xb0, 0x7e, 0xd1,
-	0xa1, 0x73, 0x5f, 0x46, 0x3b, 0x1f, 0x10, 0x94, 0x91, 0xc0, 0x10, 0x23, 0xc1, 0x0d, 0xe8, 0x94,
-	0xe6, 0xb5, 0x6c, 0x34, 0x68, 0xcf, 0x94, 0x61, 0x6d, 0xd5, 0x58, 0x57, 0x13, 0x6a, 0xd5, 0xb1,
-	0xee, 0x5d, 0x18, 0x1c, 0x25, 0x88, 0xcb, 0x13, 0x60, 0xdd, 0xde, 0xe0, 0x02, 0x55, 0x77, 0x1b,
-	0x36, 0xa9, 0xcb, 0xfc, 0x59, 0x45, 0x3b, 0xf3, 0xfd, 0x20, 0x13, 0xa9, 0xfa, 0x0f, 0x0a, 0x43,
-	0xfd, 0xf0, 0x28, 0x4a, 0xcd, 0xe6, 0x9b, 0x4f, 0x70, 0xf2, 0x35, 0x5c, 0x92, 0x5a, 0x3f, 0xea,
-	0xd0, 0xb2, 0xa9, 0x7b, 0xf2, 0x76, 0xb3, 0x71, 0x07, 0x36, 0x8a, 0xaa, 0x56, 0x22, 0xe4, 0x92,
-	0x42, 0x88, 0xea, 0x78, 0xbb, 0xeb, 0x29, 0xab, 0xd4, 0xfa, 0x47, 0x83, 0xde, 0xfd, 0xa2, 0x72,
-	0xbe, 0xdd, 0x64, 0xec, 0x00, 0xf0, 0x52, 0x5f, 0xe2, 0x41, 0x6d, 0x8d, 0xb9, 0xbb, 0x6d, 0x23,
-	0x91, 0x5f, 0xa9, 0xf5, 0xb3, 0x0e, 0x9d, 0x67, 0x51, 0x1c, 0x05, 0xd1, 0xf1, 0xfc, 0xed, 0x7e,
-	0xfd, 0x1e, 0x0c, 0x94, 0xae, 0x58, 0x22, 0xe1, 0x72, 0x25, 0x18, 0x16, 0xce, 0xb6, 0x37, 0xbc,
-	0xd2, 0x3a, 0xb5, 0x36, 0x61, 0x20, 0x27, 0x3c, 0xa5, 0xb8, 0xd9, 0x40, 0x54, 0x50, 0x16, 0xb6,
-	0x4f, 0xa1, 0xcb, 0x24, 0x75, 0xe2, 0x3a, 0x39, 0xe3, 0xaa, 0xa1, 0xa7, 0x52, 0x6b, 0x77, 0x98,
-	0xb2, 0xda, 0xf9, 0xa9, 0x01, 0xeb, 0x07, 0x48, 0x5f, 0x22, 0x7a, 0xe4, 0x21, 0x74, 0x0f, 0x30,
-	0xf4, 0x16, 0xbf, 0x42, 0xb7, 0x94, 0x33, 0x0a, 0x74, 0xf8, 0xce, 0x2a, 0xb4, 0xa8, 0x71, 0x6b,
-	0x63, 0xed, 0xb6, 0x46, 0x9e, 0x40, 0xf7, 0x11, 0x62, 0xbc, 0x1b, 0x85, 0x21, 0xba, 0x0c, 0x3d,
-	0x72, 0x4d, 0xad, 0xb4, 0xcb, 0x63, 0xee, 0xf0, 0xf2, 0x52, 0xe9, 0xc8, 0xa7, 0x22, 0x79, 0xe2,
-	0x53, 0xe8, 0xa8, 0xd3, 0x5d, 0xe9, 0xc0, 0x15, 0xb3, 0xe8, 0xf0, 0xfa, 0x6b, 0xc6, 0x42, 0x6b,
-	0x8d, 0xdc, 0x81, 0x66, 0x36, 0x6e, 0x10, 0x53, 0x51, 0x2e, 0xcd, 0x53, 0x25, 0xbb, 0xca, 0xb3,
-	0x89, 0xb5, 0x46, 0x1e, 0x01, 0x2c, 0x1a, 0x36, 0x51, 0x79, 0x59, 0x9a, 0x18, 0x86, 0x57, 0xcf,
-	0x90, 0x16, 0x87, 0x7d, 0x05, 0xbd, 0x72, 0xeb, 0x22, 0xa3, 0x95, 0xdd, 0x49, 0x89, 0x88, 0xe1,
-	0x8d, 0x73, 0x34, 0x8a, 0x83, 0xbf, 0x81, 0x7e, 0xb5, 0x23, 0x11, 0x6b, 0xe5, 0xc6, 0x52, 0x77,
-	0x1b, 0xde, 0x3c, 0x57, 0x47, 0x25, 0x61, 0x11, 0x95, 0x25, 0x12, 0x96, 0x22, 0xb8, 0x44, 0xc2,
-	0x72, 0x28, 0x5b, 0x6b, 0x87, 0x4d, 0xf1, 0xbf, 0xc8, 0x47, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff,
-	0x88, 0xac, 0x76, 0x9c, 0x27, 0x11, 0x00, 0x00,
+	// 1888 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6f, 0x1c, 0x49,
+	0x15, 0x4f, 0xcf, 0x8c, 0xc7, 0x33, 0x6f, 0x3e, 0x3c, 0x53, 0x76, 0xbc, 0x93, 0x59, 0x92, 0x4c,
+	0x7a, 0x91, 0xd6, 0x09, 0x8b, 0x59, 0xb2, 0x2b, 0x81, 0x04, 0x68, 0x95, 0x38, 0xde, 0xc5, 0xca,
+	0xc7, 0x3a, 0x3d, 0x21, 0x48, 0x48, 0xa8, 0xa9, 0xe9, 0x2e, 0xdb, 0x25, 0xf7, 0x74, 0x37, 0x5d,
+	0x35, 0x8e, 0x67, 0x39, 0x70, 0x80, 0x1b, 0x12, 0x17, 0xce, 0xdc, 0xf9, 0x1b, 0x38, 0x70, 0xe1,
+	0xc8, 0x9d, 0x3b, 0xff, 0x02, 0x57, 0x84, 0xb4, 0xaa, 0xaf, 0xee, 0xea, 0x99, 0xb1, 0x1d, 0xaf,
+	0xb4, 0x87, 0xdc, 0xaa, 0xdf, 0x7b, 0xf5, 0xde, 0xab, 0xdf, 0xab, 0xf7, 0x51, 0x0d, 0xed, 0x29,
+	0x66, 0x9c, 0x64, 0xbb, 0x69, 0x96, 0xf0, 0x04, 0x35, 0xd5, 0x97, 0x9f, 0x4e, 0xdc, 0x3f, 0xd5,
+	0xa1, 0xf9, 0x73, 0x82, 0x33, 0x3e, 0x21, 0x98, 0xa3, 0x2e, 0x54, 0x68, 0x3a, 0x70, 0x46, 0xce,
+	0x4e, 0xd3, 0xab, 0xd0, 0x14, 0x21, 0xa8, 0xa5, 0x49, 0xc6, 0x07, 0x95, 0x91, 0xb3, 0xd3, 0xf1,
+	0xe4, 0x1a, 0xdd, 0x06, 0x48, 0x67, 0x93, 0x88, 0x06, 0xfe, 0x2c, 0x8b, 0x06, 0x55, 0x29, 0xdb,
+	0x54, 0x94, 0x5f, 0x64, 0x11, 0xda, 0x81, 0xde, 0x14, 0x9f, 0xfb, 0x67, 0x49, 0x34, 0x9b, 0x12,
+	0x3f, 0x48, 0x66, 0x31, 0x1f, 0xd4, 0xe4, 0xf6, 0xee, 0x14, 0x9f, 0xbf, 0x96, 0xe4, 0x3d, 0x41,
+	0x45, 0x23, 0xe1, 0xd5, 0xb9, 0x7f, 0x44, 0x23, 0xe2, 0x9f, 0x92, 0xf9, 0x60, 0x6d, 0xe4, 0xec,
+	0xd4, 0x3c, 0x98, 0xe2, 0xf3, 0xcf, 0x69, 0x44, 0x9e, 0x92, 0x39, 0xba, 0x0b, 0xad, 0x10, 0x73,
+	0xec, 0x07, 0x24, 0xe6, 0x24, 0x1b, 0xd4, 0xa5, 0x2d, 0x10, 0xa4, 0x3d, 0x49, 0x11, 0xfe, 0x65,
+	0x38, 0x38, 0x1d, 0xac, 0x4b, 0x8e, 0x5c, 0x0b, 0xff, 0x70, 0x38, 0xa5, 0xb1, 0x2f, 0x3d, 0x6f,
+	0x48, 0xd3, 0x4d, 0x49, 0x39, 0x14, 0xee, 0xff, 0x0c, 0xd6, 0x95, 0x6f, 0x6c, 0xd0, 0x1c, 0x55,
+	0x77, 0x5a, 0x0f, 0x3f, 0xd8, 0xcd, 0xd1, 0xd8, 0x55, 0xee, 0x1d, 0xc4, 0x47, 0x49, 0x36, 0xc5,
+	0x9c, 0x26, 0xf1, 0x73, 0xc2, 0x18, 0x3e, 0x26, 0x9e, 0xd9, 0x83, 0x0e, 0xa0, 0x15, 0x93, 0x37,
+	0xbe, 0x51, 0x01, 0x52, 0xc5, 0xce, 0x92, 0x8a, 0xf1, 0x49, 0x92, 0xf1, 0x15, 0x7a, 0x20, 0x26,
+	0x6f, 0x5e, 0x6b, 0x55, 0x2f, 0x61, 0x23, 0x24, 0x11, 0xe1, 0x24, 0xcc, 0xd5, 0xb5, 0xae, 0xa9,
+	0xae, 0xab, 0x15, 0x18, 0x95, 0xdf, 0x85, 0xee, 0x09, 0x66, 0x7e, 0x9c, 0xe4, 0x1a, 0xdb, 0x23,
+	0x67, 0xa7, 0xe1, 0xb5, 0x4f, 0x30, 0x7b, 0x91, 0x18, 0xa9, 0x2f, 0xa0, 0x49, 0x02, 0x9f, 0x9d,
+	0xe0, 0x2c, 0x64, 0x83, 0x9e, 0x34, 0xf9, 0x60, 0xc9, 0xe4, 0x7e, 0x30, 0x16, 0x02, 0x2b, 0x8c,
+	0x36, 0x88, 0x62, 0x31, 0xf4, 0x02, 0x3a, 0x02, 0x8c, 0x42, 0x59, 0xff, 0xda, 0xca, 0x04, 0x9a,
+	0xfb, 0x46, 0xdf, 0x6b, 0xe8, 0x1b, 0x44, 0x0a, 0x9d, 0xe8, 0xda, 0x3a, 0x0d, 0xac, 0xb9, 0xde,
+	0x0f, 0xa1, 0xa7, 0x61, 0x29, 0xd4, 0x6e, 0x4a, 0x60, 0x3a, 0x12, 0x18, 0x23, 0xe8, 0xfe, 0xdd,
+	0x81, 0x7e, 0x9e, 0x0d, 0x1e, 0x61, 0x69, 0x12, 0x33, 0x82, 0x1e, 0x40, 0x5f, 0x5f, 0x67, 0x46,
+	0xbf, 0x22, 0x7e, 0x44, 0xa7, 0x94, 0xcb, 0x24, 0xa9, 0x79, 0x1b, 0x8a, 0x31, 0xa6, 0x5f, 0x91,
+	0x67, 0x82, 0x8c, 0xb6, 0xa1, 0x1e, 0x11, 0x1c, 0x92, 0x4c, 0xe6, 0x4c, 0xd3, 0xd3, 0x5f, 0xe8,
+	0x43, 0xd8, 0x98, 0x12, 0x9e, 0xd1, 0x80, 0xf9, 0x38, 0x0c, 0x33, 0xc2, 0x98, 0x4e, 0x9d, 0xae,
+	0x26, 0x3f, 0x52, 0x54, 0xf4, 0x63, 0x18, 0x18, 0x41, 0x2a, 0xee, 0xf8, 0x19, 0x8e, 0x7c, 0x46,
+	0x82, 0x24, 0x0e, 0x99, 0xce, 0xa3, 0x6d, 0xcd, 0x3f, 0xd0, 0xec, 0xb1, 0xe2, 0xba, 0x7f, 0xad,
+	0xc2, 0xe0, 0xa2, 0x0b, 0x2c, 0x33, 0x3b, 0x94, 0x4e, 0x77, 0xbc, 0x0a, 0x0d, 0x45, 0xe6, 0x88,
+	0xc3, 0x48, 0x2f, 0x6b, 0x9e, 0x5c, 0xa3, 0x3b, 0x00, 0x41, 0x12, 0x45, 0x24, 0x10, 0x1b, 0xb5,
+	0x7b, 0x16, 0x45, 0x64, 0x96, 0x4c, 0xd6, 0x22, 0xa9, 0x6b, 0x5e, 0x53, 0x50, 0x54, 0x3e, 0xdf,
+	0x83, 0xb6, 0x02, 0x5e, 0x0b, 0xa8, 0x7c, 0x6e, 0x29, 0x9a, 0x12, 0xf9, 0x08, 0x90, 0x09, 0xf0,
+	0x64, 0x9e, 0x0b, 0xd6, 0xa5, 0x60, 0x4f, 0x73, 0x1e, 0xcf, 0x8d, 0xf4, 0xfb, 0xd0, 0xcc, 0x08,
+	0x0e, 0xfd, 0x24, 0x8e, 0xe6, 0x32, 0xc5, 0x1b, 0x5e, 0x43, 0x10, 0xbe, 0x8c, 0xa3, 0x39, 0xfa,
+	0x1e, 0xf4, 0x33, 0x92, 0x46, 0x34, 0xc0, 0x7e, 0x1a, 0xe1, 0x80, 0x4c, 0x49, 0x6c, 0xb2, 0xbd,
+	0xa7, 0x19, 0x87, 0x86, 0x8e, 0x06, 0xb0, 0x7e, 0x46, 0x32, 0x26, 0x8e, 0xd5, 0x94, 0x22, 0xe6,
+	0x13, 0xf5, 0xa0, 0xca, 0x79, 0x34, 0x00, 0x49, 0x15, 0x4b, 0x74, 0x1f, 0x7a, 0x41, 0x32, 0x4d,
+	0x71, 0xc0, 0xfd, 0x8c, 0x9c, 0x51, 0xb9, 0xa9, 0x25, 0xd9, 0x1b, 0x9a, 0xee, 0x69, 0xb2, 0x38,
+	0xce, 0x34, 0x09, 0xe9, 0x11, 0x25, 0xa1, 0x8f, 0xb9, 0x0e, 0x93, 0x4c, 0xb9, 0xaa, 0xd7, 0x33,
+	0x9c, 0x47, 0x5c, 0x05, 0xc8, 0xfd, 0x9b, 0x03, 0xb7, 0x2f, 0x4d, 0xe7, 0xa5, 0x20, 0x5d, 0x15,
+	0x90, 0x6f, 0x0b, 0x03, 0x77, 0x06, 0x77, 0xaf, 0x48, 0xb2, 0x2b, 0x7c, 0xad, 0x2c, 0xf9, 0xea,
+	0x42, 0x87, 0x04, 0x3e, 0x8d, 0x43, 0x72, 0xee, 0x4f, 0x28, 0x57, 0xd7, 0xbf, 0xe3, 0xb5, 0x48,
+	0x70, 0x20, 0x68, 0x8f, 0x29, 0x67, 0xee, 0x3a, 0xac, 0xed, 0x4f, 0x53, 0x3e, 0x77, 0xff, 0xe1,
+	0xc0, 0xc6, 0x78, 0x96, 0x92, 0xec, 0x71, 0x94, 0x04, 0xa7, 0xfb, 0xe7, 0x3c, 0xc3, 0xe8, 0x4b,
+	0xe8, 0x92, 0x0c, 0xb3, 0x59, 0x26, 0xae, 0x4d, 0x48, 0xe3, 0x63, 0x69, 0xbc, 0x5c, 0x2d, 0x17,
+	0xf6, 0xec, 0xee, 0xab, 0x0d, 0x7b, 0x52, 0xde, 0xeb, 0x10, 0xfb, 0x73, 0xf8, 0x2b, 0xe8, 0x94,
+	0xf8, 0x22, 0x27, 0x44, 0x6f, 0xd1, 0x87, 0x92, 0x6b, 0x91, 0xcf, 0x29, 0xce, 0x28, 0x9f, 0xeb,
+	0x1e, 0xa8, 0xbf, 0x44, 0x2e, 0xe8, 0x9a, 0x40, 0x43, 0x71, 0x96, 0xaa, 0xe8, 0x32, 0x8a, 0x72,
+	0x10, 0x32, 0xf7, 0x3e, 0x6c, 0xee, 0x45, 0x94, 0xc4, 0xfc, 0x19, 0x65, 0x9c, 0xc4, 0x1e, 0xf9,
+	0xed, 0x8c, 0x30, 0x2e, 0x2c, 0xc4, 0x78, 0x4a, 0x74, 0x87, 0x95, 0x6b, 0xf7, 0xf7, 0xd0, 0x55,
+	0x58, 0x3f, 0x4b, 0x02, 0x89, 0xb0, 0x88, 0x87, 0x68, 0xad, 0x4a, 0x48, 0x2c, 0x17, 0x7a, 0x6e,
+	0x65, 0xb1, 0xe7, 0xde, 0x82, 0x86, 0x6c, 0x4a, 0x85, 0x2b, 0xeb, 0xa2, 0xcf, 0xd0, 0x90, 0x15,
+	0x49, 0x19, 0x2a, 0x76, 0x4d, 0xb2, 0x5b, 0xa6, 0x6f, 0xd0, 0x90, 0xb9, 0xaf, 0x60, 0xf3, 0x59,
+	0x92, 0x9c, 0xce, 0x52, 0xe5, 0x86, 0xf1, 0xb5, 0x7c, 0x42, 0x67, 0x54, 0x15, 0x36, 0xf3, 0x13,
+	0x5e, 0x15, 0x6f, 0xf7, 0xbf, 0x0e, 0x6c, 0x95, 0xd5, 0xea, 0x6a, 0xfa, 0x1b, 0xd8, 0xcc, 0xf5,
+	0xfa, 0x91, 0x3e, 0xb3, 0x32, 0xd0, 0x7a, 0xf8, 0xb1, 0x15, 0xcc, 0x55, 0xbb, 0x4d, 0x87, 0x0e,
+	0x0d, 0x58, 0x5e, 0xff, 0x6c, 0x81, 0xc2, 0x86, 0xe7, 0xd0, 0x5b, 0x14, 0x13, 0xb5, 0x24, 0xb7,
+	0xaa, 0x91, 0x6d, 0x98, 0x9d, 0xe8, 0x87, 0xd0, 0x2c, 0x1c, 0xa9, 0x48, 0x47, 0x36, 0x4b, 0x8e,
+	0x68, 0x5b, 0x85, 0x14, 0xda, 0x82, 0x35, 0x92, 0x65, 0x49, 0xa6, 0xb3, 0x52, 0x7d, 0xb8, 0x3f,
+	0x81, 0xc6, 0x37, 0x8e, 0xa2, 0xfb, 0x2f, 0x07, 0x3a, 0x8f, 0x18, 0xa3, 0xc7, 0xf9, 0x75, 0xd9,
+	0x82, 0x35, 0x55, 0x21, 0x55, 0xb3, 0x51, 0x1f, 0x68, 0x04, 0x2d, 0x9d, 0xdc, 0x16, 0xf4, 0x36,
+	0xe9, 0xca, 0xba, 0xa1, 0x13, 0xbe, 0xa6, 0x5c, 0x13, 0x45, 0x6f, 0x61, 0xd2, 0x5a, 0xbb, 0x70,
+	0xd2, 0xaa, 0x5b, 0x93, 0xd6, 0xfb, 0xd0, 0x94, 0x9b, 0xe2, 0x24, 0x24, 0x7a, 0x04, 0x6b, 0x08,
+	0xc2, 0x8b, 0x24, 0x24, 0xee, 0x5f, 0x1c, 0xe8, 0x9a, 0xd3, 0xe8, 0xc8, 0xf7, 0xa0, 0x7a, 0x94,
+	0xa3, 0x2f, 0x96, 0x06, 0xa3, 0xca, 0x45, 0x18, 0x2d, 0x4d, 0x97, 0x39, 0x22, 0x35, 0x1b, 0x91,
+	0x3c, 0x18, 0x6b, 0x56, 0x30, 0x84, 0xcb, 0x78, 0xc6, 0x4f, 0x8c, 0xcb, 0x62, 0xed, 0x1e, 0x43,
+	0x7f, 0xcc, 0x31, 0xa7, 0x8c, 0xd3, 0x80, 0x19, 0x98, 0x17, 0x00, 0x75, 0xae, 0x02, 0xb4, 0x72,
+	0x11, 0xa0, 0xd5, 0x1c, 0x50, 0xf7, 0x9f, 0x0e, 0x20, 0xdb, 0x92, 0x86, 0xe0, 0x5b, 0x30, 0x25,
+	0x20, 0xe3, 0x09, 0x17, 0x63, 0x82, 0x68, 0xe8, 0xba, 0x2d, 0x4b, 0x8a, 0x18, 0x4b, 0x44, 0x94,
+	0x66, 0x8c, 0x84, 0x8a, 0xab, 0x7a, 0x72, 0x43, 0x10, 0x24, 0xb3, 0xdc, 0xd2, 0xeb, 0x0b, 0x2d,
+	0xdd, 0x7d, 0x04, 0xad, 0x31, 0x4f, 0x32, 0x7c, 0x4c, 0x5e, 0xcd, 0xd3, 0xb7, 0xf1, 0x5e, 0x7b,
+	0x57, 0x29, 0x80, 0x18, 0x01, 0xec, 0x15, 0xde, 0xaf, 0x2a, 0x80, 0xbf, 0x83, 0x9b, 0x85, 0x84,
+	0xa8, 0x97, 0x26, 0x2e, 0x9f, 0xc2, 0x36, 0x8d, 0x83, 0x68, 0x16, 0x12, 0x3f, 0x16, 0xed, 0x27,
+	0xca, 0xa7, 0x5a, 0x47, 0x0e, 0x03, 0x5b, 0x9a, 0xfb, 0x42, 0x32, 0xcd, 0x74, 0xfb, 0x11, 0x20,
+	0xb3, 0x8b, 0x04, 0xf9, 0x8e, 0x8a, 0xdc, 0xd1, 0xd3, 0x9c, 0xfd, 0x40, 0x4b, 0xbb, 0x2f, 0x61,
+	0x7b, 0xd1, 0xb8, 0x0e, 0xd5, 0x8f, 0xa0, 0x55, 0xc0, 0x6e, 0xea, 0xd3, 0x4d, 0xab, 0x2c, 0x14,
+	0xfb, 0x3c, 0x5b, 0xd2, 0xfd, 0x3e, 0xbc, 0x57, 0xb0, 0x9e, 0xc8, 0x42, 0x7b, 0x59, 0xfd, 0x1f,
+	0xc2, 0x60, 0x59, 0x5c, 0xf9, 0xe0, 0xfe, 0xa7, 0x02, 0xed, 0x27, 0x3a, 0xa3, 0x44, 0x0f, 0xb6,
+	0xba, 0x6e, 0x53, 0x76, 0xdd, 0x7b, 0xd0, 0x2e, 0xbd, 0xb4, 0xd4, 0x38, 0xd7, 0x3a, 0xb3, 0x9e,
+	0x59, 0xab, 0x1e, 0x64, 0x55, 0x29, 0xb6, 0xf8, 0x20, 0x7b, 0x00, 0xfd, 0xa3, 0x8c, 0x90, 0xe5,
+	0xb7, 0x5b, 0xcd, 0xdb, 0x10, 0x0c, 0x5b, 0x76, 0x17, 0x36, 0x71, 0xc0, 0xe9, 0xd9, 0x82, 0xb4,
+	0xba, 0x5f, 0x7d, 0xc5, 0xb2, 0xe5, 0x3f, 0xcf, 0x1d, 0xa5, 0xf1, 0x51, 0xc2, 0x06, 0xf5, 0xb7,
+	0x7f, 0x7b, 0xe9, 0xd3, 0x08, 0x0e, 0x43, 0x87, 0xd0, 0x35, 0x33, 0xbc, 0xd6, 0xb4, 0x7e, 0xed,
+	0xf7, 0x41, 0x9b, 0x14, 0x2c, 0xe6, 0xfe, 0xb1, 0x02, 0x0d, 0x0f, 0x07, 0xa7, 0xef, 0x36, 0xbe,
+	0x9f, 0xc1, 0x46, 0x5e, 0x8b, 0x4b, 0x10, 0xbf, 0x67, 0x01, 0x63, 0x5f, 0x25, 0xaf, 0x13, 0x5a,
+	0x5f, 0xcc, 0xfd, 0xbf, 0x03, 0xdd, 0x27, 0x79, 0xbd, 0x7f, 0xb7, 0xc1, 0x78, 0x08, 0x20, 0x1a,
+	0x54, 0x09, 0x07, 0xbb, 0xa1, 0x9b, 0x70, 0x7b, 0xcd, 0x4c, 0xaf, 0x98, 0xfb, 0xe7, 0x0a, 0xb4,
+	0x5f, 0x25, 0x69, 0x12, 0x25, 0xc7, 0xf3, 0x77, 0xfb, 0xf4, 0xfb, 0xd0, 0xb7, 0x7a, 0x79, 0x09,
+	0x84, 0x5b, 0x0b, 0x97, 0xa1, 0x08, 0xb6, 0xb7, 0x11, 0x96, 0xbe, 0x99, 0xbb, 0x09, 0x7d, 0x3d,
+	0x97, 0x16, 0x25, 0xd9, 0xfd, 0x83, 0x03, 0xc8, 0xa6, 0xea, 0x5a, 0xf9, 0x53, 0xe8, 0x70, 0x8d,
+	0x9d, 0xb4, 0xa7, 0x47, 0x73, 0xfb, 0xee, 0xd9, 0xd8, 0x7a, 0x6d, 0x6e, 0x23, 0xfd, 0x03, 0xd8,
+	0x5a, 0x7a, 0x5f, 0xfb, 0xd3, 0x89, 0x46, 0xb8, 0xbf, 0xf0, 0xc4, 0x7e, 0x3e, 0x71, 0x3f, 0x85,
+	0x9b, 0x6a, 0x38, 0x34, 0x75, 0xdc, 0xd4, 0xd7, 0xa5, 0x29, 0xaf, 0x53, 0x4c, 0x79, 0xee, 0xff,
+	0x1c, 0xd8, 0x5e, 0xdc, 0xa6, 0xfd, 0xbf, 0x6c, 0x1f, 0xc2, 0x80, 0x74, 0xbd, 0xb1, 0xe7, 0x55,
+	0x35, 0x26, 0x7e, 0xb2, 0x34, 0xaf, 0x2e, 0xea, 0xde, 0x35, 0x75, 0xa8, 0x18, 0x59, 0x7b, 0xac,
+	0x4c, 0x60, 0x43, 0x0c, 0xfd, 0x25, 0x31, 0x31, 0xd5, 0x1b, 0xbb, 0xda, 0xa7, 0x75, 0xbd, 0xf1,
+	0x1b, 0x0c, 0xac, 0xee, 0x5d, 0xb8, 0xfd, 0x05, 0xe1, 0xcf, 0xa5, 0xcc, 0x5e, 0x12, 0x1f, 0xd1,
+	0xe3, 0x59, 0xa6, 0x84, 0x8a, 0xd0, 0xde, 0xb9, 0x48, 0x42, 0xc3, 0xb4, 0xe2, 0x27, 0x86, 0x73,
+	0xed, 0x9f, 0x18, 0x95, 0xcb, 0x7e, 0x62, 0x3c, 0xfc, 0x77, 0x1d, 0xd6, 0xc7, 0x04, 0xbf, 0x21,
+	0x24, 0x44, 0x07, 0xd0, 0x19, 0x93, 0x38, 0x2c, 0x7e, 0x4f, 0x6e, 0x59, 0x67, 0xcc, 0xa9, 0xc3,
+	0xef, 0xac, 0xa2, 0xe6, 0x2d, 0xf4, 0xc6, 0x8e, 0xf3, 0xb1, 0x83, 0x0e, 0xa1, 0xf3, 0x94, 0x90,
+	0x74, 0x2f, 0x89, 0x63, 0x12, 0x70, 0x12, 0xa2, 0x3b, 0x76, 0x23, 0x5f, 0x7e, 0xa9, 0x0d, 0x6f,
+	0x2d, 0xf5, 0x13, 0x03, 0xaa, 0xd6, 0xf8, 0x12, 0xda, 0xf6, 0x03, 0xa5, 0xa4, 0x70, 0xc5, 0x73,
+	0x6a, 0x78, 0xf7, 0x8a, 0x97, 0x8d, 0x7b, 0x03, 0x7d, 0x06, 0x75, 0x35, 0x31, 0xa3, 0x81, 0x25,
+	0x5c, 0x7a, 0x12, 0x94, 0xfc, 0x2a, 0x8f, 0xd7, 0xee, 0x0d, 0xf4, 0x14, 0xa0, 0x98, 0x39, 0x91,
+	0x8d, 0xcb, 0xd2, 0xd0, 0x3b, 0xbc, 0x7d, 0x01, 0x37, 0x57, 0xf6, 0x4b, 0xe8, 0x96, 0x27, 0x23,
+	0x34, 0x5a, 0x39, 0xfc, 0x58, 0xe5, 0x61, 0x78, 0xef, 0x12, 0x89, 0x5c, 0xf1, 0xaf, 0xa1, 0xb7,
+	0x38, 0xf0, 0x20, 0x77, 0xe5, 0xc6, 0xd2, 0xf0, 0x34, 0xfc, 0xe0, 0x52, 0x19, 0x1b, 0x84, 0xa2,
+	0x42, 0x95, 0x40, 0x58, 0x2a, 0x67, 0x25, 0x10, 0x96, 0xcb, 0x9a, 0x02, 0xa1, 0x9c, 0xd6, 0x25,
+	0x10, 0x56, 0x16, 0xa1, 0x12, 0x08, 0xab, 0x6b, 0x82, 0x7b, 0x03, 0x25, 0xb0, 0xbd, 0x3a, 0xd9,
+	0x90, 0xfd, 0x3f, 0xe3, 0xd2, 0x8c, 0x1d, 0xde, 0x7f, 0x0b, 0x49, 0x63, 0x70, 0x52, 0x97, 0xbf,
+	0xfe, 0x3f, 0xf9, 0x3a, 0x00, 0x00, 0xff, 0xff, 0x28, 0x1f, 0x4c, 0x0e, 0x0a, 0x18, 0x00, 0x00,
 }

weed/pb/volume_server.proto

@@ -19,12 +19,12 @@ service VolumeServer {
 
     rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
     }
-    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
+    rpc AllocateVolume (AllocateVolumeRequest) returns (AllocateVolumeResponse) {
     }
 
     rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
     }
-    rpc VolumeFollow (VolumeFollowRequest) returns (stream VolumeFollowResponse) {
+    rpc VolumeIncrementalCopy (VolumeIncrementalCopyRequest) returns (stream VolumeIncrementalCopyResponse) {
     }
 
     rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
@@ -33,14 +33,40 @@ service VolumeServer {
     }
     rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
     }
+    rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
+    }
 
-    rpc ReplicateVolume (ReplicateVolumeRequest) returns (ReplicateVolumeResponse) {
+    // copy the .idx .dat files, and mount this volume
+    rpc VolumeCopy (VolumeCopyRequest) returns (VolumeCopyResponse) {
     }
     rpc ReadVolumeFileStatus (ReadVolumeFileStatusRequest) returns (ReadVolumeFileStatusResponse) {
     }
     rpc CopyFile (CopyFileRequest) returns (stream CopyFileResponse) {
     }
 
+    rpc VolumeTailSender (VolumeTailSenderRequest) returns (stream VolumeTailSenderResponse) {
+    }
+    rpc VolumeTailReceiver (VolumeTailReceiverRequest) returns (VolumeTailReceiverResponse) {
+    }
+
+    // erasure coding
+    rpc VolumeEcShardsGenerate (VolumeEcShardsGenerateRequest) returns (VolumeEcShardsGenerateResponse) {
+    }
+    rpc VolumeEcShardsRebuild (VolumeEcShardsRebuildRequest) returns (VolumeEcShardsRebuildResponse) {
+    }
+    rpc VolumeEcShardsCopy (VolumeEcShardsCopyRequest) returns (VolumeEcShardsCopyResponse) {
+    }
+    rpc VolumeEcShardsDelete (VolumeEcShardsDeleteRequest) returns (VolumeEcShardsDeleteResponse) {
+    }
+    rpc VolumeEcShardsMount (VolumeEcShardsMountRequest) returns (VolumeEcShardsMountResponse) {
+    }
+    rpc VolumeEcShardsUnmount (VolumeEcShardsUnmountRequest) returns (VolumeEcShardsUnmountResponse) {
+    }
+    rpc VolumeEcShardRead (VolumeEcShardReadRequest) returns (stream VolumeEcShardReadResponse) {
+    }
+    rpc VolumeEcBlobDelete (VolumeEcBlobDeleteRequest) returns (VolumeEcBlobDeleteResponse) {
+    }
+
 }
 
 //////////////////////////////////////////////////
@@ -57,6 +83,7 @@ message DeleteResult {
     int32 status = 2;
     string error = 3;
     uint32 size = 4;
+    uint32 version = 5;
 }
 
 message Empty {
@@ -94,14 +121,14 @@ message DeleteCollectionRequest {
 message DeleteCollectionResponse {
 }
 
-message AssignVolumeRequest {
+message AllocateVolumeRequest {
     uint32 volume_id = 1;
     string collection = 2;
     int64 preallocate = 3;
     string replication = 4;
     string ttl = 5;
 }
-message AssignVolumeResponse {
+message AllocateVolumeResponse {
 }
 
 message VolumeSyncStatusRequest {
@@ -117,11 +144,11 @@ message VolumeSyncStatusResponse {
     uint64 idx_file_size = 8;
 }
 
-message VolumeFollowRequest {
+message VolumeIncrementalCopyRequest {
     uint32 volume_id = 1;
-    uint64 since = 2;
+    uint64 since_ns = 2;
 }
-message VolumeFollowResponse {
+message VolumeIncrementalCopyResponse {
     bytes file_content = 1;
 }
 
@@ -143,34 +170,136 @@ message VolumeDeleteRequest {
 message VolumeDeleteResponse {
 }
 
-message ReplicateVolumeRequest {
+message VolumeMarkReadonlyRequest {
+    uint32 volume_id = 1;
+}
+message VolumeMarkReadonlyResponse {
+}
+
+message VolumeCopyRequest {
     uint32 volume_id = 1;
     string collection = 2;
     string replication = 3;
     string ttl = 4;
     string source_data_node = 5;
 }
-message ReplicateVolumeResponse {
+message VolumeCopyResponse {
+    uint64 last_append_at_ns = 1;
 }
 
 message CopyFileRequest {
     uint32 volume_id = 1;
-    bool is_idx_file = 2;
-    bool is_dat_file = 3;
+    string ext = 2;
+    uint32 compaction_revision = 3;
+    uint64 stop_offset = 4;
+    string collection = 5;
+    bool is_ec_volume = 6;
 }
 message CopyFileResponse {
     bytes file_content = 1;
 }
 
+message VolumeTailSenderRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+}
+message VolumeTailSenderResponse {
+    bytes needle_header = 1;
+    bytes needle_body = 2;
+    bool is_last_chunk = 3;
+}
+
+message VolumeTailReceiverRequest {
+    uint32 volume_id = 1;
+    uint64 since_ns = 2;
+    uint32 idle_timeout_seconds = 3;
+    string source_volume_server = 4;
+}
+message VolumeTailReceiverResponse {
+}
+
+message VolumeEcShardsGenerateRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsGenerateResponse {
+}
+
+message VolumeEcShardsRebuildRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+}
+message VolumeEcShardsRebuildResponse {
+    repeated uint32 rebuilt_shard_ids = 1;
+}
+
+message VolumeEcShardsCopyRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+    bool copy_ecx_file = 4;
+    string source_data_node = 5;
+}
+message VolumeEcShardsCopyResponse {
+}
+
+message VolumeEcShardsDeleteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsDeleteResponse {
+}
+
+message VolumeEcShardsMountRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsMountResponse {
+}
+
+message VolumeEcShardsUnmountRequest {
+    uint32 volume_id = 1;
+    repeated uint32 shard_ids = 3;
+}
+message VolumeEcShardsUnmountResponse {
+}
+
+message VolumeEcShardReadRequest {
+    uint32 volume_id = 1;
+    uint32 shard_id = 2;
+    int64 offset = 3;
+    int64 size = 4;
+    uint64 file_key = 5;
+}
+message VolumeEcShardReadResponse {
+    bytes data = 1;
+    bool is_deleted = 2;
+}
+
+message VolumeEcBlobDeleteRequest {
+    uint32 volume_id = 1;
+    string collection = 2;
+    uint64 file_key = 3;
+    uint32 version = 4;
+}
+message VolumeEcBlobDeleteResponse {
+}
+
 message ReadVolumeFileStatusRequest {
     uint32 volume_id = 1;
 }
 message ReadVolumeFileStatusResponse {
     uint32 volume_id = 1;
-    uint64 idx_file_timestamp = 2;
+    uint64 idx_file_timestamp_seconds = 2;
     uint64 idx_file_size = 3;
-    uint64 dat_file_timestamp = 4;
+    uint64 dat_file_timestamp_seconds = 4;
     uint64 dat_file_size = 5;
+    uint64 file_count = 6;
+    uint32 compaction_revision = 7;
+    string collection = 8;
 }
 
 message DiskStatus {

weed/replication/replicator.go

@@ -2,6 +2,7 @@ package replication
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 	"strings"
 
@@ -51,12 +52,17 @@ func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_p
 		return nil
 	}
 
-	foundExisting, err := r.sink.UpdateEntry(ctx, key, message.OldEntry, message.NewEntry, message.DeleteChunks)
+	foundExisting, err := r.sink.UpdateEntry(ctx, key, message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks)
 	if foundExisting {
 		glog.V(4).Infof("updated %v", key)
 		return err
 	}
 
+	err = r.sink.DeleteEntry(ctx, key, message.OldEntry.IsDirectory, false)
+	if err != nil {
+		return fmt.Errorf("delete old entry %v: %v", key, err)
+	}
+
 	glog.V(4).Infof("creating missing %v", key)
 	return r.sink.CreateEntry(ctx, key, message.NewEntry)
 }

weed/replication/sink/azuresink/azure_sink.go

@@ -132,7 +132,7 @@ func (g *AzureSink) CreateEntry(ctx context.Context, key string, entry *filer_pb
 
 }
 
-func (g *AzureSink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (g *AzureSink) UpdateEntry(ctx context.Context, key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 	key = cleanKey(key)
 	// TODO improve efficiency
 	return false, nil

weed/replication/sink/b2sink/b2_sink.go

@@ -124,7 +124,7 @@ func (g *B2Sink) CreateEntry(ctx context.Context, key string, entry *filer_pb.En
 
 }
 
-func (g *B2Sink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (g *B2Sink) UpdateEntry(ctx context.Context, key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 
 	key = cleanKey(key)
 

weed/replication/sink/filersink/fetch_write.go

@@ -3,6 +3,7 @@ package filersink
 import (
 	"context"
 	"fmt"
+	"google.golang.org/grpc"
 	"strings"
 	"sync"
 
@@ -38,7 +39,7 @@ func (fs *FilerSink) replicateOneChunk(ctx context.Context, sourceChunk *filer_p
 
 	fileId, err := fs.fetchAndWrite(ctx, sourceChunk)
 	if err != nil {
-		return nil, fmt.Errorf("copy %s: %v", sourceChunk.FileId, err)
+		return nil, fmt.Errorf("copy %s: %v", sourceChunk.GetFileIdString(), err)
 	}
 
 	return &filer_pb.FileChunk{
@@ -47,15 +48,15 @@ func (fs *FilerSink) replicateOneChunk(ctx context.Context, sourceChunk *filer_p
 		Size:         sourceChunk.Size,
 		Mtime:        sourceChunk.Mtime,
 		ETag:         sourceChunk.ETag,
-		SourceFileId: sourceChunk.FileId,
+		SourceFileId: sourceChunk.GetFileIdString(),
 	}, nil
 }
 
 func (fs *FilerSink) fetchAndWrite(ctx context.Context, sourceChunk *filer_pb.FileChunk) (fileId string, err error) {
 
-	filename, header, readCloser, err := fs.filerSource.ReadPart(ctx, sourceChunk.FileId)
+	filename, header, readCloser, err := fs.filerSource.ReadPart(ctx, sourceChunk.GetFileIdString())
 	if err != nil {
-		return "", fmt.Errorf("read part %s: %v", sourceChunk.FileId, err)
+		return "", fmt.Errorf("read part %s: %v", sourceChunk.GetFileIdString(), err)
 	}
 	defer readCloser.Close()
 
@@ -105,15 +106,11 @@ func (fs *FilerSink) fetchAndWrite(ctx context.Context, sourceChunk *filer_pb.Fi
 
 func (fs *FilerSink) withFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(ctx, fs.grpcAddress, fs.grpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", fs.grpcAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, fs.grpcAddress, fs.grpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 func volumeId(fileId string) string {

weed/replication/sink/filersink/filer_sink.go

@@ -133,7 +133,7 @@ func (fs *FilerSink) CreateEntry(ctx context.Context, key string, entry *filer_p
 	})
 }
 
-func (fs *FilerSink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (fs *FilerSink) UpdateEntry(ctx context.Context, key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 
 	dir, name := filer2.FullPath(key).DirAndName()
 
@@ -179,7 +179,7 @@ func (fs *FilerSink) UpdateEntry(ctx context.Context, key string, oldEntry, newE
 		// delete the chunks that are deleted from the source
 		if deleteIncludeChunks {
 			// remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks
-			existingEntry.Chunks = minusChunks(existingEntry.Chunks, deletedChunks)
+			existingEntry.Chunks = filer2.MinusChunks(existingEntry.Chunks, deletedChunks)
 		}
 
 		// replicate the chunks that are new in the source
@@ -194,7 +194,7 @@ func (fs *FilerSink) UpdateEntry(ctx context.Context, key string, oldEntry, newE
 	return true, fs.withFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
-			Directory: dir,
+			Directory: newParentPath,
 			Entry:     existingEntry,
 		}
 
@@ -207,23 +207,7 @@ func (fs *FilerSink) UpdateEntry(ctx context.Context, key string, oldEntry, newE
 
 }
 func compareChunks(oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk) {
-	deletedChunks = minusChunks(oldEntry.Chunks, newEntry.Chunks)
-	newChunks = minusChunks(newEntry.Chunks, oldEntry.Chunks)
-	return
-}
-
-func minusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
-	for _, a := range as {
-		found := false
-		for _, b := range bs {
-			if a.FileId == b.FileId {
-				found = true
-				break
-			}
-		}
-		if !found {
-			delta = append(delta, a)
-		}
-	}
+	deletedChunks = filer2.MinusChunks(oldEntry.Chunks, newEntry.Chunks)
+	newChunks = filer2.MinusChunks(newEntry.Chunks, oldEntry.Chunks)
 	return
 }

weed/replication/sink/gcssink/gcs_sink.go

@@ -119,7 +119,7 @@ func (g *GcsSink) CreateEntry(ctx context.Context, key string, entry *filer_pb.E
 
 }
 
-func (g *GcsSink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (g *GcsSink) UpdateEntry(ctx context.Context, key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 	// TODO improve efficiency
 	return false, nil
 }

weed/replication/sink/replication_sink.go

@@ -12,7 +12,7 @@ type ReplicationSink interface {
 	Initialize(configuration util.Configuration) error
 	DeleteEntry(ctx context.Context, key string, isDirectory, deleteIncludeChunks bool) error
 	CreateEntry(ctx context.Context, key string, entry *filer_pb.Entry) error
-	UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error)
+	UpdateEntry(ctx context.Context, key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error)
 	GetSinkToDirectory() string
 	SetSourceFiler(s *source.FilerSource)
 }

weed/replication/sink/s3sink/s3_sink.go

@@ -130,7 +130,7 @@ func (s3sink *S3Sink) CreateEntry(ctx context.Context, key string, entry *filer_
 
 }
 
-func (s3sink *S3Sink) UpdateEntry(ctx context.Context, key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+func (s3sink *S3Sink) UpdateEntry(ctx context.Context, key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
 	key = cleanKey(key)
 	// TODO improve efficiency
 	return false, nil

weed/replication/source/filer_source.go

@@ -91,15 +91,11 @@ func (fs *FilerSource) ReadPart(ctx context.Context, part string) (filename stri
 
 func (fs *FilerSource) withFilerClient(ctx context.Context, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(ctx, fs.grpcAddress, grpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", fs.grpcAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, fs.grpcAddress, fs.grpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 func volumeId(fileId string) string {

weed/replication/sub/notification_gocdk_pub_sub.go

@@ -0,0 +1,50 @@
+package sub
+
+import (
+	"context"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"gocloud.dev/pubsub"
+	_ "gocloud.dev/pubsub/awssnssqs"
+	_ "gocloud.dev/pubsub/azuresb"
+	_ "gocloud.dev/pubsub/gcppubsub"
+	_ "gocloud.dev/pubsub/natspubsub"
+	_ "gocloud.dev/pubsub/rabbitpubsub"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &GoCDKPubSubInput{})
+}
+
+type GoCDKPubSubInput struct {
+	sub *pubsub.Subscription
+}
+
+func (k *GoCDKPubSubInput) GetName() string {
+	return "gocdk_pub_sub"
+}
+
+func (k *GoCDKPubSubInput) Initialize(config util.Configuration) error {
+	subURL := config.GetString("sub_url")
+	glog.V(0).Infof("notification.gocdk_pub_sub.sub_url: %v", subURL)
+	sub, err := pubsub.OpenSubscription(context.Background(), subURL)
+	if err != nil {
+		return err
+	}
+	k.sub = sub
+	return nil
+}
+
+func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) {
+	msg, err := k.sub.Receive(context.Background())
+	key = msg.Metadata["key"]
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(msg.Body, message)
+	if err != nil {
+		return "", nil, err
+	}
+	return key, message, nil
+}

weed/s3api/custom_types.go

@@ -0,0 +1,3 @@
+package s3api
+
+const s3TimeFormat = "2006-01-02T15:04:05.999Z07:00"

weed/s3api/filer_multipart.go

@@ -69,7 +69,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(ctx context.Context, input *s3.C
 		if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
 			for _, chunk := range entry.Chunks {
 				p := &filer_pb.FileChunk{
-					FileId: chunk.FileId,
+					FileId: chunk.GetFileIdString(),
 					Offset: offset,
 					Size:   chunk.Size,
 					Mtime:  chunk.Mtime,

weed/s3api/s3api_handlers.go

@@ -9,6 +9,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
 	"net/http"
 	"net/url"
 	"time"
@@ -38,15 +39,11 @@ func encodeResponse(response interface{}) []byte {
 
 func (s3a *S3ApiServer) withFilerClient(ctx context.Context, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(ctx, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", s3a.option.FilerGrpcAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(ctx, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 // If none of the http routes match respond with MethodNotAllowed