1.15

fix https://github.com/chrislusf/seaweedfs/issues/801

.gitignore

@@ -77,3 +77,6 @@ crashlytics-build.properties
 
 test_data
 build
+target
+*.class
+other/java/hdfs/dependency-reduced-pom.xml

.project

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-	<name>seaweedfs</name>
-	<comment></comment>
-	<projects>
-	</projects>
-	<buildSpec>
-	</buildSpec>
-	<natures>
-		<nature>goclipse.goNature</nature>
-		<nature>org.eclipse.wst.common.project.facet.core.nature</nature>
-	</natures>
-</projectDescription>

.travis.yml

@@ -1,7 +1,8 @@
 sudo: false
 language: go
 go:
-- 1.9
+- 1.10.x
+- 1.11.x
 - tip
 
 before_install:

Dockerfile.go_build

@@ -1,6 +0,0 @@
-FROM cydev/go
-RUN go get github.com/chrislusf/seaweedfs/weed
-EXPOSE 8080
-EXPOSE 9333
-VOLUME /data
-ENTRYPOINT ["weed"]

Makefile

@@ -8,7 +8,7 @@ appname := weed
 
 sources := $(wildcard *.go)
 
-build = GOOS=$(1) GOARCH=$(2) go build -o build/$(appname)$(3) $(SOURCE_DIR)
+build = CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build -ldflags "-extldflags -static" -o build/$(appname)$(3) $(SOURCE_DIR)
 tar = cd build && tar -cvzf $(1)_$(2).tar.gz $(appname)$(3) && rm $(appname)$(3)
 zip = cd build && zip $(1)_$(2).zip $(appname)$(3) && rm $(appname)$(3)
 

README.md

@@ -8,17 +8,17 @@
 
 <h2 align="center">Supporting SeaweedFS</h2>
 
-SeaweedFS is Apache-licensed open source project, independent project with its ongoing development made 
-possible entirely thanks to the support by these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
-If you'd like to grow SeaweedFS even stronger, please consider to 
-<a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a>.
+SeaweedFS is an independent Apache-licensed open source project with its ongoing development made 
+possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
+If you'd like to grow SeaweedFS even stronger, please consider joining our
+<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
 
 Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
 Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
 
 Your support will be really appreciated by me and other supporters!
 
-<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsors SeaweedFS via Patreon</a></h3>
+<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
 
 <h4 align="center">Platinum</h4>
 
@@ -61,35 +61,41 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 
 SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
 
-There is only a 40 bytes disk storage overhead for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases.
+There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
 SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
 
-SeaweedFS can work very well with just the object store. [[Filer]] is added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
+SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
 
 ## Additional Features
-* Can choose no replication or different replication level, rack and data center aware
+* Can choose no replication or different replication levels, rack and data center aware
 * Automatic master servers failover - no single point of failure (SPOF)
 * Automatic Gzip compression depending on file mime type
-* Automatic compaction to reclaimed disk spaces after deletion or update
+* Automatic compaction to reclaim disk space after deletion or update
 * Servers in the same cluster can have different disk spaces, file systems, OS etc.
 * Adding/Removing servers does **not** cause any data re-balancing
-* Optional [filer server][Filer] provides "normal" directories and files via http
-* Optional [mount filer][Mount] to read and write files directly as a local directory
-* Optional [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
 * Optionally fix the orientation for jpeg pictures
 * Support Etag, Accept-Range, Last-Modified, etc.
 * Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
 
+## Filer Features
+* [filer server][Filer] provide "normal" directories and files via http.
+* [mount filer][Mount] to read and write files directly as a local directory via FUSE.
+* [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
+* [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
+* [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
 [Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
 [AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
+[BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
+[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
 
 ## Example Usage
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.
-Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. I just use localhost as an example.
+Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
 
-SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in JSON or JSONP format.
+SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
 
 ### Start Master Server
 
@@ -104,24 +110,23 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in
 > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
 ```
 
-
 ### Write File ###
 
 To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url:
 
 ```
-> curl -X POST http://localhost:9333/dir/assign
+> curl http://localhost:9333/dir/assign
 {"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
 ```
 
-Second, to store the file content, send a HTTP multi-part PUT or POST request to `url + '/' + fid` from the response:
+Second, to store the file content, send a HTTP multi-part POST request to `url + '/' + fid` from the response:
 
 ```
-> curl -X PUT -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
+> curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
 {"size": 43234}
 ```
 
-To update, send another PUT or POST request with updated file content.
+To update, send another POST request with updated file content.
 
 For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 
@@ -130,17 +135,17 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 ```
 ### Save File Id ###
 
-Now you can save the `fid`, 3,01637037d6 in this case, to a database field.
+Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
 
 The number 3 at the start represents a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
 
 The volume id is an unsigned 32-bit integer. The file key is an unsigned 64-bit integer. The file cookie is an unsigned 32-bit integer, used to prevent URL guessing.
 
-The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as string.
+The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as a string.
 
-If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most usage would not need 2^32 volumes.
+If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most uses will not need 2^32 volumes.
 
-If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are more than enough.
+If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
 
 ### Read File ###
 
@@ -186,7 +191,7 @@ http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fill
 SeaweedFS applies the replication strategy at a volume level. So, when you are getting a file id, you can specify the replication strategy. For example:
 
 ```
-curl -X POST http://localhost:9333/dir/assign?replication=001
+curl http://localhost:9333/dir/assign?replication=001
 ```
 
 The replication parameter options are:
@@ -208,7 +213,7 @@ You can also set the default replication strategy when starting the master serve
 
 ### Allocate File Key on specific data center ###
 
-Volume servers can start with a specific data center name:
+Volume servers can be started with a specific data center name:
 
 ```
  weed volume -dir=/tmp/1 -port=8080 -dataCenter=dc1
@@ -236,9 +241,9 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
 
 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
 
-The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, might not scale well for many concurrent users.
+The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, so it might not scale well for many concurrent users.
 
-Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and is generally stable.
+Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
 
 The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
 
@@ -248,7 +253,7 @@ For comparison, consider that an xfs inode structure in Linux is 536 bytes.
 
 The architecture is fairly simple. The actual data is stored in volumes on storage nodes. One volume server can have multiple volumes, and can both support read and write access with basic authentication.
 
-All volumes are managed by a master server. The master server contains volume id to volume server mapping. This is fairly static information, and could be cached easily.
+All volumes are managed by a master server. The master server contains the volume id to volume server mapping. This is fairly static information, and can be easily cached.
 
 On each write request, the master server also generates a file key, which is a growing 64-bit unsigned integer. Since write requests are not generally as frequent as read requests, one master server should be able to handle the concurrency well.
 
@@ -256,36 +261,27 @@ On each write request, the master server also generates a file key, which is a g
 
 When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content.
 
-When a client needs to read a file based on (volume id, file key, file cookie), it can ask the master server by the volume id for the (volume node url, volume node public url), or retrieve this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
+When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node url, volume node public url), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
 
 Please see the example for details on the write-read process.
 
 ### Storage Size ###
 
-In the current implementation, each volume can be 8x2^32 bytes (32GiB). This is because of we align content to 8 bytes. We can easily increase this to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
+In the current implementation, each volume can hold 32 gibibytes (32GiB or 8x2^32 bytes). This is because we align content to 8 bytes. We can easily increase this to 64GiB, or 128GiB, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
 
-There can be 2^32 volumes. So total system size is 8 x 2^32 bytes x 2^32 = 8 x 4GiB x 4Gi = 128EiB (2^67 bytes, or 128 exbibytes).
+There can be 4 gibibytes (4GiB or 2^32 bytes) of volumes. So the total system size is 8 x 4GiB x 4GiB which is 128 exbibytes (128EiB or 2^67 bytes).
 
 Each individual file size is limited to the volume size.
 
 ### Saving memory ###
 
-All file meta information on volume server is readable from memory without disk access. Each file just takes an 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own the space cost for the map. But usually the disk runs out before the memory does.
-
+All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
 ## Compared to Other File Systems ##
 
 Most other distributed file systems seem more complicated than necessary.
 
-### Compared to Ceph ###
-
-Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
-
-SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Please raise an issue with any questions or update this file with clarifications.
-
-SeaweedFS has a centralized master to look up free volumes, while Ceph uses hashing to locate its objects. Having a centralized master makes it easy to code and manage. HDFS/GFS has the single name node for years. SeaweedFS now support multiple master nodes.
-
-Ceph hashing avoids SPOF, but makes it complicated when moving or adding servers.
+SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
 
 ### Compared to HDFS ###
 
@@ -301,9 +297,9 @@ SeaweedFS can also store extra large files by splitting them into manageable dat
 The architectures are mostly the same. SeaweedFS aims to store and read files fast, with a simple and flat architecture. The main differences are
 
 * SeaweedFS optimizes for small files, ensuring O(1) disk seek operation, and can also handle large files.
-* SeaweedFS statically assign a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
-* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Cassandra, Redis, MySql, PostGres, etc, and is easy to customized.
-* SeaweedFS Volume server also communicate directly with clients via HTTP, supporting range queries, direct uploads, etc.
+* SeaweedFS statically assigns a volume id for a file. Locating file content becomes just a lookup of the volume id, which can be easily cached.
+* SeaweedFS Filer metadata store can be any well-known and proven data stores, e.g., Cassandra, Redis, MySql, Postgres, etc, and is easy to customized.
+* SeaweedFS Volume server also communicates directly with clients via HTTP, supporting range queries, direct uploads, etc.
 
 | System         | File Meta                       | File Content Read| POSIX  | REST API | Optimized for small files |
 | -------------  | ------------------------------- | ---------------- | ------ | -------- | ------------------------- |
@@ -320,10 +316,18 @@ GlusterFS hashes the path and filename into ids, and assigned to virtual volumes
 
 ### Compared to Ceph ###
 
-Same as SeaweedFS, Ceph is also based on a object store RADOS. Ceph is rather complicated with mixed reviews.
+Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more complicated, with the need to support layers on top of it. [Here is a more detailed comparison](https://github.com/chrislusf/seaweedfs/issues/120)
+
+SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. 
+
+Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
 
 Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS places data by assigned volumes.
 
+SeaweedFS is optimized for small files. Small files are stored as one continuous block of content, with at most 8 unused bytes between files. Small file access is O(1) disk read.
+
+SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Cassandra, to manage file directories. There are proven, scalable, and easier to manage.
+
 | SeaweedFS         | comparable to Ceph | advantage |
 | -------------  | ------------- | ---------------- |
 | Master  | MDS | simpler |
@@ -334,37 +338,43 @@ Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS pl
 ## Dev plan ##
 
 More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
-Other key features include: Erasure Encoding, support S3 API, JWT security.
+Other key features include: Erasure Encoding, JWT security.
 
-This is a super exciting project! And I need helpers and [support](https://www.patreon.com/seaweedfs)!
+This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
 
 ## Installation guide for users who are not familiar with golang
 
-step 1: install go on your machine and setup the environment by following the instructions from the following link:
+Step 1: install go on your machine and setup the environment by following the instructions at:
 
 https://golang.org/doc/install
 
 make sure you set up your $GOPATH
 
 
-step 2: also you may need to install Mercurial by following the instructions below
+Step 2: also you may need to install Mercurial by following the instructions at:
 
 http://mercurial.selenic.com/downloads
 
-step 3: download, compile, and install the project by executing the following command
+Step 3: download, compile, and install the project by executing the following command
 
+```bash
 go get github.com/chrislusf/seaweedfs/weed
+```
 
-once this is done, you should see the executable "weed" under $GOPATH/bin
+Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/seaweedfs/weed
+Step 4: after you modify your code locally, you could start a local build by calling `go install` under 
+
+```
+$GOPATH/src/github.com/chrislusf/seaweedfs/weed
+```
 
 ## Disk Related topics ##
 
 ### Hard Drive Performance ###
 
-When testing read performance on SeaweedFS, it basically becomes performance test for your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
+When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
 
 ### Solid State Disk
 
@@ -437,3 +447,8 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
+
+
+## Stargazers over time
+
+[![Stargazers over time](https://starcharts.herokuapp.com/chrislusf/seaweedfs.svg)](https://starcharts.herokuapp.com/chrislusf/seaweedfs)

docker/Dockerfile

@@ -7,11 +7,22 @@ RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certif
     apk del build-dependencies && \
     rm -rf /tmp/*
 
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
 EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc+http port
 EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
 
 VOLUME /data
 
+COPY filer.toml /etc/seaweedfs/filer.toml
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 

docker/Dockerfile.go_build

@@ -0,0 +1,24 @@
+FROM golang:latest
+RUN go get github.com/chrislusf/seaweedfs/weed
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc+http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+VOLUME /data
+
+RUN mkdir -p /etc/seaweedfs
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/README.md

@@ -0,0 +1,10 @@
+# Docker
+
+## Development
+
+```
+docker build - < Dockerfile.go_build
+
+docker-compose -f docker-compose.yml up
+
+```

docker/docker-compose.yml

@@ -2,22 +2,53 @@ version: '2'
 
 services:
   master:
-    image: chrislusf/seaweedfs
+    #image: chrislusf/seaweedfs # use a remote image
+    build: . # build our container from the local Dockerfile
     ports:
-      - 9333:9333
+    - 9333:9333
     command: "master"
     networks:
       default:
         aliases:
-          - seaweed_master
+        - seaweed_master
   volume:
-    image: chrislusf/seaweedfs
+    #image: chrislusf/seaweedfs # use a remote image
+    build: . # build our container from the local Dockerfile
     ports:
-      - 8080:8080
+    - 8080:8080
+    - 18080:18080
     command: 'volume -max=5 -mserver="master:9333" -port=8080'
-    depends_on: 
-     - master
+    depends_on:
+    - master
     networks:
       default:
         aliases:
-          - seaweed_volume
+        - seaweed_volume
+  filer:
+    #image: chrislusf/seaweedfs # use a remote image
+    build: . # build our container from the local Dockerfile
+    ports:
+    - 8888:8888
+    - 18888:18888
+    command: 'filer -master="master:9333"'
+    depends_on:
+    - master
+    - volume
+    networks:
+      default:
+        aliases:
+        - seaweed_filer
+  s3:
+    #image: chrislusf/seaweedfs # use a remote image
+    build: . # build our container from the local Dockerfile
+    ports:
+    - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+    - master
+    - volume
+    - filer
+    networks:
+      default:
+        aliases:
+        - seaweed_s3

docker/entrypoint.sh

@@ -28,6 +28,22 @@ case "$1" in
   	exec /usr/bin/weed $@ $ARGS
   	;;
 
+  'filer')
+  	ARGS="-ip `hostname -i`"
+  	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
+		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  's3')
+  	ARGS="-domainName='$S3_DOMAIN_NAME' -key.file='$S3_KEY_FILE' -cert.file='$S3_CERT_FILE'"
+  	if [ -n "$FILER_PORT_8888_TCP_ADDR" ] ; then
+		ARGS="$ARGS -filer=$FILER_PORT_8888_TCP_ADDR:$FILER_PORT_8888_TCP_PORT"
+	fi
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
   *)
   	exec /usr/bin/weed $@
 	;;

docker/filer.toml

@@ -0,0 +1,3 @@
+[leveldb]
+enabled = true
+dir = "."

note/memory_usage.txt

@@ -1,278 +0,0 @@
-64 32G volumes consumes 10G memory
-Each volume has 25M index, so each cost 160MB memory
-
-
-
-Things happened when I use lots of threads ( almost 120 ) keeping read file from SeaweedFS.
-But I'm not so familiar with linux so I can't tell you exactly what happened.
-Next I'll show you things I know , if you need more info , contact me
-
-My SeaweedFS version is about 0.12
-
-1. top
-
-top - 12:07:37 up 1 day,  3:17,  2 users,  load average: 0.00, 0.00, 0.00
-Tasks: 152 total,   1 running, 151 sleeping,   0 stopped,   0 zombie
-Cpu(s):  0.0%us,  0.0%sy,  0.0%ni, 99.8%id,  0.1%wa,  0.0%hi,  0.0%si,  0.0%st
-Mem:  16269880k total, 16192364k used,    77516k free,    58172k buffers
-Swap:  2064376k total,    12324k used,  2052052k free,  2827520k cached
-
-  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
- 1499 root      20   0 11.6g  10g 1424 S  0.0 65.6   7:32.53 weedvolume
- 1498 root      20   0 3204m 2.1g 1428 S  0.0 13.5   4:36.59 weedvolume
- 1737 root      20   0 98868 4932 2920 S  0.0  0.0   0:00.56 sshd
- 1497 root      20   0  151m 4404 1152 S  0.0  0.0   1:21.40 weedmaster
- 1335 root      20   0 97816 3044 2896 S  0.0  0.0   0:00.76 sshd
-
-After system became steady ,  weedvolume used 65.6% memory .
-
-2. free -m
-
-             total       used       free     shared    buffers     cached
-Mem:         15888      15809         79          0         56       2758
--/+ buffers/cache:      12994       2894
-Swap:         2015         12       2003
-
-3. startup cmd
-
-screen -d -m /opt/weed/weedmaster -mdir /data/weeddata/ > /data/logs/weed/master.log &
-screen -d -m /opt/weed/weedvolume -volumes=0-64 -dir /data/weeddata/ -pulseSeconds=20 -publicUrl="x.y.z:9334" -port 9334 > /data/logs/weed/s01.log &
-screen -d -m /opt/weed/weedvolume -volumes=65-107 -dir /data/weeddata/ -pulseSeconds=20 -publicUrl="x.y.z:9335" -port 9335 > /data/logs/weed/s02.log &
-
-4. du -sh .
-
-32G     0.dat
-26M     0.idx
-8.2G    100.dat
-6.8M    100.idx
-8.2G    101.dat
-6.9M    101.idx
-8.2G    102.dat
-6.8M    102.idx
-8.2G    103.dat
-6.8M    103.idx
-8.2G    104.dat
-6.8M    104.idx
-8.2G    105.dat
-6.9M    105.idx
-8.2G    106.dat
-6.9M    106.idx
-8.2G    107.dat
-6.9M    107.idx
-32G     10.dat
-25M     10.idx
-32G     11.dat
-25M     11.idx
-32G     12.dat
-25M     12.idx
-32G     13.dat
-25M     13.idx
-32G     14.dat
-25M     14.idx
-32G     15.dat
-25M     15.idx
-32G     16.dat
-25M     16.idx
-32G     17.dat
-25M     17.idx
-32G     18.dat
-25M     18.idx
-32G     19.dat
-25M     19.idx
-32G     1.dat
-26M     1.idx
-32G     20.dat
-25M     20.idx
-32G     21.dat
-25M     21.idx
-32G     22.dat
-25M     22.idx
-32G     23.dat
-25M     23.idx
-32G     24.dat
-25M     24.idx
-32G     25.dat
-25M     25.idx
-32G     26.dat
-25M     26.idx
-32G     27.dat
-25M     27.idx
-32G     28.dat
-25M     28.idx
-32G     29.dat
-25M     29.idx
-32G     2.dat
-26M     2.idx
-32G     30.dat
-25M     30.idx
-32G     31.dat
-25M     31.idx
-32G     32.dat
-25M     32.idx
-32G     33.dat
-25M     33.idx
-32G     34.dat
-25M     34.idx
-32G     35.dat
-25M     35.idx
-32G     36.dat
-25M     36.idx
-32G     37.dat
-25M     37.idx
-32G     38.dat
-25M     38.idx
-32G     39.dat
-25M     39.idx
-32G     3.dat
-26M     3.idx
-32G     40.dat
-25M     40.idx
-32G     41.dat
-25M     41.idx
-32G     42.dat
-25M     42.idx
-32G     43.dat
-25M     43.idx
-32G     44.dat
-25M     44.idx
-32G     45.dat
-25M     45.idx
-32G     46.dat
-25M     46.idx
-32G     47.dat
-25M     47.idx
-32G     48.dat
-25M     48.idx
-32G     49.dat
-25M     49.idx
-32G     4.dat
-26M     4.idx
-32G     50.dat
-25M     50.idx
-32G     51.dat
-25M     51.idx
-32G     52.dat
-25M     52.idx
-32G     53.dat
-25M     53.idx
-32G     54.dat
-25M     54.idx
-32G     55.dat
-25M     55.idx
-32G     56.dat
-25M     56.idx
-32G     57.dat
-25M     57.idx
-32G     58.dat
-25M     58.idx
-32G     59.dat
-25M     59.idx
-32G     5.dat
-26M     5.idx
-32G     60.dat
-25M     60.idx
-32G     61.dat
-25M     61.idx
-32G     62.dat
-25M     62.idx
-32G     63.dat
-25M     63.idx
-32G     64.dat
-25M     64.idx
-8.2G    65.dat
-6.9M    65.idx
-8.2G    66.dat
-6.9M    66.idx
-8.2G    67.dat
-6.9M    67.idx
-8.2G    68.dat
-6.8M    68.idx
-8.2G    69.dat
-6.9M    69.idx
-32G     6.dat
-25M     6.idx
-8.2G    70.dat
-6.8M    70.idx
-8.2G    71.dat
-6.9M    71.idx
-8.2G    72.dat
-6.9M    72.idx
-8.2G    73.dat
-6.9M    73.idx
-8.2G    74.dat
-6.9M    74.idx
-8.2G    75.dat
-6.9M    75.idx
-8.1G    76.dat
-6.8M    76.idx
-8.2G    77.dat
-6.8M    77.idx
-8.2G    78.dat
-6.8M    78.idx
-8.1G    79.dat
-6.8M    79.idx
-32G     7.dat
-25M     7.idx
-8.2G    80.dat
-6.8M    80.idx
-8.2G    81.dat
-6.9M    81.idx
-8.2G    82.dat
-6.9M    82.idx
-8.2G    83.dat
-6.9M    83.idx
-8.2G    84.dat
-6.9M    84.idx
-8.2G    85.dat
-6.8M    85.idx
-8.2G    86.dat
-6.9M    86.idx
-8.2G    87.dat
-6.9M    87.idx
-8.2G    88.dat
-6.9M    88.idx
-8.2G    89.dat
-6.8M    89.idx
-32G     8.dat
-25M     8.idx
-8.2G    90.dat
-6.9M    90.idx
-8.1G    91.dat
-6.8M    91.idx
-8.1G    92.dat
-6.8M    92.idx
-8.1G    93.dat
-6.8M    93.idx
-8.2G    94.dat
-6.9M    94.idx
-8.2G    95.dat
-6.9M    95.idx
-8.2G    96.dat
-6.9M    96.idx
-8.2G    97.dat
-6.9M    97.idx
-8.2G    98.dat
-6.9M    98.idx
-8.2G    99.dat
-6.9M    99.idx
-32G     9.dat
-25M     9.idx
-4.0K    directory.seq
-
-You can see the volume 1-64 is now full.
-
-5. more log
-
-see logs.zip
-
-In messages you can see these lines: (Line 51095)
-
-Sep 26 06:14:31 wedb-01 kernel: auditd: page allocation failure. order:0, mode:0x20
-Sep 26 06:14:31 wedb-01 kernel: Pid: 1009, comm: auditd Not tainted 2.6.32-220.el6.x86_64 #1
-Sep 26 06:14:31 wedb-01 kernel: Call Trace:
-
-After those lines , the system deny any new network connect request
-
-6. /dir/status
-
-{"Machines":[{"Server":{"Url":"127.0.0.1:9335","PublicUrl":"x.y.z:9335"},"Volumes":[{"Id":106,"Size":8728909632},{"Id":66,"Size":8729852744},{"Id":90,"Size":8747834896},{"Id":103,"Size":8718106024},{"Id":87,"Size":8732133512},{"Id":96,"Size":8737251904},{"Id":80,"Size":8704130712},{"Id":77,"Size":8717989496},{"Id":70,"Size":8731474744},{"Id":94,"Size":8758656144},{"Id":107,"Size":8729599232},{"Id":67,"Size":8736848088},{"Id":91,"Size":8665847760},{"Id":100,"Size":8703272552},{"Id":84,"Size":8745121528},{"Id":97,"Size":8713031744},{"Id":81,"Size":8726088872},{"Id":74,"Size":8738588152},{"Id":71,"Size":8729349920},{"Id":95,"Size":8741526896},{"Id":104,"Size":8699374736},{"Id":88,"Size":8740362880},{"Id":101,"Size":8711832992},{"Id":85,"Size":8723479552},{"Id":78,"Size":8700345400},{"Id":75,"Size":8727796912},{"Id":68,"Size":8698607440},{"Id":92,"Size":8682683056},{"Id":105,"Size":8741226152},{"Id":65,"Size":8725365752},{"Id":89,"Size":8703062600},{"Id":98,"Size":8742331560},{"Id":82,"Size":8762554952},{"Id":79,"Size":8696300376},{"Id":72,"Size":8708217304},{"Id":69,"Size":8740268144},{"Id":93,"Size":8685060320},{"Id":102,"Size":8708695352},{"Id":86,"Size":8783247776},{"Id":99,"Size":8753463608},{"Id":83,"Size":8725963952},{"Id":76,"Size":8694693536},{"Id":73,"Size":8733560832}]},{"Server":{"Url":"127.0.0.1:9334","PublicUrl":"x.y.z:9334"},"Volumes":[{"Id":34,"Size":33415706800},{"Id":58,"Size":33569224784},{"Id":18,"Size":33474649968},{"Id":55,"Size":33542422680},{"Id":15,"Size":33517247576},{"Id":48,"Size":33574860328},{"Id":8,"Size":33511257144},{"Id":45,"Size":33463948408},{"Id":5,"Size":34317702920},{"Id":29,"Size":33465695776},{"Id":38,"Size":33553119624},{"Id":62,"Size":33448316736},{"Id":22,"Size":33566586296},{"Id":35,"Size":33493733728},{"Id":59,"Size":33498554904},{"Id":19,"Size":33493313784},{"Id":52,"Size":33552978448},{"Id":12,"Size":33505183752},{"Id":49,"Size":33603029896},{"Id":9,"Size":33515778064},{"Id":42,"Size":33500402248},{"Id":2,"Size":34223232992},{"Id":26,"Size":33526519600},{"Id":39,"Size":33580414336},{"Id":63,"Size":33476332456},{"Id":23,"Size":33543872592},{"Id":32,"Size":33515290168},{"Id":56,"Size":33499171184},{"Id":16,"Size":33556591168},{"Id":64,"Size":33495148616},{"Id":53,"Size":33467738560},{"Id":13,"Size":33596873960},{"Id":46,"Size":33508120448},{"Id":6,"Size":33417470256},{"Id":30,"Size":33532933992},{"Id":43,"Size":33591802008},{"Id":3,"Size":34270682080},{"Id":27,"Size":33525736944},{"Id":36,"Size":33443597824},{"Id":60,"Size":33427931336},{"Id":20,"Size":33499083096},{"Id":33,"Size":33531396280},{"Id":57,"Size":33578015104},{"Id":17,"Size":33510525480},{"Id":50,"Size":33503123704},{"Id":10,"Size":33502391608},{"Id":47,"Size":33521868568},{"Id":7,"Size":33497101664},{"Id":31,"Size":33426905232},{"Id":40,"Size":33472978696},{"Id":0,"Size":34337344304},{"Id":24,"Size":33550157192},{"Id":37,"Size":33477162720},{"Id":61,"Size":33537175080},{"Id":21,"Size":33517192456},{"Id":54,"Size":33480720288},{"Id":14,"Size":33513192896},{"Id":51,"Size":33531336080},{"Id":11,"Size":33562385088},{"Id":44,"Size":33554479104},{"Id":4,"Size":34333127520},{"Id":28,"Size":33510503000},{"Id":41,"Size":33574922928},{"Id":1,"Size":34307181368},{"Id":25,"Size":33542834568}]}],"Writers":[106,66,90,103,87,96,80,77,70,94,107,67,91,100,84,97,81,74,71,95,104,88,101,85,78,75,68,92,105,65,89,98,82,79,72,69,93,102,86,99,83,76,73,34,58,18,55,15,48,8,45,5,29,38,62,22,35,59,19,52,12,49,9,42,2,26,39,63,23,32,56,16,64,53,13,46,6,30,43,3,27,36,60,20,33,57,17,50,10,47,7,31,40,0,24,37,61,21,54,14,51,11,44,4,28,41,1,25],"FileIdSequence":110250000}

note/replication.txt

@@ -1,109 +0,0 @@
-1. each file can choose the replication factor
-2. replication granularity is in volume level
-3. if not enough spaces, we can automatically decrease some volume's the replication factor, especially for cold data
-4. plan to support migrating data to cheaper storage
-5. plan to manual volume placement, access-based volume placement, auction based volume placement
-
-When a new volume server is started, it reports 
-  1. how many volumes it can hold
-  2. current list of existing volumes and each volume's replication type
-Each volume server remembers:
-  1. current volume ids
-  2. replica locations are read from the master
-
-The master assign volume ids based on
-  1. replication factor
-     data center, rack
-  2. concurrent write support
-On master, stores the replication configuration
-{
-  replication:{
-    {type:"00", min_volume_count:3, weight:10},
-    {type:"01", min_volume_count:2, weight:20},
-    {type:"10", min_volume_count:2, weight:20},
-    {type:"11", min_volume_count:3, weight:30},
-    {type:"20", min_volume_count:2, weight:20}
-  },
-  port:9333,
-}
-Or manually via command line
-  1. add volume with specified replication factor
-  2. add volume with specified volume id
-
-
-If duplicated volume ids are reported from different volume servers, 
-the master determines the replication factor of the volume,
-if less than the replication factor, the volume is in readonly mode
-if more than the replication factor, the volume will purge the smallest/oldest volume
-if equal, the volume will function as usual
-
-
-Use cases:
-  on volume server
-    1. weed volume -mserver="xx.xx.xx.xx:9333" -publicUrl="good.com:8080" -dir="/tmp" -volumes=50
-  on weed master
-    1. weed master -port=9333
-      generate a default json configuration file if doesn't exist
-      
-Bootstrap
-  1. at the very beginning, the system has no volumes at all.
-When data node starts:
-  1. each data node send to master its existing volumes and max volume blocks
-  2. master remembers the topology/data_center/rack/data_node/volumes
-     for each replication level, stores
-       volume id ~ data node
-       writable volume ids
-If any "assign" request comes in
-  1. find a writable volume with the right replicationLevel
-  2. if not found, grow the volumes with the right replication level
-  3. return a writable volume to the user
-
-  
-  for data node:
-    0. detect existing volumes DONE
-    1. onStartUp, and periodically, send existing volumes and maxVolumeCount  store.Join(), DONE
-    2. accept command to grow a volume( id + replication level)  DONE
-       /admin/assign_volume?volume=some_id&replicationType=01
-    3. accept setting volumeLocationList  DONE
-       /admin/set_volume_locations_list?volumeLocationsList=[{Vid:xxx,Locations:[loc1,loc2,loc3]}]
-    4. for each write, pass the write to the next location, (Step 2)
-       POST method should accept an index, like ttl, get decremented every hop
-  for master:
-    1. accept data node's report of existing volumes and maxVolumeCount ALREADY EXISTS /dir/join
-    2. periodically refresh for active data nodes, and adjust writable volumes
-    3. send command to grow a volume(id + replication level)  DONE
-    5. accept lookup for volume locations    ALREADY EXISTS /dir/lookup
-    6. read topology/datacenter/rack layout
-
-An algorithm to allocate volumes evenly, but may be inefficient if free volumes are plenty:
-input: replication=xyz
-algorithm:
-ret_dcs = []
-foreach dc that has y+z+1 volumes{
-  ret_racks = []
-  foreach rack with z+1 volumes{
-    ret = select z+1 servers with 1 volume
-  if ret.size()==z+1 {
-    ret_racks.append(ret)
-  }
-  }
-  randomly pick one rack from ret_racks
-  ret += select y racks with 1 volume each
-  if ret.size()==y+z+1{
-    ret_dcs.append(ret)
-  }
-}
-randomly pick one dc from ret_dcs
-ret += select x data centers with 1 volume each
-
-A simple replica placement algorithm, but may fail when free volume slots are not plenty:
-ret := []volumes
-dc = randomly pick 1 data center with y+z+1 volumes
-  rack = randomly pick 1 rack with z+1 volumes
-    ret = ret.append(randomly pick z+1 volumes)
-  ret = ret.append(randomly pick y racks with 1 volume)
-ret = ret.append(randomly pick x data centers with 1 volume)
-
-
-TODO:
-  1. replicate content to the other server if the replication type needs replicas

note/security.txt

@@ -1,51 +0,0 @@
-Design for SeaweedFS security
-
-Design Objectives
-	Security can mean many different things. The original vision is that: if you have one machine lying around
-	somewhere with some disk space, it should be able to join your file system to contribute some disk space and
-	network bandwidth, securely!
-
-	To achieve this purpose, the security should be able to:
-	1. Secure the inter-server communication. Only real cluster servers can join and communicate.
-	2. allow clients to securely write to volume servers
-
-Non Objective
-	Multi-tenant support. Avoid filers or clients cross-updating files.
-	User specific access control.
-
-Design Architect
-	master, and volume servers all talk securely via 2-way SSL for admin operations.
-	upon joining, master gives its secret key to volume servers.
-	filer or clients talk to master to get secret key, and use the key to generate JWT to write on volume server.
-	A side benefit:
-		a time limited read feature?
-	4. volume server needs to expose https ports
-
-HTTP Connections
-	clear http
-		filer~>master, need to get a JWT from master
-		filer~>volume
-	2-way https
-		master~ssl~>volume
-		volume~ssl~>master
-
-file uploading:
-	when volume server starts, it asks master for the secret key to decode JWT
-	when filer/clients wants to upload, master generate a JWT
-		filer~>volume(public port)
-		master~>volume(public port)
-
-Currently, volume server has 2 ip addresses: ip and publicUrl.
-	The ip is for admin purpose, and master talk to volume server this way.
-	The publicUrl is for clients to access the server, via http GET/POST/DELETE etc.
-		The write operations are secured by JWT.
-	clients talk to master also via https? possible. Decide on this later.
-
-Dev plan:
-	1. volume server separate admin from public GET/POST/DELETE handlers
-		The step 1 may be good enough for most use cases.
-
-	If 2-way ssl are still needed
-	2. volume server add ssl support
-	3. https connections to operate on volume servers
-

note/weedfs.txt

@@ -1,46 +0,0 @@
-How to submit a content
-1. Find physical volumes
-1.c Create a hash value
-1.d find a write logic volume id, and return [logic volume id, {physical volume ids}]
-2. submit to physical volumes
-2.c
-  generate the cookie
-  generate a unique id as key
-  choose the right altKey
-  send bytes to physical volumes
-2.s each
-  save bytes
-  store map[key uint64, altKey uint32]<offset, size>
-    for updated entry, set old entry's offset to zero
-3.c
-  wait for all physical volumes to finish
-  store the /<logic volume id>/<key>_<cookie>_<altKey>.<ext>
-
-How to retrieve a content
-1.c
-  send logic volume id
-1.d
-  find least busy volume's id
-2.c
-  send URI /<physical volume id>/<key>_<cookie>_<altKey>.<ext>
-
-
-How to submit a content
-1. send bytes to SeaweedFS, got <volume id, key uint64, cookie code>
-   store <key uint64, volume id uint32, cookie code uint32, ext>, and other information
-
-To read a content
-2. use logic volume id to lookup a <machine id>
-   render url as /<machine id>/<volume id>/<key>/<cookie>.ext
-
-The directory server
-0.init
-  load and collect <logic volume id, machine ids> mapping
-1.on submit content
-  find a free logic volume id, start sending content to 3 machines
-  if all of them finishes, return <logic volume id, key, cookie code>
-2.on read content
-  based on logic volume id, pick a machine with less load,
-    return <machine id>
-
-

other/java/client/pom.xml

@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-client</artifactId>
+    <version>1.0.5</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
+
+    <properties>
+        <protobuf.version>3.5.1</protobuf.version>
+        <grpc.version>1.16.1</grpc.version>
+        <guava.version>26.0-jre</guava.version>
+    </properties>
+
+    <dependencies>
+        <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>${protobuf.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-netty-shaded</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-protobuf</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-stub</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+    </dependencies>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
+    <build>
+        <extensions>
+            <extension>
+                <groupId>kr.motd.maven</groupId>
+                <artifactId>os-maven-plugin</artifactId>
+                <version>1.5.0.Final</version>
+            </extension>
+        </extensions>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>7</source>
+                    <target>7</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.xolstice.maven.plugins</groupId>
+                <artifactId>protobuf-maven-plugin</artifactId>
+                <version>0.5.1</version>
+                <configuration>
+                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}-1:exe:${os.detected.classifier}</protocArtifact>
+                    <pluginId>grpc-java</pluginId>
+                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>compile-custom</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.5</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.7</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -0,0 +1,225 @@
+package seaweedfs.client;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class FilerClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
+
+    private FilerGrpcClient filerGrpcClient;
+
+    public FilerClient(String host, int grpcPort) {
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+    }
+
+    public FilerClient(FilerGrpcClient filerGrpcClient) {
+        this.filerGrpcClient = filerGrpcClient;
+    }
+
+    public boolean mkdirs(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return mkdirs(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean mkdirs(String path, int mode, String userName, String[] groupNames) {
+        return mkdirs(path, mode, 0, 0, userName, groupNames);
+    }
+
+    public boolean mkdirs(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        if ("/".equals(path)) {
+            return true;
+        }
+
+        mkdirs(parent, mode, uid, gid, userName, groupNames);
+
+        FilerProto.Entry existingEntry = lookupEntry(parent, name);
+
+        if (existingEntry != null) {
+            return true;
+        }
+
+        return createEntry(
+            parent,
+            newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
+        );
+
+    }
+
+    public boolean rm(String path, boolean isRecursive) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        return deleteEntry(
+            parent,
+            name,
+            true,
+            isRecursive);
+    }
+
+    public boolean touch(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return touch(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        FilerProto.Entry entry = lookupEntry(parent, name);
+        if (entry == null) {
+            return createEntry(
+                parent,
+                newFileEntry(name, mode, uid, gid, userName, groupNames).build()
+            );
+        }
+        long now = System.currentTimeMillis() / 1000L;
+        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
+            .setMtime(now)
+            .setUid(uid)
+            .setGid(gid)
+            .setUserName(userName)
+            .clearGroupName()
+            .addAllGroupName(Arrays.asList(groupNames));
+        return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
+    }
+
+    public FilerProto.Entry.Builder newDirectoryEntry(String name, int mode,
+                                                      int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+            .setName(name)
+            .setIsDirectory(true)
+            .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                .setMtime(now)
+                .setCrtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setFileMode(mode | 1 << 31)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public FilerProto.Entry.Builder newFileEntry(String name, int mode,
+                                                 int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+            .setName(name)
+            .setIsDirectory(false)
+            .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                .setMtime(now)
+                .setCrtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setFileMode(mode)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public List<FilerProto.Entry> listEntries(String path) {
+        List<FilerProto.Entry> results = new ArrayList<FilerProto.Entry>();
+        String lastFileName = "";
+        for (int limit = Integer.MAX_VALUE; limit > 0; ) {
+            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024);
+            if (t == null) {
+                break;
+            }
+            int nSize = t.size();
+            if (nSize > 0) {
+                limit -= nSize;
+                lastFileName = t.get(nSize - 1).getName();
+            }
+            results.addAll(t);
+            if (t.size() < 1024) {
+                break;
+            }
+        }
+        return results;
+    }
+
+    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
+        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+            .setDirectory(path)
+            .setPrefix(entryPrefix)
+            .setStartFromFileName(lastEntryName)
+            .setLimit(limit)
+            .build()).getEntriesList();
+    }
+
+    public FilerProto.Entry lookupEntry(String directory, String entryName) {
+        try {
+            return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+                FilerProto.LookupDirectoryEntryRequest.newBuilder()
+                    .setDirectory(directory)
+                    .setName(entryName)
+                    .build()).getEntry();
+        } catch (Exception e) {
+            LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
+            return null;
+        }
+    }
+
+
+    public boolean createEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setEntry(entry)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean updateEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setEntry(entry)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive) {
+        try {
+            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setName(entryName)
+                .setIsDeleteData(isDeleteFileChunk)
+                .setIsRecursive(isRecursive)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("deleteEntry {}/{}: {}", parent, entryName, e);
+            return false;
+        }
+        return true;
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -0,0 +1,45 @@
+package seaweedfs.client;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Logger;
+
+public class FilerGrpcClient {
+
+    private static final Logger logger = Logger.getLogger(FilerGrpcClient.class.getName());
+
+    private final ManagedChannel channel;
+    private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
+    private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
+    private final SeaweedFilerGrpc.SeaweedFilerFutureStub futureStub;
+
+
+    public FilerGrpcClient(String host, int grpcPort) {
+        this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
+    }
+
+    public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
+        channel = channelBuilder.build();
+        blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
+        asyncStub = SeaweedFilerGrpc.newStub(channel);
+        futureStub = SeaweedFilerGrpc.newFutureStub(channel);
+    }
+
+    public void shutdown() throws InterruptedException {
+        channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
+    }
+
+    public SeaweedFilerGrpc.SeaweedFilerBlockingStub getBlockingStub() {
+        return blockingStub;
+    }
+
+    public SeaweedFilerGrpc.SeaweedFilerStub getAsyncStub() {
+        return asyncStub;
+    }
+
+    public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() {
+        return futureStub;
+    }
+}

other/java/client/src/main/proto/filer.proto

@@ -0,0 +1,178 @@
+syntax = "proto3";
+
+package filer_pb;
+
+option java_package = "seaweedfs.client";
+option java_outer_classname = "FilerProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedFiler {
+
+    rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) {
+    }
+
+    rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) {
+    }
+
+    rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
+    }
+
+    rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
+    }
+
+    rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
+    }
+
+    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
+    }
+
+    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+    }
+
+    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
+    }
+
+    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+
+message LookupDirectoryEntryRequest {
+    string directory = 1;
+    string name = 2;
+}
+
+message LookupDirectoryEntryResponse {
+    Entry entry = 1;
+}
+
+message ListEntriesRequest {
+    string directory = 1;
+    string prefix = 2;
+    string startFromFileName = 3;
+    bool inclusiveStartFrom = 4;
+    uint32 limit = 5;
+}
+
+message ListEntriesResponse {
+    repeated Entry entries = 1;
+}
+
+message Entry {
+    string name = 1;
+    bool is_directory = 2;
+    repeated FileChunk chunks = 3;
+    FuseAttributes attributes = 4;
+    map<string, bytes> extended = 5;
+}
+
+message EventNotification {
+    Entry old_entry = 1;
+    Entry new_entry = 2;
+    bool delete_chunks = 3;
+}
+
+message FileChunk {
+    string file_id = 1;
+    int64 offset = 2;
+    uint64 size = 3;
+    int64 mtime = 4;
+    string e_tag = 5;
+    string source_file_id = 6;
+}
+
+message FuseAttributes {
+    uint64 file_size = 1;
+    int64 mtime = 2; // unix time in seconds
+    uint32 file_mode = 3;
+    uint32 uid = 4;
+    uint32 gid = 5;
+    int64 crtime = 6; // unix time in seconds
+    string mime = 7;
+    string replication = 8;
+    string collection = 9;
+    int32 ttl_sec = 10;
+    string user_name = 11; // for hdfs
+    repeated string group_name = 12; // for hdfs
+}
+
+message CreateEntryRequest {
+    string directory = 1;
+    Entry entry = 2;
+}
+
+message CreateEntryResponse {
+}
+
+message UpdateEntryRequest {
+    string directory = 1;
+    Entry entry = 2;
+}
+message UpdateEntryResponse {
+}
+
+message DeleteEntryRequest {
+    string directory = 1;
+    string name = 2;
+    // bool is_directory = 3;
+    bool is_delete_data = 4;
+    bool is_recursive = 5;
+}
+
+message DeleteEntryResponse {
+}
+
+message AssignVolumeRequest {
+    int32 count = 1;
+    string collection = 2;
+    string replication = 3;
+    int32 ttl_sec = 4;
+    string data_center = 5;
+}
+
+message AssignVolumeResponse {
+    string file_id = 1;
+    string url = 2;
+    string public_url = 3;
+    int32 count = 4;
+}
+
+message LookupVolumeRequest {
+    repeated string volume_ids = 1;
+}
+
+message Locations {
+    repeated Location locations = 1;
+}
+
+message Location {
+    string url = 1;
+    string public_url = 2;
+}
+message LookupVolumeResponse {
+    map<string, Locations> locations_map = 1;
+}
+
+message DeleteCollectionRequest {
+    string collection = 1;
+}
+
+message DeleteCollectionResponse {
+}
+
+message StatisticsRequest {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+}
+message StatisticsResponse {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+    uint64 total_size = 4;
+    uint64 used_size = 5;
+    uint64 file_count = 6;
+}

other/java/hdfs/pom.xml

@@ -0,0 +1,170 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <properties>
+        <seaweedfs.client.version>1.0.5</seaweedfs.client.version>
+        <hadoop.version>3.1.1</hadoop.version>
+    </properties>
+
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-hadoop-client</artifactId>
+    <version>${seaweedfs.client.version}</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>7</source>
+                    <target>7</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                        <exclude>org/slf4j/**</exclude>
+                                        <exclude>META-INF/maven/org.slf4j/**</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google</pattern>
+                                    <shadedPattern>shaded.com.google</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>io.grpc.internal</pattern>
+                                    <shadedPattern>shaded.io.grpc.internal</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons</pattern>
+                                    <shadedPattern>shaded.org.apache.commons</shadedPattern>
+                                    <excludes>
+                                        <exclude>org.apache.hadoop</exclude>
+                                        <exclude>org.apache.log4j</exclude>
+                                    </excludes>
+                                </relocation>
+                            </relocations>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.5</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.7</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-client</artifactId>
+            <version>${seaweedfs.client.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.6</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBuffer.java

@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package seaweed.hdfs;
+
+import java.util.concurrent.CountDownLatch;
+
+class ReadBuffer {
+
+  private SeaweedInputStream stream;
+  private long offset;                   // offset within the file for the buffer
+  private int length;                    // actual length, set after the buffer is filles
+  private int requestedLength;           // requested length of the read
+  private byte[] buffer;                 // the buffer itself
+  private int bufferindex = -1;          // index in the buffers array in Buffer manager
+  private ReadBufferStatus status;             // status of the buffer
+  private CountDownLatch latch = null;   // signaled when the buffer is done reading, so any client
+  // waiting on this buffer gets unblocked
+
+  // fields to help with eviction logic
+  private long timeStamp = 0;  // tick at which buffer became available to read
+  private boolean isFirstByteConsumed = false;
+  private boolean isLastByteConsumed = false;
+  private boolean isAnyByteConsumed = false;
+
+  public SeaweedInputStream getStream() {
+    return stream;
+  }
+
+  public void setStream(SeaweedInputStream stream) {
+    this.stream = stream;
+  }
+
+  public long getOffset() {
+    return offset;
+  }
+
+  public void setOffset(long offset) {
+    this.offset = offset;
+  }
+
+  public int getLength() {
+    return length;
+  }
+
+  public void setLength(int length) {
+    this.length = length;
+  }
+
+  public int getRequestedLength() {
+    return requestedLength;
+  }
+
+  public void setRequestedLength(int requestedLength) {
+    this.requestedLength = requestedLength;
+  }
+
+  public byte[] getBuffer() {
+    return buffer;
+  }
+
+  public void setBuffer(byte[] buffer) {
+    this.buffer = buffer;
+  }
+
+  public int getBufferindex() {
+    return bufferindex;
+  }
+
+  public void setBufferindex(int bufferindex) {
+    this.bufferindex = bufferindex;
+  }
+
+  public ReadBufferStatus getStatus() {
+    return status;
+  }
+
+  public void setStatus(ReadBufferStatus status) {
+    this.status = status;
+  }
+
+  public CountDownLatch getLatch() {
+    return latch;
+  }
+
+  public void setLatch(CountDownLatch latch) {
+    this.latch = latch;
+  }
+
+  public long getTimeStamp() {
+    return timeStamp;
+  }
+
+  public void setTimeStamp(long timeStamp) {
+    this.timeStamp = timeStamp;
+  }
+
+  public boolean isFirstByteConsumed() {
+    return isFirstByteConsumed;
+  }
+
+  public void setFirstByteConsumed(boolean isFirstByteConsumed) {
+    this.isFirstByteConsumed = isFirstByteConsumed;
+  }
+
+  public boolean isLastByteConsumed() {
+    return isLastByteConsumed;
+  }
+
+  public void setLastByteConsumed(boolean isLastByteConsumed) {
+    this.isLastByteConsumed = isLastByteConsumed;
+  }
+
+  public boolean isAnyByteConsumed() {
+    return isAnyByteConsumed;
+  }
+
+  public void setAnyByteConsumed(boolean isAnyByteConsumed) {
+    this.isAnyByteConsumed = isAnyByteConsumed;
+  }
+
+}

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBufferManager.java

@@ -0,0 +1,394 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package seaweed.hdfs;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.Stack;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * The Read Buffer Manager for Rest AbfsClient.
+ */
+final class ReadBufferManager {
+  private static final Logger LOGGER = LoggerFactory.getLogger(ReadBufferManager.class);
+
+  private static final int NUM_BUFFERS = 16;
+  private static final int BLOCK_SIZE = 4 * 1024 * 1024;
+  private static final int NUM_THREADS = 8;
+  private static final int THRESHOLD_AGE_MILLISECONDS = 3000; // have to see if 3 seconds is a good threshold
+
+  private Thread[] threads = new Thread[NUM_THREADS];
+  private byte[][] buffers;    // array of byte[] buffers, to hold the data that is read
+  private Stack<Integer> freeList = new Stack<>();   // indices in buffers[] array that are available
+
+  private Queue<ReadBuffer> readAheadQueue = new LinkedList<>(); // queue of requests that are not picked up by any worker thread yet
+  private LinkedList<ReadBuffer> inProgressList = new LinkedList<>(); // requests being processed by worker threads
+  private LinkedList<ReadBuffer> completedReadList = new LinkedList<>(); // buffers available for reading
+  private static final ReadBufferManager BUFFER_MANAGER; // singleton, initialized in static initialization block
+
+  static {
+    BUFFER_MANAGER = new ReadBufferManager();
+    BUFFER_MANAGER.init();
+  }
+
+  static ReadBufferManager getBufferManager() {
+    return BUFFER_MANAGER;
+  }
+
+  private void init() {
+    buffers = new byte[NUM_BUFFERS][];
+    for (int i = 0; i < NUM_BUFFERS; i++) {
+      buffers[i] = new byte[BLOCK_SIZE];  // same buffers are reused. The byte array never goes back to GC
+      freeList.add(i);
+    }
+    for (int i = 0; i < NUM_THREADS; i++) {
+      Thread t = new Thread(new ReadBufferWorker(i));
+      t.setDaemon(true);
+      threads[i] = t;
+      t.setName("SeaweedFS-prefetch-" + i);
+      t.start();
+    }
+    ReadBufferWorker.UNLEASH_WORKERS.countDown();
+  }
+
+  // hide instance constructor
+  private ReadBufferManager() {
+  }
+
+
+  /*
+   *
+   *  SeaweedInputStream-facing methods
+   *
+   */
+
+
+  /**
+   * {@link SeaweedInputStream} calls this method to queue read-aheads.
+   *
+   * @param stream          The {@link SeaweedInputStream} for which to do the read-ahead
+   * @param requestedOffset The offset in the file which shoukd be read
+   * @param requestedLength The length to read
+   */
+  void queueReadAhead(final SeaweedInputStream stream, final long requestedOffset, final int requestedLength) {
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("Start Queueing readAhead for {} offset {} length {}",
+          stream.getPath(), requestedOffset, requestedLength);
+    }
+    ReadBuffer buffer;
+    synchronized (this) {
+      if (isAlreadyQueued(stream, requestedOffset)) {
+        return; // already queued, do not queue again
+      }
+      if (freeList.isEmpty() && !tryEvict()) {
+        return; // no buffers available, cannot queue anything
+      }
+
+      buffer = new ReadBuffer();
+      buffer.setStream(stream);
+      buffer.setOffset(requestedOffset);
+      buffer.setLength(0);
+      buffer.setRequestedLength(requestedLength);
+      buffer.setStatus(ReadBufferStatus.NOT_AVAILABLE);
+      buffer.setLatch(new CountDownLatch(1));
+
+      Integer bufferIndex = freeList.pop();  // will return a value, since we have checked size > 0 already
+
+      buffer.setBuffer(buffers[bufferIndex]);
+      buffer.setBufferindex(bufferIndex);
+      readAheadQueue.add(buffer);
+      notifyAll();
+    }
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("Done q-ing readAhead for file {} offset {} buffer idx {}",
+          stream.getPath(), requestedOffset, buffer.getBufferindex());
+    }
+  }
+
+
+  /**
+   * {@link SeaweedInputStream} calls this method read any bytes already available in a buffer (thereby saving a
+   * remote read). This returns the bytes if the data already exists in buffer. If there is a buffer that is reading
+   * the requested offset, then this method blocks until that read completes. If the data is queued in a read-ahead
+   * but not picked up by a worker thread yet, then it cancels that read-ahead and reports cache miss. This is because
+   * depending on worker thread availability, the read-ahead may take a while - the calling thread can do it's own
+   * read to get the data faster (copmared to the read waiting in queue for an indeterminate amount of time).
+   *
+   * @param stream   the file to read bytes for
+   * @param position the offset in the file to do a read for
+   * @param length   the length to read
+   * @param buffer   the buffer to read data into. Note that the buffer will be written into from offset 0.
+   * @return the number of bytes read
+   */
+  int getBlock(final SeaweedInputStream stream, final long position, final int length, final byte[] buffer) {
+    // not synchronized, so have to be careful with locking
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("getBlock for file {}  position {}  thread {}",
+          stream.getPath(), position, Thread.currentThread().getName());
+    }
+
+    waitForProcess(stream, position);
+
+    int bytesRead = 0;
+    synchronized (this) {
+      bytesRead = getBlockFromCompletedQueue(stream, position, length, buffer);
+    }
+    if (bytesRead > 0) {
+      if (LOGGER.isTraceEnabled()) {
+        LOGGER.trace("Done read from Cache for {} position {} length {}",
+            stream.getPath(), position, bytesRead);
+      }
+      return bytesRead;
+    }
+
+    // otherwise, just say we got nothing - calling thread can do its own read
+    return 0;
+  }
+
+  /*
+   *
+   *  Internal methods
+   *
+   */
+
+  private void waitForProcess(final SeaweedInputStream stream, final long position) {
+    ReadBuffer readBuf;
+    synchronized (this) {
+      clearFromReadAheadQueue(stream, position);
+      readBuf = getFromList(inProgressList, stream, position);
+    }
+    if (readBuf != null) {         // if in in-progress queue, then block for it
+      try {
+        if (LOGGER.isTraceEnabled()) {
+          LOGGER.trace("got a relevant read buffer for file {} offset {} buffer idx {}",
+              stream.getPath(), readBuf.getOffset(), readBuf.getBufferindex());
+        }
+        readBuf.getLatch().await();  // blocking wait on the caller stream's thread
+        // Note on correctness: readBuf gets out of inProgressList only in 1 place: after worker thread
+        // is done processing it (in doneReading). There, the latch is set after removing the buffer from
+        // inProgressList. So this latch is safe to be outside the synchronized block.
+        // Putting it in synchronized would result in a deadlock, since this thread would be holding the lock
+        // while waiting, so no one will be able to  change any state. If this becomes more complex in the future,
+        // then the latch cane be removed and replaced with wait/notify whenever inProgressList is touched.
+      } catch (InterruptedException ex) {
+        Thread.currentThread().interrupt();
+      }
+      if (LOGGER.isTraceEnabled()) {
+        LOGGER.trace("latch done for file {} buffer idx {} length {}",
+            stream.getPath(), readBuf.getBufferindex(), readBuf.getLength());
+      }
+    }
+  }
+
+  /**
+   * If any buffer in the completedlist can be reclaimed then reclaim it and return the buffer to free list.
+   * The objective is to find just one buffer - there is no advantage to evicting more than one.
+   *
+   * @return whether the eviction succeeeded - i.e., were we able to free up one buffer
+   */
+  private synchronized boolean tryEvict() {
+    ReadBuffer nodeToEvict = null;
+    if (completedReadList.size() <= 0) {
+      return false;  // there are no evict-able buffers
+    }
+
+    // first, try buffers where all bytes have been consumed (approximated as first and last bytes consumed)
+    for (ReadBuffer buf : completedReadList) {
+      if (buf.isFirstByteConsumed() && buf.isLastByteConsumed()) {
+        nodeToEvict = buf;
+        break;
+      }
+    }
+    if (nodeToEvict != null) {
+      return evict(nodeToEvict);
+    }
+
+    // next, try buffers where any bytes have been consumed (may be a bad idea? have to experiment and see)
+    for (ReadBuffer buf : completedReadList) {
+      if (buf.isAnyByteConsumed()) {
+        nodeToEvict = buf;
+        break;
+      }
+    }
+
+    if (nodeToEvict != null) {
+      return evict(nodeToEvict);
+    }
+
+    // next, try any old nodes that have not been consumed
+    long earliestBirthday = Long.MAX_VALUE;
+    for (ReadBuffer buf : completedReadList) {
+      if (buf.getTimeStamp() < earliestBirthday) {
+        nodeToEvict = buf;
+        earliestBirthday = buf.getTimeStamp();
+      }
+    }
+    if ((currentTimeMillis() - earliestBirthday > THRESHOLD_AGE_MILLISECONDS) && (nodeToEvict != null)) {
+      return evict(nodeToEvict);
+    }
+
+    // nothing can be evicted
+    return false;
+  }
+
+  private boolean evict(final ReadBuffer buf) {
+    freeList.push(buf.getBufferindex());
+    completedReadList.remove(buf);
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("Evicting buffer idx {}; was used for file {} offset {} length {}",
+          buf.getBufferindex(), buf.getStream().getPath(), buf.getOffset(), buf.getLength());
+    }
+    return true;
+  }
+
+  private boolean isAlreadyQueued(final SeaweedInputStream stream, final long requestedOffset) {
+    // returns true if any part of the buffer is already queued
+    return (isInList(readAheadQueue, stream, requestedOffset)
+        || isInList(inProgressList, stream, requestedOffset)
+        || isInList(completedReadList, stream, requestedOffset));
+  }
+
+  private boolean isInList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
+    return (getFromList(list, stream, requestedOffset) != null);
+  }
+
+  private ReadBuffer getFromList(final Collection<ReadBuffer> list, final SeaweedInputStream stream, final long requestedOffset) {
+    for (ReadBuffer buffer : list) {
+      if (buffer.getStream() == stream) {
+        if (buffer.getStatus() == ReadBufferStatus.AVAILABLE
+            && requestedOffset >= buffer.getOffset()
+            && requestedOffset < buffer.getOffset() + buffer.getLength()) {
+          return buffer;
+        } else if (requestedOffset >= buffer.getOffset()
+            && requestedOffset < buffer.getOffset() + buffer.getRequestedLength()) {
+          return buffer;
+        }
+      }
+    }
+    return null;
+  }
+
+  private void clearFromReadAheadQueue(final SeaweedInputStream stream, final long requestedOffset) {
+    ReadBuffer buffer = getFromList(readAheadQueue, stream, requestedOffset);
+    if (buffer != null) {
+      readAheadQueue.remove(buffer);
+      notifyAll();   // lock is held in calling method
+      freeList.push(buffer.getBufferindex());
+    }
+  }
+
+  private int getBlockFromCompletedQueue(final SeaweedInputStream stream, final long position, final int length,
+                                         final byte[] buffer) {
+    ReadBuffer buf = getFromList(completedReadList, stream, position);
+    if (buf == null || position >= buf.getOffset() + buf.getLength()) {
+      return 0;
+    }
+    int cursor = (int) (position - buf.getOffset());
+    int availableLengthInBuffer = buf.getLength() - cursor;
+    int lengthToCopy = Math.min(length, availableLengthInBuffer);
+    System.arraycopy(buf.getBuffer(), cursor, buffer, 0, lengthToCopy);
+    if (cursor == 0) {
+      buf.setFirstByteConsumed(true);
+    }
+    if (cursor + lengthToCopy == buf.getLength()) {
+      buf.setLastByteConsumed(true);
+    }
+    buf.setAnyByteConsumed(true);
+    return lengthToCopy;
+  }
+
+  /*
+   *
+   *  ReadBufferWorker-thread-facing methods
+   *
+   */
+
+  /**
+   * ReadBufferWorker thread calls this to get the next buffer that it should work on.
+   *
+   * @return {@link ReadBuffer}
+   * @throws InterruptedException if thread is interrupted
+   */
+  ReadBuffer getNextBlockToRead() throws InterruptedException {
+    ReadBuffer buffer = null;
+    synchronized (this) {
+      //buffer = readAheadQueue.take();  // blocking method
+      while (readAheadQueue.size() == 0) {
+        wait();
+      }
+      buffer = readAheadQueue.remove();
+      notifyAll();
+      if (buffer == null) {
+        return null;            // should never happen
+      }
+      buffer.setStatus(ReadBufferStatus.READING_IN_PROGRESS);
+      inProgressList.add(buffer);
+    }
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("ReadBufferWorker picked file {} for offset {}",
+          buffer.getStream().getPath(), buffer.getOffset());
+    }
+    return buffer;
+  }
+
+  /**
+   * ReadBufferWorker thread calls this method to post completion.
+   *
+   * @param buffer            the buffer whose read was completed
+   * @param result            the {@link ReadBufferStatus} after the read operation in the worker thread
+   * @param bytesActuallyRead the number of bytes that the worker thread was actually able to read
+   */
+  void doneReading(final ReadBuffer buffer, final ReadBufferStatus result, final int bytesActuallyRead) {
+    if (LOGGER.isTraceEnabled()) {
+      LOGGER.trace("ReadBufferWorker completed file {} for offset {} bytes {}",
+          buffer.getStream().getPath(),  buffer.getOffset(), bytesActuallyRead);
+    }
+    synchronized (this) {
+      inProgressList.remove(buffer);
+      if (result == ReadBufferStatus.AVAILABLE && bytesActuallyRead > 0) {
+        buffer.setStatus(ReadBufferStatus.AVAILABLE);
+        buffer.setTimeStamp(currentTimeMillis());
+        buffer.setLength(bytesActuallyRead);
+        completedReadList.add(buffer);
+      } else {
+        freeList.push(buffer.getBufferindex());
+        // buffer should go out of scope after the end of the calling method in ReadBufferWorker, and eligible for GC
+      }
+    }
+    //outside the synchronized, since anyone receiving a wake-up from the latch must see safe-published results
+    buffer.getLatch().countDown(); // wake up waiting threads (if any)
+  }
+
+  /**
+   * Similar to System.currentTimeMillis, except implemented with System.nanoTime().
+   * System.currentTimeMillis can go backwards when system clock is changed (e.g., with NTP time synchronization),
+   * making it unsuitable for measuring time intervals. nanotime is strictly monotonically increasing per CPU core.
+   * Note: it is not monotonic across Sockets, and even within a CPU, its only the
+   * more recent parts which share a clock across all cores.
+   *
+   * @return current time in milliseconds
+   */
+  private long currentTimeMillis() {
+    return System.nanoTime() / 1000 / 1000;
+  }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBufferStatus.java

@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package seaweed.hdfs;
+
+/**
+ * The ReadBufferStatus for Rest AbfsClient
+ */
+public enum ReadBufferStatus {
+  NOT_AVAILABLE,  // buffers sitting in readaheadqueue have this stats
+  READING_IN_PROGRESS,  // reading is in progress on this buffer. Buffer should be in inProgressList
+  AVAILABLE,  // data is available in buffer. It should be in completedList
+  READ_FAILED  // read completed, but failed.
+}

other/java/hdfs/src/main/java/seaweed/hdfs/ReadBufferWorker.java

@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package seaweed.hdfs;
+
+import java.util.concurrent.CountDownLatch;
+
+class ReadBufferWorker implements Runnable {
+
+  protected static final CountDownLatch UNLEASH_WORKERS = new CountDownLatch(1);
+  private int id;
+
+  ReadBufferWorker(final int id) {
+    this.id = id;
+  }
+
+  /**
+   * return the ID of ReadBufferWorker.
+   */
+  public int getId() {
+    return this.id;
+  }
+
+  /**
+   * Waits until a buffer becomes available in ReadAheadQueue.
+   * Once a buffer becomes available, reads the file specified in it and then posts results back to buffer manager.
+   * Rinse and repeat. Forever.
+   */
+  public void run() {
+    try {
+      UNLEASH_WORKERS.await();
+    } catch (InterruptedException ex) {
+      Thread.currentThread().interrupt();
+    }
+    ReadBufferManager bufferManager = ReadBufferManager.getBufferManager();
+    ReadBuffer buffer;
+    while (true) {
+      try {
+        buffer = bufferManager.getNextBlockToRead();   // blocks, until a buffer is available for this thread
+      } catch (InterruptedException ex) {
+        Thread.currentThread().interrupt();
+        return;
+      }
+      if (buffer != null) {
+        try {
+          // do the actual read, from the file.
+          int bytesRead = buffer.getStream().readRemote(buffer.getOffset(), buffer.getBuffer(), 0, buffer.getRequestedLength());
+          bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead);  // post result back to ReadBufferManager
+        } catch (Exception ex) {
+          bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0);
+        }
+      }
+    }
+  }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -0,0 +1,594 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Progressable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+
+public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
+
+    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
+    public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
+    public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
+    private static int BUFFER_SIZE = 16 * 1024 * 1024;
+
+    private URI uri;
+    private Path workingDirectory = new Path("/");
+    private SeaweedFileSystemStore seaweedFileSystemStore;
+
+    public URI getUri() {
+        return uri;
+    }
+
+    public String getScheme() {
+        return "seaweedfs";
+    }
+
+    @Override
+    public void initialize(URI uri, Configuration conf) throws IOException { // get
+        super.initialize(uri, conf);
+
+        // get host information from uri (overrides info in conf)
+        String host = uri.getHost();
+        host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
+        if (host == null) {
+            throw new IOException("Invalid host specified");
+        }
+        conf.set(FS_SEAWEED_FILER_HOST, host);
+
+        // get port information from uri, (overrides info in conf)
+        int port = uri.getPort();
+        port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
+        conf.setInt(FS_SEAWEED_FILER_PORT, port);
+
+        conf.setInt(IO_FILE_BUFFER_SIZE_KEY, BUFFER_SIZE);
+
+        setConf(conf);
+        this.uri = uri;
+
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
+    }
+
+    @Override
+    public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+
+        LOG.debug("open path: {} bufferSize:{}", path, bufferSize);
+
+        path = qualify(path);
+
+        try {
+            InputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize);
+            return new FSDataInputStream(inputStream);
+        } catch (Exception ex) {
+            return null;
+        }
+    }
+
+    @Override
+    public FSDataOutputStream create(Path path, FsPermission permission, final boolean overwrite, final int bufferSize,
+                                     final short replication, final long blockSize, final Progressable progress) throws IOException {
+
+        LOG.debug("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize);
+
+        path = qualify(path);
+
+        try {
+            String replicaPlacement = String.format("%03d", replication - 1);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, bufferSize, replicaPlacement);
+            return new FSDataOutputStream(outputStream, statistics);
+        } catch (Exception ex) {
+            return null;
+        }
+    }
+
+    @Override
+    public FSDataOutputStream append(Path path, int bufferSize, Progressable progressable) throws IOException {
+
+        LOG.debug("append path: {} bufferSize:{}", path, bufferSize);
+
+        path = qualify(path);
+        try {
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, bufferSize, "");
+            return new FSDataOutputStream(outputStream, statistics);
+        } catch (Exception ex) {
+            return null;
+        }
+    }
+
+    @Override
+    public boolean rename(Path src, Path dst) {
+
+        LOG.debug("rename path: {} => {}", src, dst);
+
+        if (src.isRoot()) {
+            return false;
+        }
+
+        if (src.equals(dst)) {
+            return true;
+        }
+        FileStatus dstFileStatus = getFileStatus(dst);
+
+        String sourceFileName = src.getName();
+        Path adjustedDst = dst;
+
+        if (dstFileStatus != null) {
+            if (!dstFileStatus.isDirectory()) {
+                return false;
+            }
+            adjustedDst = new Path(dst, sourceFileName);
+        }
+
+        Path qualifiedSrcPath = qualify(src);
+        Path qualifiedDstPath = qualify(adjustedDst);
+
+        seaweedFileSystemStore.rename(qualifiedSrcPath, qualifiedDstPath);
+        return true;
+    }
+
+    @Override
+    public boolean delete(Path path, boolean recursive) {
+
+        LOG.debug("delete path: {} recursive:{}", path, recursive);
+
+        path = qualify(path);
+
+        FileStatus fileStatus = getFileStatus(path);
+
+        if (fileStatus == null) {
+            return true;
+        }
+
+        return seaweedFileSystemStore.deleteEntries(path, fileStatus.isDirectory(), recursive);
+
+    }
+
+    @Override
+    public FileStatus[] listStatus(Path path) throws IOException {
+
+        LOG.debug("listStatus path: {}", path);
+
+        path = qualify(path);
+
+        return seaweedFileSystemStore.listEntries(path);
+    }
+
+    @Override
+    public Path getWorkingDirectory() {
+        return workingDirectory;
+    }
+
+    @Override
+    public void setWorkingDirectory(Path path) {
+        if (path.isAbsolute()) {
+            workingDirectory = path;
+        } else {
+            workingDirectory = new Path(workingDirectory, path);
+        }
+    }
+
+    @Override
+    public boolean mkdirs(Path path, FsPermission fsPermission) throws IOException {
+
+        LOG.debug("mkdirs path: {}", path);
+
+        path = qualify(path);
+
+        FileStatus fileStatus = getFileStatus(path);
+
+        if (fileStatus == null) {
+
+            UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+            return seaweedFileSystemStore.createDirectory(path, currentUser,
+                fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
+                FsPermission.getUMask(getConf()));
+
+        }
+
+        if (fileStatus.isDirectory()) {
+            return true;
+        } else {
+            throw new FileAlreadyExistsException("Path is a file: " + path);
+        }
+    }
+
+    @Override
+    public FileStatus getFileStatus(Path path) {
+
+        LOG.debug("getFileStatus path: {}", path);
+
+        path = qualify(path);
+
+        return seaweedFileSystemStore.getFileStatus(path);
+    }
+
+    /**
+     * Set owner of a path (i.e. a file or a directory).
+     * The parameters owner and group cannot both be null.
+     *
+     * @param path  The path
+     * @param owner If it is null, the original username remains unchanged.
+     * @param group If it is null, the original groupname remains unchanged.
+     */
+    @Override
+    public void setOwner(Path path, final String owner, final String group)
+        throws IOException {
+        LOG.debug("setOwner path: {}", path);
+        path = qualify(path);
+
+        seaweedFileSystemStore.setOwner(path, owner, group);
+    }
+
+
+    /**
+     * Set permission of a path.
+     *
+     * @param path       The path
+     * @param permission Access permission
+     */
+    @Override
+    public void setPermission(Path path, final FsPermission permission) throws IOException {
+        LOG.debug("setPermission path: {}", path);
+
+        if (permission == null) {
+            throw new IllegalArgumentException("The permission can't be null");
+        }
+
+        path = qualify(path);
+
+        seaweedFileSystemStore.setPermission(path, permission);
+    }
+
+    Path qualify(Path path) {
+        return path.makeQualified(uri, workingDirectory);
+    }
+
+    /**
+     * Concat existing files together.
+     * @param trg the path to the target destination.
+     * @param psrcs the paths to the sources to use for the concatenation.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default).
+     */
+    @Override
+    public void concat(final Path trg, final Path [] psrcs) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+            getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    /**
+     * Truncate the file in the indicated path to the indicated size.
+     * <ul>
+     *   <li>Fails if path is a directory.</li>
+     *   <li>Fails if path does not exist.</li>
+     *   <li>Fails if path is not closed.</li>
+     *   <li>Fails if new size is greater than current size.</li>
+     * </ul>
+     * @param f The path to the file to be truncated
+     * @param newLength The size the file is to be truncated to
+     *
+     * @return <code>true</code> if the file has been truncated to the desired
+     * <code>newLength</code> and is immediately available to be reused for
+     * write operations such as <code>append</code>, or
+     * <code>false</code> if a background process of adjusting the length of
+     * the last block has been started, and clients should wait for it to
+     * complete before proceeding with further file updates.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default).
+     */
+    @Override
+    public boolean truncate(Path f, long newLength) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+            getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    @Override
+    public void createSymlink(final Path target, final Path link,
+                              final boolean createParent) throws AccessControlException,
+        FileAlreadyExistsException, FileNotFoundException,
+        ParentNotDirectoryException, UnsupportedFileSystemException,
+        IOException {
+        // Supporting filesystems should override this method
+        throw new UnsupportedOperationException(
+            "Filesystem does not support symlinks!");
+    }
+
+    public boolean supportsSymlinks() {
+        return false;
+    }
+
+    /**
+     * Create a snapshot.
+     * @param path The directory where snapshots will be taken.
+     * @param snapshotName The name of the snapshot
+     * @return the snapshot path.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     */
+    @Override
+    public Path createSnapshot(Path path, String snapshotName)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support createSnapshot");
+    }
+
+    /**
+     * Rename a snapshot.
+     * @param path The directory path where the snapshot was taken
+     * @param snapshotOldName Old name of the snapshot
+     * @param snapshotNewName New name of the snapshot
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void renameSnapshot(Path path, String snapshotOldName,
+                               String snapshotNewName) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support renameSnapshot");
+    }
+
+    /**
+     * Delete a snapshot of a directory.
+     * @param path  The directory that the to-be-deleted snapshot belongs to
+     * @param snapshotName The name of the snapshot
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void deleteSnapshot(Path path, String snapshotName)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support deleteSnapshot");
+    }
+
+    /**
+     * Modifies ACL entries of files and directories.  This method can add new ACL
+     * entries or modify the permissions on existing ACL entries.  All existing
+     * ACL entries that are not specified in this call are retained without
+     * changes.  (Modifications are merged into the current ACL.)
+     *
+     * @param path Path to modify
+     * @param aclSpec List&lt;AclEntry&gt; describing modifications
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support modifyAclEntries");
+    }
+
+    /**
+     * Removes ACL entries from files and directories.  Other ACL entries are
+     * retained.
+     *
+     * @param path Path to modify
+     * @param aclSpec List describing entries to remove
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeAclEntries(Path path, List<AclEntry> aclSpec)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeAclEntries");
+    }
+
+    /**
+     * Removes all default ACL entries from files and directories.
+     *
+     * @param path Path to modify
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeDefaultAcl(Path path)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeDefaultAcl");
+    }
+
+    /**
+     * Removes all but the base ACL entries of files and directories.  The entries
+     * for user, group, and others are retained for compatibility with permission
+     * bits.
+     *
+     * @param path Path to modify
+     * @throws IOException if an ACL could not be removed
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeAcl(Path path)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeAcl");
+    }
+
+    /**
+     * Fully replaces ACL of files and directories, discarding all existing
+     * entries.
+     *
+     * @param path Path to modify
+     * @param aclSpec List describing modifications, which must include entries
+     *   for user, group, and others for compatibility with permission bits.
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support setAcl");
+    }
+
+    /**
+     * Gets the ACL of a file or directory.
+     *
+     * @param path Path to get
+     * @return AclStatus describing the ACL of the file or directory
+     * @throws IOException if an ACL could not be read
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public AclStatus getAclStatus(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getAclStatus");
+    }
+
+    /**
+     * Set an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to modify
+     * @param name xattr name.
+     * @param value xattr value.
+     * @param flag xattr set flag
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void setXAttr(Path path, String name, byte[] value,
+                         EnumSet<XAttrSetFlag> flag) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support setXAttr");
+    }
+
+    /**
+     * Get an xattr name and value for a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attribute
+     * @param name xattr name.
+     * @return byte[] xattr value.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public byte[] getXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getXAttr");
+    }
+
+    /**
+     * Get all of the xattr name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattrs name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @param names XAttr names.
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path, List<String> names)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattr names for a file or directory.
+     * Only those xattr names which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return List{@literal <String>} of the XAttr names of the file or directory
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public List<String> listXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support listXAttrs");
+    }
+
+    /**
+     * Remove an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to remove extended attribute
+     * @param name xattr name
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeXAttr");
+    }
+
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -0,0 +1,296 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerClient;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class SeaweedFileSystemStore {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
+
+    private FilerGrpcClient filerGrpcClient;
+    private FilerClient filerClient;
+
+    public SeaweedFileSystemStore(String host, int port) {
+        int grpcPort = 10000 + port;
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+        filerClient = new FilerClient(filerGrpcClient);
+    }
+
+    public static String getParentDirectory(Path path) {
+        return path.isRoot() ? "/" : path.getParent().toUri().getPath();
+    }
+
+    static int permissionToMode(FsPermission permission, boolean isDirectory) {
+        int p = permission.toShort();
+        if (isDirectory) {
+            p = p | 1 << 31;
+        }
+        return p;
+    }
+
+    public boolean createDirectory(final Path path, UserGroupInformation currentUser,
+                                   final FsPermission permission, final FsPermission umask) {
+
+        LOG.debug("createDirectory path: {} permission: {} umask: {}",
+            path,
+            permission,
+            umask);
+
+        return filerClient.mkdirs(
+            path.toUri().getPath(),
+            permissionToMode(permission, true),
+            currentUser.getUserName(),
+            currentUser.getGroupNames()
+        );
+    }
+
+    public FileStatus[] listEntries(final Path path) {
+        LOG.debug("listEntries path: {}", path);
+
+        List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
+
+        List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
+
+        for (FilerProto.Entry entry : entries) {
+
+            FileStatus fileStatus = doGetFileStatus(new Path(path, entry.getName()), entry);
+
+            fileStatuses.add(fileStatus);
+        }
+        return fileStatuses.toArray(new FileStatus[0]);
+    }
+
+    public FileStatus getFileStatus(final Path path) {
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            return null;
+        }
+        LOG.debug("doGetFileStatus path:{} entry:{}", path, entry);
+
+        FileStatus fileStatus = doGetFileStatus(path, entry);
+        return fileStatus;
+    }
+
+    public boolean deleteEntries(final Path path, boolean isDirectory, boolean recursive) {
+        LOG.debug("deleteEntries path: {} isDirectory {} recursive: {}",
+            path,
+            String.valueOf(isDirectory),
+            String.valueOf(recursive));
+
+        if (path.isRoot()) {
+            return true;
+        }
+
+        if (recursive && isDirectory) {
+            List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
+            for (FilerProto.Entry entry : entries) {
+                deleteEntries(new Path(path, entry.getName()), entry.getIsDirectory(), true);
+            }
+        }
+
+        return filerClient.deleteEntry(getParentDirectory(path), path.getName(), true, recursive);
+    }
+
+    private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) {
+        FilerProto.FuseAttributes attributes = entry.getAttributes();
+        long length = SeaweedRead.totalSize(entry.getChunksList());
+        boolean isDir = entry.getIsDirectory();
+        int block_replication = 1;
+        int blocksize = 512;
+        long modification_time = attributes.getMtime() * 1000; // milliseconds
+        long access_time = 0;
+        FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode());
+        String owner = attributes.getUserName();
+        String group = attributes.getGroupNameCount() > 0 ? attributes.getGroupName(0) : "";
+        return new FileStatus(length, isDir, block_replication, blocksize,
+            modification_time, access_time, permission, owner, group, null, path);
+    }
+
+    private FilerProto.Entry lookupEntry(Path path) {
+
+        return filerClient.lookupEntry(getParentDirectory(path), path.getName());
+
+    }
+
+    public void rename(Path source, Path destination) {
+
+        LOG.debug("rename source: {} destination:{}", source, destination);
+
+        if (source.isRoot()) {
+            return;
+        }
+        LOG.warn("rename lookupEntry source: {}", source);
+        FilerProto.Entry entry = lookupEntry(source);
+        if (entry == null) {
+            LOG.warn("rename non-existing source: {}", source);
+            return;
+        }
+        LOG.warn("rename moveEntry source: {}", source);
+        moveEntry(source.getParent(), entry, destination);
+    }
+
+    private boolean moveEntry(Path oldParent, FilerProto.Entry entry, Path destination) {
+
+        LOG.debug("moveEntry: {}/{}  => {}", oldParent, entry.getName(), destination);
+
+        FilerProto.Entry.Builder newEntry = entry.toBuilder().setName(destination.getName());
+        boolean isDirectoryCreated = filerClient.createEntry(getParentDirectory(destination), newEntry.build());
+
+        if (!isDirectoryCreated) {
+            return false;
+        }
+
+        if (entry.getIsDirectory()) {
+            Path entryPath = new Path(oldParent, entry.getName());
+            List<FilerProto.Entry> entries = filerClient.listEntries(entryPath.toUri().getPath());
+            for (FilerProto.Entry ent : entries) {
+                boolean isSucess = moveEntry(entryPath, ent, new Path(destination, ent.getName()));
+                if (!isSucess) {
+                    return false;
+                }
+            }
+        }
+
+        return filerClient.deleteEntry(
+            oldParent.toUri().getPath(), entry.getName(), false, false);
+
+    }
+
+    public OutputStream createFile(final Path path,
+                                   final boolean overwrite,
+                                   FsPermission permission,
+                                   int bufferSize,
+                                   String replication) throws IOException {
+
+        permission = permission == null ? FsPermission.getFileDefault() : permission;
+
+        LOG.debug("createFile path: {} overwrite: {} permission: {}",
+            path,
+            overwrite,
+            permission.toString());
+
+        UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser();
+        long now = System.currentTimeMillis() / 1000L;
+
+        FilerProto.Entry.Builder entry = null;
+        long writePosition = 0;
+        if (!overwrite) {
+            FilerProto.Entry existingEntry = lookupEntry(path);
+            LOG.debug("createFile merged entry path:{} existingEntry:{}", path, existingEntry);
+            if (existingEntry != null) {
+                entry = FilerProto.Entry.newBuilder();
+                entry.mergeFrom(existingEntry);
+                entry.getAttributesBuilder().setMtime(now);
+            }
+            LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
+            writePosition = SeaweedRead.totalSize(existingEntry.getChunksList());
+            replication = existingEntry.getAttributes().getReplication();
+        }
+        if (entry == null) {
+            entry = FilerProto.Entry.newBuilder()
+                .setName(path.getName())
+                .setIsDirectory(false)
+                .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                    .setFileMode(permissionToMode(permission, false))
+                    .setReplication(replication)
+                    .setCrtime(now)
+                    .setMtime(now)
+                    .setUserName(userGroupInformation.getUserName())
+                    .clearGroupName()
+                    .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames()))
+                );
+        }
+
+        return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication);
+
+    }
+
+    public InputStream openFileForRead(final Path path, FileSystem.Statistics statistics,
+                                       int bufferSize) throws IOException {
+
+        LOG.debug("openFileForRead path:{} bufferSize:{}", path, bufferSize);
+
+        int readAheadQueueDepth = 2;
+        FilerProto.Entry entry = lookupEntry(path);
+
+        if (entry == null) {
+            throw new FileNotFoundException("read non-exist file " + path);
+        }
+
+        return new SeaweedInputStream(filerGrpcClient,
+            statistics,
+            path.toUri().getPath(),
+            entry,
+            bufferSize,
+            readAheadQueueDepth);
+    }
+
+    public void setOwner(Path path, String owner, String group) {
+
+        LOG.debug("setOwner path:{} owner:{} group:{}", path, owner, group);
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            LOG.debug("setOwner path:{} entry:{}", path, entry);
+            return;
+        }
+
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
+
+        if (owner != null) {
+            attributesBuilder.setUserName(owner);
+        }
+        if (group != null) {
+            attributesBuilder.clearGroupName();
+            attributesBuilder.addGroupName(group);
+        }
+
+        entryBuilder.setAttributes(attributesBuilder);
+
+        LOG.debug("setOwner path:{} entry:{}", path, entryBuilder);
+
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
+
+    }
+
+    public void setPermission(Path path, FsPermission permission) {
+
+        LOG.debug("setPermission path:{} permission:{}", path, permission);
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            LOG.debug("setPermission path:{} entry:{}", path, entry);
+            return;
+        }
+
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
+
+        attributesBuilder.setFileMode(permissionToMode(permission, entry.getIsDirectory()));
+
+        entryBuilder.setAttributes(attributesBuilder);
+
+        LOG.debug("setPermission path:{} entry:{}", path, entryBuilder);
+
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
+
+    }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedInputStream.java

@@ -0,0 +1,370 @@
+package seaweed.hdfs;
+
+// based on org.apache.hadoop.fs.azurebfs.services.AbfsInputStream
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.List;
+
+public class SeaweedInputStream extends FSInputStream {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class);
+
+    private final FilerGrpcClient filerGrpcClient;
+    private final Statistics statistics;
+    private final String path;
+    private final FilerProto.Entry entry;
+    private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
+    private final long contentLength;
+    private final int bufferSize; // default buffer size
+    private final int readAheadQueueDepth;         // initialized in constructor
+    private final boolean readAheadEnabled; // whether enable readAhead;
+
+    private byte[] buffer = null;            // will be initialized on first use
+
+    private long fCursor = 0;  // cursor of buffer within file - offset of next byte to read from remote server
+    private long fCursorAfterLastRead = -1;
+    private int bCursor = 0;   // cursor of read within buffer - offset of next byte to be returned from buffer
+    private int limit = 0;     // offset of next byte to be read into buffer from service (i.e., upper marker+1
+    //                                                      of valid bytes in buffer)
+    private boolean closed = false;
+
+    public SeaweedInputStream(
+        final FilerGrpcClient filerGrpcClient,
+        final Statistics statistics,
+        final String path,
+        final FilerProto.Entry entry,
+        final int bufferSize,
+        final int readAheadQueueDepth) {
+        this.filerGrpcClient = filerGrpcClient;
+        this.statistics = statistics;
+        this.path = path;
+        this.entry = entry;
+        this.contentLength = SeaweedRead.totalSize(entry.getChunksList());
+        this.bufferSize = bufferSize;
+        this.readAheadQueueDepth = (readAheadQueueDepth >= 0) ? readAheadQueueDepth : Runtime.getRuntime().availableProcessors();
+        this.readAheadEnabled = true;
+
+        this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(entry.getChunksList());
+
+        LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
+
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    @Override
+    public int read() throws IOException {
+        byte[] b = new byte[1];
+        int numberOfBytesRead = read(b, 0, 1);
+        if (numberOfBytesRead < 0) {
+            return -1;
+        } else {
+            return (b[0] & 0xFF);
+        }
+    }
+
+    @Override
+    public synchronized int read(final byte[] b, final int off, final int len) throws IOException {
+        int currentOff = off;
+        int currentLen = len;
+        int lastReadBytes;
+        int totalReadBytes = 0;
+        do {
+            lastReadBytes = readOneBlock(b, currentOff, currentLen);
+            if (lastReadBytes > 0) {
+                currentOff += lastReadBytes;
+                currentLen -= lastReadBytes;
+                totalReadBytes += lastReadBytes;
+            }
+            if (currentLen <= 0 || currentLen > b.length - currentOff) {
+                break;
+            }
+        } while (lastReadBytes > 0);
+        return totalReadBytes > 0 ? totalReadBytes : lastReadBytes;
+    }
+
+    private int readOneBlock(final byte[] b, final int off, final int len) throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+
+        Preconditions.checkNotNull(b);
+
+        if (len == 0) {
+            return 0;
+        }
+
+        if (this.available() == 0) {
+            return -1;
+        }
+
+        if (off < 0 || len < 0 || len > b.length - off) {
+            throw new IndexOutOfBoundsException();
+        }
+
+        //If buffer is empty, then fill the buffer.
+        if (bCursor == limit) {
+            //If EOF, then return -1
+            if (fCursor >= contentLength) {
+                return -1;
+            }
+
+            long bytesRead = 0;
+            //reset buffer to initial state - i.e., throw away existing data
+            bCursor = 0;
+            limit = 0;
+            if (buffer == null) {
+                buffer = new byte[bufferSize];
+            }
+
+            // Enable readAhead when reading sequentially
+            if (-1 == fCursorAfterLastRead || fCursorAfterLastRead == fCursor || b.length >= bufferSize) {
+                bytesRead = readInternal(fCursor, buffer, 0, bufferSize, false);
+            } else {
+                bytesRead = readInternal(fCursor, buffer, 0, b.length, true);
+            }
+
+            if (bytesRead == -1) {
+                return -1;
+            }
+
+            limit += bytesRead;
+            fCursor += bytesRead;
+            fCursorAfterLastRead = fCursor;
+        }
+
+        //If there is anything in the buffer, then return lesser of (requested bytes) and (bytes in buffer)
+        //(bytes returned may be less than requested)
+        int bytesRemaining = limit - bCursor;
+        int bytesToRead = Math.min(len, bytesRemaining);
+        System.arraycopy(buffer, bCursor, b, off, bytesToRead);
+        bCursor += bytesToRead;
+        if (statistics != null) {
+            statistics.incrementBytesRead(bytesToRead);
+        }
+        return bytesToRead;
+    }
+
+
+    private int readInternal(final long position, final byte[] b, final int offset, final int length,
+                             final boolean bypassReadAhead) throws IOException {
+        if (readAheadEnabled && !bypassReadAhead) {
+            // try reading from read-ahead
+            if (offset != 0) {
+                throw new IllegalArgumentException("readahead buffers cannot have non-zero buffer offsets");
+            }
+            int receivedBytes;
+
+            // queue read-aheads
+            int numReadAheads = this.readAheadQueueDepth;
+            long nextSize;
+            long nextOffset = position;
+            while (numReadAheads > 0 && nextOffset < contentLength) {
+                nextSize = Math.min((long) bufferSize, contentLength - nextOffset);
+                ReadBufferManager.getBufferManager().queueReadAhead(this, nextOffset, (int) nextSize);
+                nextOffset = nextOffset + nextSize;
+                numReadAheads--;
+            }
+
+            // try reading from buffers first
+            receivedBytes = ReadBufferManager.getBufferManager().getBlock(this, position, length, b);
+            if (receivedBytes > 0) {
+                return receivedBytes;
+            }
+
+            // got nothing from read-ahead, do our own read now
+            receivedBytes = readRemote(position, b, offset, length);
+            return receivedBytes;
+        } else {
+            return readRemote(position, b, offset, length);
+        }
+    }
+
+    int readRemote(long position, byte[] b, int offset, int length) throws IOException {
+        if (position < 0) {
+            throw new IllegalArgumentException("attempting to read from negative offset");
+        }
+        if (position >= contentLength) {
+            return -1;  // Hadoop prefers -1 to EOFException
+        }
+        if (b == null) {
+            throw new IllegalArgumentException("null byte array passed in to read() method");
+        }
+        if (offset >= b.length) {
+            throw new IllegalArgumentException("offset greater than length of array");
+        }
+        if (length < 0) {
+            throw new IllegalArgumentException("requested read length is less than zero");
+        }
+        if (length > (b.length - offset)) {
+            throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
+        }
+
+        long bytesRead = SeaweedRead.read(filerGrpcClient, visibleIntervalList, position, b, offset, length);
+        if (bytesRead > Integer.MAX_VALUE) {
+            throw new IOException("Unexpected Content-Length");
+        }
+        return (int) bytesRead;
+    }
+
+    /**
+     * Seek to given position in stream.
+     *
+     * @param n position to seek to
+     * @throws IOException  if there is an error
+     * @throws EOFException if attempting to seek past end of file
+     */
+    @Override
+    public synchronized void seek(long n) throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        if (n < 0) {
+            throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+        }
+        if (n > contentLength) {
+            throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+        }
+
+        if (n >= fCursor - limit && n <= fCursor) { // within buffer
+            bCursor = (int) (n - (fCursor - limit));
+            return;
+        }
+
+        // next read will read from here
+        fCursor = n;
+
+        //invalidate buffer
+        limit = 0;
+        bCursor = 0;
+    }
+
+    @Override
+    public synchronized long skip(long n) throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        long currentPos = getPos();
+        if (currentPos == contentLength) {
+            if (n > 0) {
+                throw new EOFException(FSExceptionMessages.CANNOT_SEEK_PAST_EOF);
+            }
+        }
+        long newPos = currentPos + n;
+        if (newPos < 0) {
+            newPos = 0;
+            n = newPos - currentPos;
+        }
+        if (newPos > contentLength) {
+            newPos = contentLength;
+            n = newPos - currentPos;
+        }
+        seek(newPos);
+        return n;
+    }
+
+    /**
+     * Return the size of the remaining available bytes
+     * if the size is less than or equal to {@link Integer#MAX_VALUE},
+     * otherwise, return {@link Integer#MAX_VALUE}.
+     * <p>
+     * This is to match the behavior of DFSInputStream.available(),
+     * which some clients may rely on (HBase write-ahead log reading in
+     * particular).
+     */
+    @Override
+    public synchronized int available() throws IOException {
+        if (closed) {
+            throw new IOException(
+                FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        final long remaining = this.contentLength - this.getPos();
+        return remaining <= Integer.MAX_VALUE
+            ? (int) remaining : Integer.MAX_VALUE;
+    }
+
+    /**
+     * Returns the length of the file that this stream refers to. Note that the length returned is the length
+     * as of the time the Stream was opened. Specifically, if there have been subsequent appends to the file,
+     * they wont be reflected in the returned length.
+     *
+     * @return length of the file.
+     * @throws IOException if the stream is closed
+     */
+    public long length() throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        return contentLength;
+    }
+
+    /**
+     * Return the current offset from the start of the file
+     *
+     * @throws IOException throws {@link IOException} if there is an error
+     */
+    @Override
+    public synchronized long getPos() throws IOException {
+        if (closed) {
+            throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+        }
+        return fCursor - limit + bCursor;
+    }
+
+    /**
+     * Seeks a different copy of the data.  Returns true if
+     * found a new source, false otherwise.
+     *
+     * @throws IOException throws {@link IOException} if there is an error
+     */
+    @Override
+    public boolean seekToNewSource(long l) throws IOException {
+        return false;
+    }
+
+    @Override
+    public synchronized void close() throws IOException {
+        closed = true;
+        buffer = null; // de-reference the buffer so it can be GC'ed sooner
+    }
+
+    /**
+     * Not supported by this stream. Throws {@link UnsupportedOperationException}
+     *
+     * @param readlimit ignored
+     */
+    @Override
+    public synchronized void mark(int readlimit) {
+        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
+    }
+
+    /**
+     * Not supported by this stream. Throws {@link UnsupportedOperationException}
+     */
+    @Override
+    public synchronized void reset() throws IOException {
+        throw new UnsupportedOperationException("mark()/reset() not supported on this stream");
+    }
+
+    /**
+     * gets whether mark and reset are supported by {@code ADLFileInputStream}. Always returns false.
+     *
+     * @return always {@code false}
+     */
+    @Override
+    public boolean markSupported() {
+        return false;
+    }
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedOutputStream.java

@@ -0,0 +1,332 @@
+package seaweed.hdfs;
+
+// adapted from org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.Syncable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.util.Locale;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public class SeaweedOutputStream extends OutputStream implements Syncable, StreamCapabilities {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class);
+
+    private final FilerGrpcClient filerGrpcClient;
+    private final Path path;
+    private final int bufferSize;
+    private final int maxConcurrentRequestCount;
+    private final ThreadPoolExecutor threadExecutor;
+    private final ExecutorCompletionService<Void> completionService;
+    private FilerProto.Entry.Builder entry;
+    private long position;
+    private boolean closed;
+    private boolean supportFlush = true;
+    private volatile IOException lastError;
+    private long lastFlushOffset;
+    private long lastTotalAppendOffset = 0;
+    private byte[] buffer;
+    private int bufferIndex;
+    private ConcurrentLinkedDeque<WriteOperation> writeOperations;
+    private String replication = "000";
+
+    public SeaweedOutputStream(FilerGrpcClient filerGrpcClient, final Path path, FilerProto.Entry.Builder entry,
+                               final long position, final int bufferSize, final String replication) {
+        this.filerGrpcClient = filerGrpcClient;
+        this.replication = replication;
+        this.path = path;
+        this.position = position;
+        this.closed = false;
+        this.lastError = null;
+        this.lastFlushOffset = 0;
+        this.bufferSize = bufferSize;
+        this.buffer = new byte[bufferSize];
+        this.bufferIndex = 0;
+        this.writeOperations = new ConcurrentLinkedDeque<>();
+
+        this.maxConcurrentRequestCount = 4 * Runtime.getRuntime().availableProcessors();
+
+        this.threadExecutor
+            = new ThreadPoolExecutor(maxConcurrentRequestCount,
+            maxConcurrentRequestCount,
+            10L,
+            TimeUnit.SECONDS,
+            new LinkedBlockingQueue<Runnable>());
+        this.completionService = new ExecutorCompletionService<>(this.threadExecutor);
+
+        this.entry = entry;
+
+    }
+
+    private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException {
+
+        LOG.debug("SeaweedWrite.writeMeta path: {} entry:{}", path, entry);
+
+        try {
+            SeaweedWrite.writeMeta(filerGrpcClient, path, entry);
+        } catch (Exception ex) {
+            throw new IOException(ex);
+        }
+        this.lastFlushOffset = offset;
+    }
+
+    @Override
+    public void write(final int byteVal) throws IOException {
+        write(new byte[]{(byte) (byteVal & 0xFF)});
+    }
+
+    @Override
+    public synchronized void write(final byte[] data, final int off, final int length)
+        throws IOException {
+        maybeThrowLastError();
+
+        Preconditions.checkArgument(data != null, "null data");
+
+        if (off < 0 || length < 0 || length > data.length - off) {
+            throw new IndexOutOfBoundsException();
+        }
+
+        int currentOffset = off;
+        int writableBytes = bufferSize - bufferIndex;
+        int numberOfBytesToWrite = length;
+
+        while (numberOfBytesToWrite > 0) {
+            if (writableBytes <= numberOfBytesToWrite) {
+                System.arraycopy(data, currentOffset, buffer, bufferIndex, writableBytes);
+                bufferIndex += writableBytes;
+                writeCurrentBufferToService();
+                currentOffset += writableBytes;
+                numberOfBytesToWrite = numberOfBytesToWrite - writableBytes;
+            } else {
+                System.arraycopy(data, currentOffset, buffer, bufferIndex, numberOfBytesToWrite);
+                bufferIndex += numberOfBytesToWrite;
+                numberOfBytesToWrite = 0;
+            }
+
+            writableBytes = bufferSize - bufferIndex;
+        }
+    }
+
+    /**
+     * Flushes this output stream and forces any buffered output bytes to be
+     * written out. If any data remains in the payload it is committed to the
+     * service. Data is queued for writing and forced out to the service
+     * before the call returns.
+     */
+    @Override
+    public void flush() throws IOException {
+        if (supportFlush) {
+            flushInternalAsync();
+        }
+    }
+
+    /**
+     * Similar to posix fsync, flush out the data in client's user buffer
+     * all the way to the disk device (but the disk may have it in its cache).
+     *
+     * @throws IOException if error occurs
+     */
+    @Override
+    public void hsync() throws IOException {
+        if (supportFlush) {
+            flushInternal();
+        }
+    }
+
+    /**
+     * Flush out the data in client's user buffer. After the return of
+     * this call, new readers will see the data.
+     *
+     * @throws IOException if any error occurs
+     */
+    @Override
+    public void hflush() throws IOException {
+        if (supportFlush) {
+            flushInternal();
+        }
+    }
+
+    /**
+     * Query the stream for a specific capability.
+     *
+     * @param capability string to query the stream support for.
+     * @return true for hsync and hflush.
+     */
+    @Override
+    public boolean hasCapability(String capability) {
+        switch (capability.toLowerCase(Locale.ENGLISH)) {
+            case StreamCapabilities.HSYNC:
+            case StreamCapabilities.HFLUSH:
+                return supportFlush;
+            default:
+                return false;
+        }
+    }
+
+    /**
+     * Force all data in the output stream to be written to Azure storage.
+     * Wait to return until this is complete. Close the access to the stream and
+     * shutdown the upload thread pool.
+     * If the blob was created, its lease will be released.
+     * Any error encountered caught in threads and stored will be rethrown here
+     * after cleanup.
+     */
+    @Override
+    public synchronized void close() throws IOException {
+        if (closed) {
+            return;
+        }
+
+        LOG.debug("close path: {}", path);
+        try {
+            flushInternal();
+            threadExecutor.shutdown();
+        } finally {
+            lastError = new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+            buffer = null;
+            bufferIndex = 0;
+            closed = true;
+            writeOperations.clear();
+            if (!threadExecutor.isShutdown()) {
+                threadExecutor.shutdownNow();
+            }
+        }
+    }
+
+    private synchronized void writeCurrentBufferToService() throws IOException {
+        if (bufferIndex == 0) {
+            return;
+        }
+
+        final byte[] bytes = buffer;
+        final int bytesLength = bufferIndex;
+
+        buffer = new byte[bufferSize];
+        bufferIndex = 0;
+        final long offset = position;
+        position += bytesLength;
+
+        if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount * 2) {
+            waitForTaskToComplete();
+        }
+
+        final Future<Void> job = completionService.submit(new Callable<Void>() {
+            @Override
+            public Void call() throws Exception {
+                // originally: client.append(path, offset, bytes, 0, bytesLength);
+                SeaweedWrite.writeData(entry, replication, filerGrpcClient, offset, bytes, 0, bytesLength);
+                return null;
+            }
+        });
+
+        writeOperations.add(new WriteOperation(job, offset, bytesLength));
+
+        // Try to shrink the queue
+        shrinkWriteOperationQueue();
+    }
+
+    private void waitForTaskToComplete() throws IOException {
+        boolean completed;
+        for (completed = false; completionService.poll() != null; completed = true) {
+            // keep polling until there is no data
+        }
+
+        if (!completed) {
+            try {
+                completionService.take();
+            } catch (InterruptedException e) {
+                lastError = (IOException) new InterruptedIOException(e.toString()).initCause(e);
+                throw lastError;
+            }
+        }
+    }
+
+    private void maybeThrowLastError() throws IOException {
+        if (lastError != null) {
+            throw lastError;
+        }
+    }
+
+    /**
+     * Try to remove the completed write operations from the beginning of write
+     * operation FIFO queue.
+     */
+    private synchronized void shrinkWriteOperationQueue() throws IOException {
+        try {
+            while (writeOperations.peek() != null && writeOperations.peek().task.isDone()) {
+                writeOperations.peek().task.get();
+                lastTotalAppendOffset += writeOperations.peek().length;
+                writeOperations.remove();
+            }
+        } catch (Exception e) {
+            lastError = new IOException(e);
+            throw lastError;
+        }
+    }
+
+    private synchronized void flushInternal() throws IOException {
+        maybeThrowLastError();
+        writeCurrentBufferToService();
+        flushWrittenBytesToService();
+    }
+
+    private synchronized void flushInternalAsync() throws IOException {
+        maybeThrowLastError();
+        writeCurrentBufferToService();
+        flushWrittenBytesToServiceAsync();
+    }
+
+    private synchronized void flushWrittenBytesToService() throws IOException {
+        for (WriteOperation writeOperation : writeOperations) {
+            try {
+                writeOperation.task.get();
+            } catch (Exception ex) {
+                lastError = new IOException(ex);
+                throw lastError;
+            }
+        }
+        LOG.debug("flushWrittenBytesToService: {} position:{}", path, position);
+        flushWrittenBytesToServiceInternal(position);
+    }
+
+    private synchronized void flushWrittenBytesToServiceAsync() throws IOException {
+        shrinkWriteOperationQueue();
+
+        if (this.lastTotalAppendOffset > this.lastFlushOffset) {
+            this.flushWrittenBytesToServiceInternal(this.lastTotalAppendOffset);
+        }
+    }
+
+    private static class WriteOperation {
+        private final Future<Void> task;
+        private final long startOffset;
+        private final long length;
+
+        WriteOperation(final Future<Void> task, final long startOffset, final long length) {
+            Preconditions.checkNotNull(task, "task");
+            Preconditions.checkArgument(startOffset >= 0, "startOffset");
+            Preconditions.checkArgument(length >= 0, "length");
+
+            this.task = task;
+            this.startOffset = startOffset;
+            this.length = length;
+        }
+    }
+
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedRead.java

@@ -0,0 +1,265 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.hdfs.util.ByteBufferOutputStream;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+public class SeaweedRead {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
+
+    // returns bytesRead
+    public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
+                            final long position, final byte[] buffer, final int bufferOffset,
+                            final int bufferLength) {
+
+        List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, bufferLength);
+
+        FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder();
+        for (ChunkView chunkView : chunkViews) {
+            String vid = parseVolumeId(chunkView.fileId);
+            lookupRequest.addVolumeIds(vid);
+        }
+
+        FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient
+            .getBlockingStub().lookupVolume(lookupRequest.build());
+
+        Map<String, FilerProto.Locations> vid2Locations = lookupResponse.getLocationsMapMap();
+
+        //TODO parallel this
+        long readCount = 0;
+        int startOffset = bufferOffset;
+        for (ChunkView chunkView : chunkViews) {
+            FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId));
+            if (locations.getLocationsCount() == 0) {
+                // log here!
+                return 0;
+            }
+
+            int len = readChunkView(position, buffer, startOffset, chunkView, locations);
+
+            readCount += len;
+            startOffset += len;
+
+        }
+
+        return readCount;
+    }
+
+    private static int readChunkView(long position, byte[] buffer, int startOffset, ChunkView chunkView, FilerProto.Locations locations) {
+        HttpClient client = HttpClientBuilder.create().build();
+        HttpGet request = new HttpGet(
+            String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));
+
+        if (!chunkView.isFullChunk) {
+            request.setHeader(HttpHeaders.ACCEPT_ENCODING, "");
+            request.setHeader(HttpHeaders.RANGE,
+                String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size));
+        }
+
+        try {
+            HttpResponse response = client.execute(request);
+            HttpEntity entity = response.getEntity();
+
+            int len = (int) (chunkView.logicOffset - position + chunkView.size);
+            OutputStream outputStream = new ByteBufferOutputStream(ByteBuffer.wrap(buffer, startOffset, len));
+            entity.writeTo(outputStream);
+            LOG.debug("* read chunkView:{} startOffset:{} length:{}", chunkView, startOffset, len);
+
+            return len;
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return 0;
+    }
+
+    public static List<ChunkView> viewFromVisibles(List<VisibleInterval> visibleIntervals, long offset, long size) {
+        List<ChunkView> views = new ArrayList<>();
+
+        long stop = offset + size;
+        for (VisibleInterval chunk : visibleIntervals) {
+            if (chunk.start <= offset && offset < chunk.stop && offset < stop) {
+                boolean isFullChunk = chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop;
+                views.add(new ChunkView(
+                    chunk.fileId,
+                    offset - chunk.start,
+                    Math.min(chunk.stop, stop) - offset,
+                    offset,
+                    isFullChunk
+                ));
+                offset = Math.min(chunk.stop, stop);
+            }
+        }
+        return views;
+    }
+
+    public static List<VisibleInterval> nonOverlappingVisibleIntervals(List<FilerProto.FileChunk> chunkList) {
+        FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]);
+        Arrays.sort(chunks, new Comparator<FilerProto.FileChunk>() {
+            @Override
+            public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) {
+                return (int) (a.getMtime() - b.getMtime());
+            }
+        });
+
+        List<VisibleInterval> visibles = new ArrayList<>();
+        for (FilerProto.FileChunk chunk : chunks) {
+            List<VisibleInterval> newVisibles = new ArrayList<>();
+            visibles = mergeIntoVisibles(visibles, newVisibles, chunk);
+        }
+
+        return visibles;
+    }
+
+    private static List<VisibleInterval> mergeIntoVisibles(List<VisibleInterval> visibles,
+                                                           List<VisibleInterval> newVisibles,
+                                                           FilerProto.FileChunk chunk) {
+        VisibleInterval newV = new VisibleInterval(
+            chunk.getOffset(),
+            chunk.getOffset() + chunk.getSize(),
+            chunk.getFileId(),
+            chunk.getMtime(),
+            true
+        );
+
+        // easy cases to speed up
+        if (visibles.size() == 0) {
+            visibles.add(newV);
+            return visibles;
+        }
+        if (visibles.get(visibles.size() - 1).stop <= chunk.getOffset()) {
+            visibles.add(newV);
+            return visibles;
+        }
+
+        for (VisibleInterval v : visibles) {
+            if (v.start < chunk.getOffset() && chunk.getOffset() < v.stop) {
+                newVisibles.add(new VisibleInterval(
+                    v.start,
+                    chunk.getOffset(),
+                    v.fileId,
+                    v.modifiedTime,
+                    false
+                ));
+            }
+            long chunkStop = chunk.getOffset() + chunk.getSize();
+            if (v.start < chunkStop && chunkStop < v.stop) {
+                newVisibles.add(new VisibleInterval(
+                    chunkStop,
+                    v.stop,
+                    v.fileId,
+                    v.modifiedTime,
+                    false
+                ));
+            }
+            if (chunkStop <= v.start || v.stop <= chunk.getOffset()) {
+                newVisibles.add(v);
+            }
+        }
+        newVisibles.add(newV);
+
+        // keep everything sorted
+        for (int i = newVisibles.size() - 1; i >= 0; i--) {
+            if (i > 0 && newV.start < newVisibles.get(i - 1).start) {
+                newVisibles.set(i, newVisibles.get(i - 1));
+            } else {
+                newVisibles.set(i, newV);
+                break;
+            }
+        }
+
+        return newVisibles;
+    }
+
+    public static String parseVolumeId(String fileId) {
+        int commaIndex = fileId.lastIndexOf(',');
+        if (commaIndex > 0) {
+            return fileId.substring(0, commaIndex);
+        }
+        return fileId;
+    }
+
+    public static long totalSize(List<FilerProto.FileChunk> chunksList) {
+        long size = 0;
+        for (FilerProto.FileChunk chunk : chunksList) {
+            long t = chunk.getOffset() + chunk.getSize();
+            if (size < t) {
+                size = t;
+            }
+        }
+        return size;
+    }
+
+    public static class VisibleInterval {
+        public final long start;
+        public final long stop;
+        public final long modifiedTime;
+        public final String fileId;
+        public final boolean isFullChunk;
+
+        public VisibleInterval(long start, long stop, String fileId, long modifiedTime, boolean isFullChunk) {
+            this.start = start;
+            this.stop = stop;
+            this.modifiedTime = modifiedTime;
+            this.fileId = fileId;
+            this.isFullChunk = isFullChunk;
+        }
+
+        @Override
+        public String toString() {
+            return "VisibleInterval{" +
+                "start=" + start +
+                ", stop=" + stop +
+                ", modifiedTime=" + modifiedTime +
+                ", fileId='" + fileId + '\'' +
+                ", isFullChunk=" + isFullChunk +
+                '}';
+        }
+    }
+
+    public static class ChunkView {
+        public final String fileId;
+        public final long offset;
+        public final long size;
+        public final long logicOffset;
+        public final boolean isFullChunk;
+
+        public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk) {
+            this.fileId = fileId;
+            this.offset = offset;
+            this.size = size;
+            this.logicOffset = logicOffset;
+            this.isFullChunk = isFullChunk;
+        }
+
+        @Override
+        public String toString() {
+            return "ChunkView{" +
+                "fileId='" + fileId + '\'' +
+                ", offset=" + offset +
+                ", size=" + size +
+                ", logicOffset=" + logicOffset +
+                ", isFullChunk=" + isFullChunk +
+                '}';
+        }
+    }
+
+}

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedWrite.java

@@ -0,0 +1,91 @@
+package seaweed.hdfs;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.mime.HttpMultipartMode;
+import org.apache.http.entity.mime.MultipartEntityBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import seaweedfs.client.FilerGrpcClient;
+import seaweedfs.client.FilerProto;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory;
+
+public class SeaweedWrite {
+
+    public static void writeData(FilerProto.Entry.Builder entry,
+                                 final String replication,
+                                 final FilerGrpcClient filerGrpcClient,
+                                 final long offset,
+                                 final byte[] bytes,
+                                 final long bytesOffset, final long bytesLength) throws IOException {
+        FilerProto.AssignVolumeResponse response = filerGrpcClient.getBlockingStub().assignVolume(
+            FilerProto.AssignVolumeRequest.newBuilder()
+                .setCollection("")
+                .setReplication(replication)
+                .setDataCenter("")
+                .setReplication("")
+                .setTtlSec(0)
+                .build());
+        String fileId = response.getFileId();
+        String url = response.getUrl();
+        String targetUrl = String.format("http://%s/%s", url, fileId);
+
+        String etag = multipartUpload(targetUrl, bytes, bytesOffset, bytesLength);
+
+        entry.addChunks(FilerProto.FileChunk.newBuilder()
+            .setFileId(fileId)
+            .setOffset(offset)
+            .setSize(bytesLength)
+            .setMtime(System.currentTimeMillis() / 10000L)
+            .setETag(etag)
+        );
+
+    }
+
+    public static void writeMeta(final FilerGrpcClient filerGrpcClient,
+                                 final Path path, final FilerProto.Entry.Builder entry) {
+        filerGrpcClient.getBlockingStub().createEntry(
+            FilerProto.CreateEntryRequest.newBuilder()
+                .setDirectory(getParentDirectory(path))
+                .setEntry(entry)
+                .build()
+        );
+    }
+
+    private static String multipartUpload(String targetUrl,
+                                          final byte[] bytes,
+                                          final long bytesOffset, final long bytesLength) throws IOException {
+
+        CloseableHttpClient client = HttpClientBuilder.create().setUserAgent("hdfs-client").build();
+
+        InputStream inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
+
+        HttpPost post = new HttpPost(targetUrl);
+
+        post.setEntity(MultipartEntityBuilder.create()
+            .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
+            .addBinaryBody("upload", inputStream)
+            .build());
+
+        try {
+            HttpResponse response = client.execute(post);
+
+            String etag = response.getLastHeader("ETag").getValue();
+
+            if (etag != null && etag.startsWith("\"") && etag.endsWith("\"")) {
+                etag = etag.substring(1, etag.length() - 1);
+            }
+
+            return etag;
+        } finally {
+            client.close();
+        }
+
+    }
+}

other/java/hdfs/src/test/java/seaweedfs/hdfs/SeaweedReadTest.java

@@ -0,0 +1,66 @@
+package seaweedfs.hdfs;
+
+import org.junit.Test;
+import seaweed.hdfs.SeaweedRead;
+import seaweedfs.client.FilerProto;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class SeaweedReadTest {
+
+    @Test
+    public void testNonOverlappingVisibleIntervals() {
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        chunks.add(FilerProto.FileChunk.newBuilder()
+            .setFileId("aaa")
+            .setOffset(0)
+            .setSize(100)
+            .setMtime(1000)
+            .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+            .setFileId("bbb")
+            .setOffset(100)
+            .setSize(133)
+            .setMtime(2000)
+            .build());
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+        }
+
+        assertEquals(visibleIntervals.size(), 2);
+
+        SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0);
+        assertEquals(visibleInterval.start, 0);
+        assertEquals(visibleInterval.stop, 100);
+        assertEquals(visibleInterval.modifiedTime, 1000);
+        assertEquals(visibleInterval.fileId, "aaa");
+
+        visibleInterval = visibleIntervals.get(1);
+        assertEquals(visibleInterval.start, 100);
+        assertEquals(visibleInterval.stop, 233);
+        assertEquals(visibleInterval.modifiedTime, 2000);
+        assertEquals(visibleInterval.fileId, "bbb");
+
+        List<SeaweedRead.ChunkView> chunkViews = SeaweedRead.viewFromVisibles(visibleIntervals, 0, 233);
+
+        SeaweedRead.ChunkView chunkView = chunkViews.get(0);
+        assertEquals(chunkView.offset, 0);
+        assertEquals(chunkView.size, 100);
+        assertEquals(chunkView.logicOffset, 0);
+        assertEquals(chunkView.fileId, "aaa");
+
+        chunkView = chunkViews.get(1);
+        assertEquals(chunkView.offset, 0);
+        assertEquals(chunkView.size, 133);
+        assertEquals(chunkView.logicOffset, 100);
+        assertEquals(chunkView.fileId, "bbb");
+
+
+    }
+
+}

other/java/s3copier/pom.xml

@@ -0,0 +1,35 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.seaweedfs.s3</groupId>
+  <artifactId>copier</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>copier</name>
+  <url>http://maven.apache.org</url>
+
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>com.amazonaws</groupId>
+        <artifactId>aws-java-sdk-bom</artifactId>
+        <version>1.11.327</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.amazonaws</groupId>
+      <artifactId>aws-java-sdk-s3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

other/java/s3copier/src/main/java/com/seaweedfs/s3/HighLevelMultipartUpload.java

@@ -0,0 +1,63 @@
+package com.seaweedfs.s3;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import com.amazonaws.services.s3.transfer.Upload;
+
+import java.io.File;
+
+public class HighLevelMultipartUpload {
+
+    public static void main(String[] args) throws Exception {
+        String bucketName = "javabucket";
+        String filePath = args[0];
+        File file = new File(filePath);
+        String keyName = "path/to/" + file.getName();
+
+        try {
+            AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
+            ClientConfiguration clientConfiguration = new ClientConfiguration();
+            clientConfiguration.setSignerOverride("AWSS3V4SignerType");
+
+            AmazonS3 s3Client = AmazonS3ClientBuilder
+                .standard()
+                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
+                    "http://localhost:8333", Regions.US_WEST_1.name()))
+                .withPathStyleAccessEnabled(true)
+                .withClientConfiguration(clientConfiguration)
+                .withCredentials(new AWSStaticCredentialsProvider(credentials))
+                .build();
+
+            TransferManager tm = TransferManagerBuilder.standard()
+                .withS3Client(s3Client)
+                .build();
+
+            // TransferManager processes all transfers asynchronously,
+            // so this call returns immediately.
+            Upload upload = tm.upload(bucketName, keyName, file);
+            System.out.println("Object upload started");
+
+            // Optionally, wait for the upload to finish before continuing.
+            upload.waitForCompletion();
+            System.out.println("Object upload complete");
+        } catch (AmazonServiceException e) {
+            // The call was transmitted successfully, but Amazon S3 couldn't process
+            // it, so it returned an error response.
+            e.printStackTrace();
+        } catch (SdkClientException e) {
+            // Amazon S3 couldn't be contacted for a response, or the client
+            // couldn't parse the response from Amazon S3.
+            e.printStackTrace();
+        }
+    }
+}

other/java/s3copier/src/main/java/com/seaweedfs/s3/PutObject.java

@@ -0,0 +1,101 @@
+package com.seaweedfs.s3;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.util.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Hello world!
+ */
+public class PutObject {
+
+    private static Log log = LogFactory.getLog(PutObject.class);
+
+    public static void main(String[] args) {
+
+        AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
+        ClientConfiguration clientConfiguration = new ClientConfiguration();
+        clientConfiguration.setSignerOverride("AWSS3V4SignerType");
+
+        AmazonS3 s3Client = AmazonS3ClientBuilder
+            .standard()
+            .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
+                "http://localhost:8333", Regions.US_WEST_1.name()))
+            .withPathStyleAccessEnabled(true)
+            .withClientConfiguration(clientConfiguration)
+            .withCredentials(new AWSStaticCredentialsProvider(credentials))
+            .build();
+
+        String bucketName = "javabucket";
+        String stringObjKeyName = "strObject2";
+        String fileObjKeyName = "fileObject2";
+        String fileName = args[0];
+
+        String stringContent = "Uploaded String Object v3";
+        try {
+
+            // Upload a text string as a new object.
+            s3Client.putObject(bucketName, stringObjKeyName, stringContent);
+
+            // Upload a file as a new object with ContentType and title specified.
+            PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName));
+            ObjectMetadata metadata = new ObjectMetadata();
+            metadata.setContentType("plain/text");
+            metadata.addUserMetadata("x-amz-meta-title", "someTitle");
+            request.setMetadata(metadata);
+            s3Client.putObject(request);
+
+            // test reads
+            S3Object written = s3Client.getObject(bucketName, stringObjKeyName);
+            try {
+                String expected = IOUtils.toString(written.getObjectContent());
+
+                if (!stringContent.equals(expected)){
+                    System.out.println("Failed to put and get back the content!");
+                }
+
+            } catch (IOException e) {
+                throw new SdkClientException("Error streaming content from S3 during download");
+            } finally {
+                IOUtils.closeQuietly(written, log);
+            }
+
+            // test deletes
+            s3Client.deleteObject(bucketName, stringObjKeyName);
+
+
+            // delete bucket
+            String tmpBucket = "tmpbucket";
+            s3Client.createBucket(tmpBucket);
+            s3Client.putObject(tmpBucket, stringObjKeyName, stringContent);
+            s3Client.deleteBucket(tmpBucket);
+
+        } catch (AmazonServiceException e) {
+            // The call was transmitted successfully, but Amazon S3 couldn't process
+            // it, so it returned an error response.
+            e.printStackTrace();
+        } catch (SdkClientException e) {
+            // Amazon S3 couldn't be contacted for a response, or the client
+            // couldn't parse the response from Amazon S3.
+            e.printStackTrace();
+        }
+
+        System.out.println("Hello World!");
+    }
+}

other/java/s3copier/src/test/java/com/seaweedfs/s3/PutObjectTest.java

@@ -0,0 +1,38 @@
+package com.seaweedfs.s3;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit test for simple App.
+ */
+public class PutObjectTest
+    extends TestCase
+{
+    /**
+     * Create the test case
+     *
+     * @param testName name of the test case
+     */
+    public PutObjectTest(String testName )
+    {
+        super( testName );
+    }
+
+    /**
+     * @return the suite of tests being tested
+     */
+    public static Test suite()
+    {
+        return new TestSuite( PutObjectTest.class );
+    }
+
+    /**
+     * Rigourous Test :-)
+     */
+    public void testApp()
+    {
+        assertTrue( true );
+    }
+}

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand"
+
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	master = flag.String("master", "127.0.0.1:9333", "the master server")
+	repeat = flag.Int("n", 5, "repeat how many times")
+)
+
+func main() {
+	flag.Parse()
+
+	for i := 0; i < *repeat; i++ {
+		assignResult, err := operation.Assign(*master, &operation.VolumeAssignRequest{Count: 1})
+		if err != nil {
+			log.Fatalf("assign: %v", err)
+		}
+
+		data := make([]byte, 1024)
+		rand.Read(data)
+		reader := bytes.NewReader(data)
+
+		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url,assignResult.Fid)
+
+		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, "")
+		if err != nil {
+			log.Fatalf("upload: %v", err)
+		}
+
+		util.Delete(targetUrl, "")
+
+		util.Get(fmt.Sprintf("http://%s/vol/vacuum", *master))
+
+	}
+
+}

unmaintained/see_dat/see_dat.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"flag"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+)
+
+var (
+	volumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	volumeCollection = flag.String("collection", "", "the volume collection name")
+	volumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+)
+
+func main() {
+	flag.Parse()
+
+	var version storage.Version
+	vid := storage.VolumeId(*volumeId)
+	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid,
+		storage.NeedleMapInMemory,
+		func(superBlock storage.SuperBlock) error {
+			version = superBlock.Version()
+			return nil
+		}, false, func(n *storage.Needle, offset int64) error {
+			glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x",
+				*volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
+			return nil
+		})
+	if err != nil {
+		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
+	}
+
+}

unmaintained/see_idx/see_idx.go

@@ -2,13 +2,13 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"path"
 	"strconv"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 

weed/command/backup.go

@@ -59,7 +59,7 @@ func runBackup(cmd *Command, args []string) bool {
 	}
 	volumeServer := lookup.Locations[0].Url
 
-	stats, err := operation.GetVolumeSyncStatus(volumeServer, vid.String())
+	stats, err := operation.GetVolumeSyncStatus(volumeServer, uint32(vid))
 	if err != nil {
 		fmt.Printf("Error get volume %d status: %v\n", vid, err)
 		return true

weed/command/command.go

@@ -14,6 +14,7 @@ var Commands = []*Command{
 	cmdCopy,
 	cmdFix,
 	cmdFilerExport,
+	cmdFilerReplicate,
 	cmdServer,
 	cmdMaster,
 	cmdFiler,

weed/command/download.go

@@ -55,7 +55,7 @@ func downloadToFile(server, fileId, saveDir string) error {
 	if lookupError != nil {
 		return lookupError
 	}
-	filename, rc, err := util.DownloadUrl(fileUrl)
+	filename, _, rc, err := util.DownloadFile(fileUrl)
 	if err != nil {
 		return err
 	}
@@ -108,7 +108,7 @@ func fetchContent(server string, fileId string) (filename string, content []byte
 		return "", nil, lookupError
 	}
 	var rc io.ReadCloser
-	if filename, rc, e = util.DownloadUrl(fileUrl); e != nil {
+	if filename, _, rc, e = util.DownloadFile(fileUrl); e != nil {
 		return "", nil, e
 	}
 	content, e = ioutil.ReadAll(rc)

weed/command/filer.go

@@ -3,6 +3,7 @@ package command
 import (
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -10,7 +11,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc/reflection"
-	"strings"
 )
 
 var (
@@ -32,6 +32,9 @@ type FilerOptions struct {
 	dirListingLimit         *int
 	dataCenter              *string
 	enableNotification      *bool
+
+	// default leveldb directory, used in "weed server" mode
+	defaultLevelDbDirectory *string
 }
 
 func init() {
@@ -47,7 +50,7 @@ func init() {
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
 	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
 	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 1000, "limit sub dir listing size")
+	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
 }
 
@@ -74,12 +77,12 @@ var cmdFiler = &Command{
 
 func runFiler(cmd *Command, args []string) bool {
 
-	f.start()
+	f.startFiler()
 
 	return true
 }
 
-func (fo *FilerOptions) start() {
+func (fo *FilerOptions) startFiler() {
 
 	defaultMux := http.NewServeMux()
 	publicVolumeMux := defaultMux
@@ -88,6 +91,11 @@ func (fo *FilerOptions) start() {
 		publicVolumeMux = http.NewServeMux()
 	}
 
+	defaultLevelDbDirectory := "./filerdb"
+	if fo.defaultLevelDbDirectory != nil {
+		defaultLevelDbDirectory = *fo.defaultLevelDbDirectory + "/filerdb"
+	}
+
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
 		Masters:            strings.Split(*f.masters, ","),
 		Collection:         *fo.collection,
@@ -98,6 +106,7 @@ func (fo *FilerOptions) start() {
 		SecretKey:          *fo.secretKey,
 		DirListingLimit:    *fo.dirListingLimit,
 		DataCenter:         *fo.dataCenter,
+		DefaultLevelDbDir:  defaultLevelDbDirectory,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -117,7 +126,7 @@ func (fo *FilerOptions) start() {
 		}()
 	}
 
-	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*fo.port))
+	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.VERSION, *fo.ip, *fo.port)
 	filerListener, e := util.NewListener(
 		":"+strconv.Itoa(*fo.port),
 		time.Duration(10)*time.Second,

weed/command/filer_copy.go

@@ -15,7 +15,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"io"
 	"net/http"
-	"path"
 	"strconv"
 	"time"
 )
@@ -61,12 +60,7 @@ var cmdCopy = &Command{
   All files under the folder and subfolders will be copyed.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
-  If "maxMB" is set to a positive number, files larger than it would be split into chunks and copyed separatedly.
-  The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks.
 
   `,
 }
@@ -166,7 +160,6 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 	// upload the file content
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
-	isGzipped := isGzipped(fileName)
 
 	var chunks []*filer_pb.FileChunk
 
@@ -185,7 +178,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
 
-		uploadResult, err := operation.Upload(targetUrl, fileName, f, isGzipped, mimeType, nil, "")
+		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, "")
 		if err != nil {
 			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 			return false
@@ -201,6 +194,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 			Offset: 0,
 			Size:   uint64(uploadResult.Size),
 			Mtime:  time.Now().UnixNano(),
+			ETag:   uploadResult.ETag,
 		})
 
 		fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
@@ -278,6 +272,7 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 			Offset: i * chunkSize,
 			Size:   uint64(uploadResult.Size),
 			Mtime:  time.Now().UnixNano(),
+			ETag:   uploadResult.ETag,
 		})
 		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 	}
@@ -317,10 +312,6 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 	return true
 }
 
-func isGzipped(filename string) bool {
-	return strings.ToLower(path.Ext(filename)) == ".gz"
-}
-
 func detectMimeType(f *os.File) string {
 	head := make([]byte, 512)
 	f.Seek(0, 0)

weed/command/filer_export.go

@@ -1,10 +1,12 @@
 package command
 
 import (
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/spf13/viper"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/notification"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
 )
 
 func init() {
@@ -22,13 +24,20 @@ var cmdFilerExport = &Command{
 
 	If target store is empty, only the directory tree will be listed.
 
+	If target store is "notification", the list of entries will be sent to notification.
+	This is usually used to bootstrap filer replication to a remote system.
+
   `,
 }
 
 var (
 	// filerExportOutputFile  = cmdFilerExport.Flag.String("output", "", "the output file. If empty, only list out the directory tree")
-	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml")
-	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml")
+	filerExportSourceStore = cmdFilerExport.Flag.String("sourceStore", "", "the source store name in filer.toml, default to currently enabled store")
+	filerExportTargetStore = cmdFilerExport.Flag.String("targetStore", "", "the target store name in filer.toml, or \"notification\" to export all files to message queue")
+	dir                    = cmdFilerExport.Flag.String("dir", "/", "only process files under this directory")
+	dirListLimit           = cmdFilerExport.Flag.Int("dirListLimit", 100000, "limit directory list size")
+	dryRun                 = cmdFilerExport.Flag.Bool("dryRun", false, "not actually moving data")
+	verboseFilerExport     = cmdFilerExport.Flag.Bool("v", false, "verbose entry details")
 )
 
 type statistics struct {
@@ -44,10 +53,10 @@ func runFilerExport(cmd *Command, args []string) bool {
 	var sourceStore, targetStore filer2.FilerStore
 
 	for _, store := range filer2.Stores {
-		if store.GetName() == *filerExportSourceStore {
+		if store.GetName() == *filerExportSourceStore || *filerExportSourceStore == "" && config.GetBool(store.GetName()+".enabled") {
 			viperSub := config.Sub(store.GetName())
 			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize store for %s: %+v",
+				glog.Fatalf("Failed to initialize source store for %s: %+v",
 					store.GetName(), err)
 			} else {
 				sourceStore = store
@@ -60,7 +69,7 @@ func runFilerExport(cmd *Command, args []string) bool {
 		if store.GetName() == *filerExportTargetStore {
 			viperSub := config.Sub(store.GetName())
 			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize store for %s: %+v",
+				glog.Fatalf("Failed to initialize target store for %s: %+v",
 					store.GetName(), err)
 			} else {
 				targetStore = store
@@ -78,19 +87,49 @@ func runFilerExport(cmd *Command, args []string) bool {
 		return false
 	}
 
+	if targetStore == nil && *filerExportTargetStore != "" && *filerExportTargetStore != "notification" {
+		glog.Errorf("Failed to find target store %s", *filerExportTargetStore)
+		println("existing data sources are:")
+		for _, store := range filer2.Stores {
+			println("    " + store.GetName())
+		}
+		return false
+	}
+
 	stat := statistics{}
 
 	var fn func(level int, entry *filer2.Entry) error
 
-	if targetStore == nil {
+	if *filerExportTargetStore == "notification" {
+		weed_server.LoadConfiguration("notification", false)
+		v := viper.GetViper()
+		notification.LoadConfiguration(v.Sub("notification"))
+
+		fn = func(level int, entry *filer2.Entry) error {
+			printout(level, entry)
+			if *dryRun {
+				return nil
+			}
+			return notification.Queue.SendMessage(
+				string(entry.FullPath),
+				&filer_pb.EventNotification{
+					NewEntry: entry.ToProtoEntry(),
+				},
+			)
+		}
+	} else if targetStore == nil {
 		fn = printout
 	} else {
 		fn = func(level int, entry *filer2.Entry) error {
+			printout(level, entry)
+			if *dryRun {
+				return nil
+			}
 			return targetStore.InsertEntry(entry)
 		}
 	}
 
-	doTraverse(&stat, sourceStore, filer2.FullPath("/"), 0, fn)
+	doTraverse(&stat, sourceStore, filer2.FullPath(*dir), 0, fn)
 
 	glog.Infof("processed %d directories, %d files", stat.directoryCount, stat.fileCount)
 
@@ -99,7 +138,7 @@ func runFilerExport(cmd *Command, args []string) bool {
 
 func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer2.FullPath, level int, fn func(level int, entry *filer2.Entry) error) {
 
-	limit := 1000
+	limit := *dirListLimit
 	lastEntryName := ""
 	for {
 		entries, err := filerStore.ListDirectoryEntries(parentPath, lastEntryName, false, limit)
@@ -125,8 +164,24 @@ func doTraverse(stat *statistics, filerStore filer2.FilerStore, parentPath filer
 
 func printout(level int, entry *filer2.Entry) error {
 	for i := 0; i < level; i++ {
-		print("  ")
+		if i == level-1 {
+			print("+-")
+		} else {
+			print("| ")
+		}
 	}
-	println(entry.FullPath.Name())
+	print(entry.FullPath.Name())
+	if *verboseFilerExport {
+		for _, chunk := range entry.Chunks {
+			print("[")
+			print(chunk.FileId)
+			print(",")
+			print(chunk.Offset)
+			print(",")
+			print(chunk.Size)
+			print(")")
+		}
+	}
+	println()
 	return nil
 }

weed/command/filer_replication.go

@@ -0,0 +1,139 @@
+package command
+
+import (
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/replication"
+	"github.com/chrislusf/seaweedfs/weed/replication/sink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
+	_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
+	"github.com/chrislusf/seaweedfs/weed/replication/sub"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+)
+
+func init() {
+	cmdFilerReplicate.Run = runFilerReplicate // break init cycle
+}
+
+var cmdFilerReplicate = &Command{
+	UsageLine: "filer.replicate",
+	Short:     "replicate file changes to another destination",
+	Long: `replicate file changes to another destination
+
+	filer.replicate listens on filer notifications. If any file is updated, it will fetch the updated content,
+	and write to the other destination.
+
+	Run "weed scaffold -config replication" to generate a replication.toml file and customize the parameters.
+
+  `,
+}
+
+func runFilerReplicate(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("replication", true)
+	weed_server.LoadConfiguration("notification", true)
+	config := viper.GetViper()
+
+	var notificationInput sub.NotificationInput
+
+	validateOneEnabledInput(config)
+
+	for _, input := range sub.NotificationInputs {
+		if config.GetBool("notification." + input.GetName() + ".enabled") {
+			viperSub := config.Sub("notification." + input.GetName())
+			if err := input.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize notification input for %s: %+v",
+					input.GetName(), err)
+			}
+			glog.V(0).Infof("Configure notification input to %s", input.GetName())
+			notificationInput = input
+			break
+		}
+	}
+
+	if notificationInput == nil {
+		println("No notification is defined in notification.toml file.")
+		println("Please follow 'weed scaffold -config=notification' to see example notification configurations.")
+		return true
+	}
+
+	// avoid recursive replication
+	if config.GetBool("notification.source.filer.enabled") && config.GetBool("notification.sink.filer.enabled") {
+		sourceConfig, sinkConfig := config.Sub("source.filer"), config.Sub("sink.filer")
+		if sourceConfig.GetString("grpcAddress") == sinkConfig.GetString("grpcAddress") {
+			fromDir := sourceConfig.GetString("directory")
+			toDir := sinkConfig.GetString("directory")
+			if strings.HasPrefix(toDir, fromDir) {
+				glog.Fatalf("recursive replication! source directory %s includes the sink directory %s", fromDir, toDir)
+			}
+		}
+	}
+
+	var dataSink sink.ReplicationSink
+	for _, sk := range sink.Sinks {
+		if config.GetBool("sink." + sk.GetName() + ".enabled") {
+			viperSub := config.Sub("sink." + sk.GetName())
+			if err := sk.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize sink for %s: %+v",
+					sk.GetName(), err)
+			}
+			glog.V(0).Infof("Configure sink to %s", sk.GetName())
+			dataSink = sk
+			break
+		}
+	}
+
+	if dataSink == nil {
+		println("no data sink configured in replication.toml:")
+		for _, sk := range sink.Sinks {
+			println("    " + sk.GetName())
+		}
+		return true
+	}
+
+	replicator := replication.NewReplicator(config.Sub("source.filer"), dataSink)
+
+	for {
+		key, m, err := notificationInput.ReceiveMessage()
+		if err != nil {
+			glog.Errorf("receive %s: %+v", key, err)
+			continue
+		}
+		if key == "" {
+			// long poll received no messages
+			continue
+		}
+		if m.OldEntry != nil && m.NewEntry == nil {
+			glog.V(1).Infof("delete: %s", key)
+		} else if m.OldEntry == nil && m.NewEntry != nil {
+			glog.V(1).Infof("   add: %s", key)
+		} else {
+			glog.V(1).Infof("modify: %s", key)
+		}
+		if err = replicator.Replicate(key, m); err != nil {
+			glog.Errorf("replicate %s: %+v", key, err)
+		} else {
+			glog.V(1).Infof("replicated %s", key)
+		}
+	}
+
+	return true
+}
+
+func validateOneEnabledInput(config *viper.Viper) {
+	enabledInput := ""
+	for _, input := range sub.NotificationInputs {
+		if config.GetBool("notification." + input.GetName() + ".enabled") {
+			if enabledInput == "" {
+				enabledInput = input.GetName()
+			} else {
+				glog.Fatalf("Notification input is enabled for both %s and %s", enabledInput, input.GetName())
+			}
+		}
+	}
+}

weed/command/master.go

@@ -42,7 +42,7 @@ var (
 	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
 	// mTimeout                = cmdMaster.Flag.Int("idleTimeout", 30, "connection idle seconds")
 	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	garbageThreshold      = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
+	garbageThreshold      = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	masterSecureKey       = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")

weed/command/mount.go

@@ -20,7 +20,9 @@ type MountOptions struct {
 }
 
 var (
-	mountOptions MountOptions
+	mountOptions    MountOptions
+	mountCpuProfile *string
+	mountMemProfile *string
 )
 
 func init() {
@@ -29,12 +31,14 @@ func init() {
 	mountOptions.filerGrpcPort = cmdMount.Flag.Int("filer.grpc.port", 0, "filer grpc server listen port, default to http port + 10000")
 	mountOptions.filerMountRootPath = cmdMount.Flag.String("filer.path", "/", "mount this remote path from filer server")
 	mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory")
-	mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 1000, "limit sub dir listing size")
+	mountOptions.dirListingLimit = cmdMount.Flag.Int("dirListLimit", 100000, "limit directory listing size")
 	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
-	mountOptions.replication = cmdMount.Flag.String("replication", "000", "replication to create to files")
+	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 16, "local write buffer size, also chunk large files")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
+	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
+	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
 }
 
 var cmdMount = &Command{

weed/command/mount_std.go

@@ -5,13 +5,14 @@ package command
 import (
 	"fmt"
 	"runtime"
+	"strings"
+	"time"
 
 	"bazil.org/fuse"
 	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"strings"
 )
 
 func runMount(cmd *Command, args []string) bool {
@@ -27,10 +28,13 @@ func runMount(cmd *Command, args []string) bool {
 
 	fuse.Unmount(*mountOptions.dir)
 
+	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
+
 	c, err := fuse.Mount(
 		*mountOptions.dir,
 		fuse.VolumeName("SeaweedFS"),
 		fuse.FSName("SeaweedFS"),
+		fuse.Subtype("SeaweedFS"),
 		fuse.NoAppleDouble(),
 		fuse.NoAppleXattr(),
 		fuse.ExclCreate(),
@@ -38,7 +42,7 @@ func runMount(cmd *Command, args []string) bool {
 		fuse.AllowOther(),
 		fuse.AllowSUID(),
 		fuse.DefaultPermissions(),
-		// fuse.MaxReadahead(1024*128), // TODO: not tested yet, possibly improving read performance
+		fuse.MaxReadahead(1024*128),
 		fuse.AsyncRead(),
 		fuse.WritebackCache(),
 	)
@@ -72,6 +76,7 @@ func runMount(cmd *Command, args []string) bool {
 		ChunkSizeLimit:     int64(*mountOptions.chunkSizeLimitMB) * 1024 * 1024,
 		DataCenter:         *mountOptions.dataCenter,
 		DirListingLimit:    *mountOptions.dirListingLimit,
+		EntryCacheTtl:      3 * time.Second,
 	}))
 	if err != nil {
 		fuse.Unmount(*mountOptions.dir)

weed/command/scaffold.go

@@ -19,7 +19,7 @@ var cmdScaffold = &Command{
 
 var (
 	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
-	config     = cmdScaffold.Flag.String("config", "filer", "the configuration file to generate")
+	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication] the configuration file to generate")
 )
 
 func runScaffold(cmd *Command, args []string) bool {
@@ -28,6 +28,10 @@ func runScaffold(cmd *Command, args []string) bool {
 	switch *config {
 	case "filer":
 		content = FILER_TOML_EXAMPLE
+	case "notification":
+		content = NOTIFICATION_TOML_EXAMPLE
+	case "replication":
+		content = REPLICATION_TOML_EXAMPLE
 	}
 	if content == "" {
 		println("need a valid -config option")
@@ -35,7 +39,7 @@ func runScaffold(cmd *Command, args []string) bool {
 	}
 
 	if *outputPath != "" {
-		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0x755)
+		ioutil.WriteFile(filepath.Join(*outputPath, *config+".toml"), []byte(content), 0644)
 	} else {
 		println(content)
 	}
@@ -45,6 +49,11 @@ func runScaffold(cmd *Command, args []string) bool {
 const (
 	FILER_TOML_EXAMPLE = `
 # A sample TOML config file for SeaweedFS filer store
+# Used with "weed filer" or "weed server -filer"
+# Put this file to one of the location, with descending priority
+#    ./filer.toml
+#    $HOME/.seaweedfs/filer.toml
+#    /etc/seaweedfs/filer.toml
 
 [memory]
 # local in memory, mostly for testing purpose
@@ -52,7 +61,7 @@ enabled = false
 
 [leveldb]
 # local on disk, mostly for simple single-machine setup, fairly scalable
-enabled = false
+enabled = true
 dir = "."					# directory to store level db files
 
 ####################################################
@@ -67,7 +76,7 @@ dir = "."					# directory to store level db files
 #   meta        BLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
-enabled = true
+enabled = false
 hostname = "localhost"
 port = 3306
 username = "root"
@@ -108,7 +117,7 @@ hosts=[
 ]
 
 [redis]
-enabled = true
+enabled = false
 address  = "localhost:6379"
 password = ""
 db = 0
@@ -124,13 +133,24 @@ addresses = [
     "localhost:30006",
 ]
 
+`
+
+	NOTIFICATION_TOML_EXAMPLE = `
+# A sample TOML config file for SeaweedFS filer store
+# Used by both "weed filer" or "weed server -filer" and "weed filer.replicate"
+# Put this file to one of the location, with descending priority
+#    ./notification.toml
+#    $HOME/.seaweedfs/notification.toml
+#    /etc/seaweedfs/notification.toml
 
 ####################################################
 # notification
-# sends filer updates for each file to an external message queue
+# send and receive filer updates for each file to an external message queue
 ####################################################
 [notification.log]
-enabled = true
+# this is only for debugging perpose and does not work with "weed filer.replicate"
+enabled = false
+
 
 [notification.kafka]
 enabled = false
@@ -138,6 +158,86 @@ hosts = [
   "localhost:9092"
 ]
 topic = "seaweedfs_filer"
+offsetFile = "./last.offset"
+offsetSaveIntervalSeconds = 10
+
+
+[notification.aws_sqs]
+# experimental, let me know if it works
+enabled = false
+aws_access_key_id     = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""        # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+sqs_queue_name = "my_filer_queue" # an existing queue name
+
+
+[notification.google_pub_sub]
+# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
+enabled = false
+google_application_credentials = "/path/to/x.json" # path to json credential file
+project_id = ""                       # an existing project id
+topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists
+
+`
+
+	REPLICATION_TOML_EXAMPLE = `
+# A sample TOML config file for replicating SeaweedFS filer
+# Used with "weed filer.replicate"
+# Put this file to one of the location, with descending priority
+#    ./replication.toml
+#    $HOME/.seaweedfs/replication.toml
+#    /etc/seaweedfs/replication.toml
+
+[source.filer]
+enabled = true
+grpcAddress = "localhost:18888"
+# all files under this directory tree are replicated.
+# this is not a directory on your hard drive, but on your filer.
+# i.e., all files with this "prefix" are sent to notification message queue.
+directory = "/buckets"    
+
+[sink.filer]
+enabled = false
+grpcAddress = "localhost:18888"
+# all replicated files are under this directory tree
+# this is not a directory on your hard drive, but on your filer.     
+# i.e., all received files will be "prefixed" to this directory.
+directory = "/backup"    
+replication = ""
+collection = ""
+ttlSec = 0
+
+[sink.s3]
+# read credentials doc at https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/sessions.html
+# default loads credentials from the shared credentials file (~/.aws/credentials). 
+enabled = false
+aws_access_key_id     = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+aws_secret_access_key = ""     # if empty, loads from the shared credentials file (~/.aws/credentials).
+region = "us-east-2"
+bucket = "your_bucket_name"    # an existing bucket
+directory = "/"                # destination directory
+
+[sink.google_cloud_storage]
+# read credentials doc at https://cloud.google.com/docs/authentication/getting-started
+enabled = false
+google_application_credentials = "/path/to/x.json" # path to json credential file
+bucket = "your_bucket_seaweedfs"    # an existing bucket
+directory = "/"                     # destination directory
+
+[sink.azure]
+# experimental, let me know if it works
+enabled = false
+account_name = ""
+account_key  = ""
+container = "mycontainer"      # an existing container
+directory = "/"                # destination directory
+
+[sink.backblaze]
+enabled = false
+b2_account_id = ""
+b2_master_application_key  = ""
+bucket = "mybucket"            # an existing bucket
+directory = "/"                # destination directory
 
 `
 )

weed/command/server.go

@@ -13,7 +13,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
 	"github.com/soheilhy/cmux"
@@ -22,6 +21,7 @@ import (
 
 type ServerOptions struct {
 	cpuprofile *string
+	v          VolumeServerOptions
 }
 
 var (
@@ -60,21 +60,15 @@ var (
 	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
 	serverPeers                   = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
 	serverSecureKey               = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
-	serverGarbageThreshold        = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
+	serverGarbageThreshold        = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterVolumePreallocate       = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
-	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
-	volumePublicPort              = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
 	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
 	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
-	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
-	volumeIndexType               = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
-	volumeFixJpgOrientation       = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.")
-	volumeReadRedirect            = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
-	volumeServerPublicUrl         = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
+	pulseSeconds                  = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")
 
 	serverWhiteList []string
@@ -91,6 +85,14 @@ func init() {
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
 	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit")
 	filerOptions.dirListingLimit = cmdServer.Flag.Int("filer.dirListLimit", 1000, "limit sub dir listing size")
+
+	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
+	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
+	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
+	serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
+	serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.")
+	serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address")
+
 }
 
 func runServer(cmd *Command, args []string) bool {
@@ -110,6 +112,14 @@ func runServer(cmd *Command, args []string) bool {
 
 	master := *serverIp + ":" + strconv.Itoa(*masterPort)
 	filerOptions.ip = serverIp
+	serverOptions.v.ip = serverIp
+	serverOptions.v.bindIp = serverBindIp
+	serverOptions.v.masters = &master
+	serverOptions.v.idleConnectionTimeout = serverTimeout
+	serverOptions.v.maxCpu = serverMaxCpu
+	serverOptions.v.dataCenter = serverDataCenter
+	serverOptions.v.rack = serverRack
+	serverOptions.v.pulseSeconds = pulseSeconds
 
 	filerOptions.dataCenter = serverDataCenter
 
@@ -117,33 +127,12 @@ func runServer(cmd *Command, args []string) bool {
 		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
 	}
 
-	if *volumePublicPort == 0 {
-		*volumePublicPort = *volumePort
-	}
-
 	if *serverMaxCpu < 1 {
 		*serverMaxCpu = runtime.NumCPU()
 	}
 	runtime.GOMAXPROCS(*serverMaxCpu)
 
 	folders := strings.Split(*volumeDataFolders, ",")
-	maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
-	var maxCounts []int
-	for _, maxString := range maxCountStrings {
-		if max, e := strconv.Atoi(maxString); e == nil {
-			maxCounts = append(maxCounts, max)
-		} else {
-			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
-		}
-	}
-	if len(folders) != len(maxCounts) {
-		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
-	}
-	for _, folder := range folders {
-		if err := util.TestFolderWritable(folder); err != nil {
-			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
-		}
-	}
 
 	if *masterVolumeSizeLimitMB > 30*1000 {
 		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
@@ -155,6 +144,7 @@ func runServer(cmd *Command, args []string) bool {
 	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
 		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
 	}
+	filerOptions.defaultLevelDbDirectory = masterMetaFolder
 
 	if *serverWhiteListOption != "" {
 		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
@@ -164,7 +154,7 @@ func runServer(cmd *Command, args []string) bool {
 		go func() {
 			time.Sleep(1 * time.Second)
 
-			filerOptions.start()
+			filerOptions.startFiler()
 
 		}()
 	}
@@ -179,11 +169,11 @@ func runServer(cmd *Command, args []string) bool {
 		r := mux.NewRouter()
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
 			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
-			*volumePulse, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
+			*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
 			serverWhiteList, *serverSecureKey,
 		)
 
-		glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
+		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
 		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
 		if e != nil {
 			glog.Fatalf("Master startup error: %v", e)
@@ -193,7 +183,7 @@ func runServer(cmd *Command, args []string) bool {
 			raftWaitForMaster.Wait()
 			time.Sleep(100 * time.Millisecond)
 			myAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
-			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
+			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
 			ms.SetRaftServer(raftServer)
 			volumeWait.Done()
 		}()
@@ -224,65 +214,8 @@ func runServer(cmd *Command, args []string) bool {
 
 	volumeWait.Wait()
 	time.Sleep(100 * time.Millisecond)
-	if *volumePublicPort == 0 {
-		*volumePublicPort = *volumePort
-	}
-	if *volumeServerPublicUrl == "" {
-		*volumeServerPublicUrl = *serverIp + ":" + strconv.Itoa(*volumePublicPort)
-	}
-	isSeperatedPublicPort := *volumePublicPort != *volumePort
-	volumeMux := http.NewServeMux()
-	publicVolumeMux := volumeMux
-	if isSeperatedPublicPort {
-		publicVolumeMux = http.NewServeMux()
-	}
-	volumeNeedleMapKind := storage.NeedleMapInMemory
-	switch *volumeIndexType {
-	case "leveldb":
-		volumeNeedleMapKind = storage.NeedleMapLevelDb
-	case "boltdb":
-		volumeNeedleMapKind = storage.NeedleMapBoltDb
-	case "btree":
-		volumeNeedleMapKind = storage.NeedleMapBtree
-	}
-	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
-		*serverIp, *volumePort, *volumeServerPublicUrl,
-		folders, maxCounts,
-		volumeNeedleMapKind,
-		[]string{master}, *volumePulse, *serverDataCenter, *serverRack,
-		serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect,
-	)
 
-	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
-	volumeListener, eListen := util.NewListener(
-		*serverBindIp+":"+strconv.Itoa(*volumePort),
-		time.Duration(*serverTimeout)*time.Second,
-	)
-	if eListen != nil {
-		glog.Fatalf("Volume server listener error: %v", eListen)
-	}
-	if isSeperatedPublicPort {
-		publicListeningAddress := *serverIp + ":" + strconv.Itoa(*volumePublicPort)
-		glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "public at", publicListeningAddress)
-		publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*serverTimeout)*time.Second)
-		if e != nil {
-			glog.Fatalf("Volume server listener error:%v", e)
-		}
-		go func() {
-			if e := http.Serve(publicListener, publicVolumeMux); e != nil {
-				glog.Fatalf("Volume server fail to serve public: %v", e)
-			}
-		}()
-	}
-
-	util.OnInterrupt(func() {
-		volumeServer.Shutdown()
-		pprof.StopCPUProfile()
-	})
-
-	if e := http.Serve(volumeListener, volumeMux); e != nil {
-		glog.Fatalf("Volume server fail to serve:%v", e)
-	}
+	serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption)
 
 	return true
 }

weed/command/upload.go

@@ -53,10 +53,6 @@ var cmdUpload = &Command{
   All files under the folder and subfolders will be uploaded, each with its own file key.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
   If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
   The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
 

weed/command/volume.go

@@ -4,14 +4,17 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"runtime/pprof"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc/reflection"
 )
 
 var (
@@ -54,7 +57,7 @@ func init() {
 	v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name")
 	v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
 	v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb|btree] mode for memory~performance balance.")
-	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.")
+	v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
 	v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.")
 	v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
 	v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file")
@@ -81,9 +84,16 @@ func runVolume(cmd *Command, args []string) bool {
 	runtime.GOMAXPROCS(*v.maxCpu)
 	util.SetupProfiling(*v.cpuProfile, *v.memProfile)
 
+	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption)
+
+	return true
+}
+
+func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string) {
+
 	//Set multiple folders and each folder's max volume count limit'
-	v.folders = strings.Split(*volumeFolders, ",")
-	maxCountStrings := strings.Split(*maxVolumeCounts, ",")
+	v.folders = strings.Split(volumeFolders, ",")
+	maxCountStrings := strings.Split(maxVolumeCounts, ",")
 	for _, maxString := range maxCountStrings {
 		if max, e := strconv.Atoi(maxString); e == nil {
 			v.folderMaxLimits = append(v.folderMaxLimits, max)
@@ -101,8 +111,8 @@ func runVolume(cmd *Command, args []string) bool {
 	}
 
 	//security related white list configuration
-	if *volumeWhiteListOption != "" {
-		v.whiteList = strings.Split(*volumeWhiteListOption, ",")
+	if volumeWhiteListOption != "" {
+		v.whiteList = strings.Split(volumeWhiteListOption, ",")
 	}
 
 	if *v.ip == "" {
@@ -145,7 +155,7 @@ func runVolume(cmd *Command, args []string) bool {
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
-	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)
+	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.VERSION, listeningAddress)
 	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)
@@ -166,10 +176,22 @@ func runVolume(cmd *Command, args []string) bool {
 
 	util.OnInterrupt(func() {
 		volumeServer.Shutdown()
+		pprof.StopCPUProfile()
 	})
 
+	// starting grpc server
+	grpcPort := *v.port + 10000
+	grpcL, err := util.NewListener(*v.bindIp+":"+strconv.Itoa(grpcPort), 0)
+	if err != nil {
+		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
+	}
+	grpcS := util.NewGrpcServer()
+	volume_server_pb.RegisterVolumeServerServer(grpcS, volumeServer)
+	reflection.Register(grpcS)
+	go grpcS.Serve(grpcL)
+
 	if e := http.Serve(listener, volumeMux); e != nil {
 		glog.Fatalf("Volume server fail to serve: %v", e)
 	}
-	return true
+
 }

weed/filer2/configuration.go

@@ -13,6 +13,8 @@ var (
 
 func (f *Filer) LoadConfiguration(config *viper.Viper) {
 
+	validateOneEnabledStore(config)
+
 	for _, store := range Stores {
 		if config.GetBool(store.GetName() + ".enabled") {
 			viperSub := config.Sub(store.GetName())
@@ -34,3 +36,16 @@ func (f *Filer) LoadConfiguration(config *viper.Viper) {
 
 	os.Exit(-1)
 }
+
+func validateOneEnabledStore(config *viper.Viper) {
+	enabledStore := ""
+	for _, store := range Stores {
+		if config.GetBool(store.GetName() + ".enabled") {
+			if enabledStore == "" {
+				enabledStore = store.GetName()
+			} else {
+				glog.Fatalf("Filer store is enabled for both %s and %s", enabledStore, store.GetName())
+			}
+		}
+	}
+}

weed/filer2/entry.go

@@ -17,6 +17,8 @@ type Attr struct {
 	Replication string      // replication
 	Collection  string      // collection name
 	TtlSec      int32       // ttl in seconds
+	UserName    string
+	GroupNames  []string
 }
 
 func (attr Attr) IsDirectory() bool {
@@ -43,3 +45,15 @@ func (entry *Entry) Timestamp() time.Time {
 		return entry.Mtime
 	}
 }
+
+func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
+	if entry == nil {
+		return nil
+	}
+	return &filer_pb.Entry{
+		Name:        string(entry.FullPath),
+		IsDirectory: entry.IsDirectory(),
+		Attributes:  EntryAttributeToPb(entry),
+		Chunks:      entry.Chunks,
+	}
+}

weed/filer2/entry_codec.go

@@ -44,6 +44,8 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
 		Collection:  entry.Attr.Collection,
 		Replication: entry.Attr.Replication,
 		TtlSec:      entry.Attr.TtlSec,
+		UserName:    entry.Attr.UserName,
+		GroupName:   entry.Attr.GroupNames,
 	}
 }
 
@@ -60,6 +62,8 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
 	t.Collection = attr.Collection
 	t.Replication = attr.Replication
 	t.TtlSec = attr.TtlSec
+	t.UserName = attr.UserName
+	t.GroupNames = attr.GroupName
 
 	return t
 }

weed/filer2/filechunks.go

@@ -1,8 +1,10 @@
 package filer2
 
 import (
-	"math"
+	"fmt"
+	"hash/fnv"
 	"sort"
+	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
@@ -17,6 +19,18 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
 	return
 }
 
+func ETag(chunks []*filer_pb.FileChunk) (etag string) {
+	if len(chunks) == 1 {
+		return chunks[0].ETag
+	}
+
+	h := fnv.New32a()
+	for _, c := range chunks {
+		h.Write([]byte(c.ETag))
+	}
+	return fmt.Sprintf("%x", h.Sum32())
+}
+
 func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
 
 	visibles := nonOverlappingVisibleIntervals(chunks)
@@ -56,6 +70,7 @@ type ChunkView struct {
 	Offset      int64
 	Size        uint64
 	LogicOffset int64
+	IsFullChunk bool
 }
 
 func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) {
@@ -66,11 +81,13 @@ func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views
 
 	for _, chunk := range visibles {
 		if chunk.start <= offset && offset < chunk.stop && offset < stop {
+			isFullChunk := chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop
 			views = append(views, &ChunkView{
 				FileId:      chunk.fileId,
 				Offset:      offset - chunk.start, // offset is the data starting location in this file id
 				Size:        uint64(min(chunk.stop, stop) - offset),
 				LogicOffset: offset,
+				IsFullChunk: isFullChunk,
 			})
 			offset = min(chunk.stop, stop)
 		}
@@ -89,137 +106,90 @@ func logPrintf(name string, visibles []*visibleInterval) {
 	*/
 }
 
+var bufPool = sync.Pool{
+	New: func() interface{} {
+		return new(visibleInterval)
+	},
+}
+
+func mergeIntoVisibles(visibles, newVisibles []*visibleInterval, chunk *filer_pb.FileChunk) []*visibleInterval {
+
+	newV := newVisibleInterval(
+		chunk.Offset,
+		chunk.Offset+int64(chunk.Size),
+		chunk.FileId,
+		chunk.Mtime,
+		true,
+	)
+
+	length := len(visibles)
+	if length == 0 {
+		return append(visibles, newV)
+	}
+	last := visibles[length-1]
+	if last.stop <= chunk.Offset {
+		return append(visibles, newV)
+	}
+
+	logPrintf("  before", visibles)
+	for _, v := range visibles {
+		if v.start < chunk.Offset && chunk.Offset < v.stop {
+			newVisibles = append(newVisibles, newVisibleInterval(
+				v.start,
+				chunk.Offset,
+				v.fileId,
+				v.modifiedTime,
+				false,
+			))
+		}
+		chunkStop := chunk.Offset + int64(chunk.Size)
+		if v.start < chunkStop && chunkStop < v.stop {
+			newVisibles = append(newVisibles, newVisibleInterval(
+				chunkStop,
+				v.stop,
+				v.fileId,
+				v.modifiedTime,
+				false,
+			))
+		}
+		if chunkStop <= v.start || v.stop <= chunk.Offset {
+			newVisibles = append(newVisibles, v)
+		}
+	}
+	newVisibles = append(newVisibles, newV)
+
+	logPrintf("  append", newVisibles)
+
+	for i := len(newVisibles) - 1; i >= 0; i-- {
+		if i > 0 && newV.start < newVisibles[i-1].start {
+			newVisibles[i] = newVisibles[i-1]
+		} else {
+			newVisibles[i] = newV
+			break
+		}
+	}
+	logPrintf("  sorted", newVisibles)
+
+	return newVisibles
+}
+
 func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) {
 
 	sort.Slice(chunks, func(i, j int) bool {
-		if chunks[i].Offset < chunks[j].Offset {
-			return true
-		}
-		if chunks[i].Offset == chunks[j].Offset {
-			return chunks[i].Mtime < chunks[j].Mtime
-		}
-		return false
+		return chunks[i].Mtime < chunks[j].Mtime
 	})
 
-	if len(chunks) == 0 {
-		return
-	}
-
-	var parallelIntervals, intervals []*visibleInterval
-	var minStopInterval, upToDateInterval *visibleInterval
-	watermarkStart := chunks[0].Offset
+	var newVislbles []*visibleInterval
 	for _, chunk := range chunks {
-		// log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
-		logPrintf("parallelIntervals", parallelIntervals)
-		for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset {
-			logPrintf("parallelIntervals loop 1", parallelIntervals)
-			logPrintf("parallelIntervals loop 1 intervals", intervals)
-			minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
-			nextStop := min(minStopInterval.stop, chunk.Offset)
-			intervals = append(intervals, newVisibleInterval(
-				max(watermarkStart, minStopInterval.start),
-				nextStop,
-				upToDateInterval.fileId,
-				upToDateInterval.modifiedTime,
-			))
-			watermarkStart = nextStop
-			logPrintf("parallelIntervals loop intervals =>", intervals)
+		newVislbles = mergeIntoVisibles(visibles, newVislbles, chunk)
+		t := visibles[:0]
+		visibles = newVislbles
+		newVislbles = t
+
+		logPrintf("add", visibles)
 
-			// remove processed intervals, possibly multiple
-			var remaining []*visibleInterval
-			for _, interval := range parallelIntervals {
-				if interval.stop != watermarkStart {
-					remaining = append(remaining, interval)
-				}
-			}
-			parallelIntervals = remaining
-			logPrintf("parallelIntervals loop 2", parallelIntervals)
-			logPrintf("parallelIntervals loop 2 intervals", intervals)
-		}
-		parallelIntervals = append(parallelIntervals, newVisibleInterval(
-			chunk.Offset,
-			chunk.Offset+int64(chunk.Size),
-			chunk.FileId,
-			chunk.Mtime,
-		))
 	}
 
-	logPrintf("parallelIntervals loop 3", parallelIntervals)
-	logPrintf("parallelIntervals loop 3 intervals", intervals)
-	for len(parallelIntervals) > 0 {
-		minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals)
-		intervals = append(intervals, newVisibleInterval(
-			max(watermarkStart, minStopInterval.start),
-			minStopInterval.stop,
-			upToDateInterval.fileId,
-			upToDateInterval.modifiedTime,
-		))
-		watermarkStart = minStopInterval.stop
-
-		// remove processed intervals, possibly multiple
-		var remaining []*visibleInterval
-		for _, interval := range parallelIntervals {
-			if interval.stop != watermarkStart {
-				remaining = append(remaining, interval)
-			}
-		}
-		parallelIntervals = remaining
-	}
-	logPrintf("parallelIntervals loop 4", parallelIntervals)
-	logPrintf("intervals", intervals)
-
-	// merge connected intervals, now the intervals are non-intersecting
-	var lastIntervalIndex int
-	var prevIntervalIndex int
-	for i, interval := range intervals {
-		if i == 0 {
-			prevIntervalIndex = i
-			lastIntervalIndex = i
-			continue
-		}
-		if intervals[i-1].fileId != interval.fileId ||
-			intervals[i-1].stop < intervals[i].start {
-			visibles = append(visibles, newVisibleInterval(
-				intervals[prevIntervalIndex].start,
-				intervals[i-1].stop,
-				intervals[prevIntervalIndex].fileId,
-				intervals[prevIntervalIndex].modifiedTime,
-			))
-			prevIntervalIndex = i
-		}
-		lastIntervalIndex = i
-		logPrintf("intervals loop 1 visibles", visibles)
-	}
-
-	visibles = append(visibles, newVisibleInterval(
-		intervals[prevIntervalIndex].start,
-		intervals[lastIntervalIndex].stop,
-		intervals[prevIntervalIndex].fileId,
-		intervals[prevIntervalIndex].modifiedTime,
-	))
-
-	logPrintf("visibles", visibles)
-
-	return
-}
-
-func findMinStopInterval(intervals []*visibleInterval) (minStopInterval, upToDateInterval *visibleInterval) {
-	var latestMtime int64
-	latestIntervalIndex := 0
-	minStop := int64(math.MaxInt64)
-	minIntervalIndex := 0
-	for i, interval := range intervals {
-		if minStop > interval.stop {
-			minIntervalIndex = i
-			minStop = interval.stop
-		}
-		if latestMtime < interval.modifiedTime {
-			latestMtime = interval.modifiedTime
-			latestIntervalIndex = i
-		}
-	}
-	minStopInterval = intervals[minIntervalIndex]
-	upToDateInterval = intervals[latestIntervalIndex]
 	return
 }
 
@@ -231,10 +201,17 @@ type visibleInterval struct {
 	stop         int64
 	modifiedTime int64
 	fileId       string
+	isFullChunk  bool
 }
 
-func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64) *visibleInterval {
-	return &visibleInterval{start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime}
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) *visibleInterval {
+	return &visibleInterval{
+		start:        start,
+		stop:         stop,
+		fileId:       fileId,
+		modifiedTime: modifiedTime,
+		isFullChunk:  isFullChunk,
+	}
 }
 
 func min(x, y int64) int64 {
@@ -243,10 +220,3 @@ func min(x, y int64) int64 {
 	}
 	return y
 }
-
-func max(x, y int64) int64 {
-	if x > y {
-		return x
-	}
-	return y
-}

weed/filer2/filechunks_test.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"testing"
 
+	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
@@ -15,20 +16,49 @@ func TestCompactFileChunks(t *testing.T) {
 		{Offset: 110, Size: 200, FileId: "jkl", Mtime: 300},
 	}
 
-	compacted, garbarge := CompactFileChunks(chunks)
-
-	log.Printf("Compacted: %+v", compacted)
-	log.Printf("Garbage  : %+v", garbarge)
+	compacted, garbage := CompactFileChunks(chunks)
 
 	if len(compacted) != 3 {
 		t.Fatalf("unexpected compacted: %d", len(compacted))
 	}
-	if len(garbarge) != 1 {
-		t.Fatalf("unexpected garbarge: %d", len(garbarge))
+	if len(garbage) != 1 {
+		t.Fatalf("unexpected garbage: %d", len(garbage))
 	}
 
 }
 
+func TestCompactFileChunks2(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{Offset: 0, Size: 100, FileId: "abc", Mtime: 50},
+		{Offset: 100, Size: 100, FileId: "def", Mtime: 100},
+		{Offset: 200, Size: 100, FileId: "ghi", Mtime: 200},
+		{Offset: 0, Size: 100, FileId: "abcf", Mtime: 300},
+		{Offset: 50, Size: 100, FileId: "fhfh", Mtime: 400},
+		{Offset: 100, Size: 100, FileId: "yuyu", Mtime: 500},
+	}
+
+	k := 3
+
+	for n := 0; n < k; n++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), Mtime: int64(n),
+		})
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), Mtime: int64(n + k),
+		})
+	}
+
+	compacted, garbage := CompactFileChunks(chunks)
+
+	if len(compacted) != 4 {
+		t.Fatalf("unexpected compacted: %d", len(compacted))
+	}
+	if len(garbage) != 8 {
+		t.Fatalf("unexpected garbage: %d", len(garbage))
+	}
+}
+
 func TestIntervalMerging(t *testing.T) {
 
 	testcases := []struct {
@@ -135,6 +165,23 @@ func TestIntervalMerging(t *testing.T) {
 				{start: 11534336, stop: 14376529, fileId: "7,0299ad723803"},
 			},
 		},
+		// case 8: real bug
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 77824, FileId: "4,0b3df938e301", Mtime: 123},
+				{Offset: 471040, Size: 472225 - 471040, FileId: "6,0b3e0650019c", Mtime: 130},
+				{Offset: 77824, Size: 208896 - 77824, FileId: "4,0b3f0c7202f0", Mtime: 140},
+				{Offset: 208896, Size: 339968 - 208896, FileId: "2,0b4031a72689", Mtime: 150},
+				{Offset: 339968, Size: 471040 - 339968, FileId: "3,0b416a557362", Mtime: 160},
+			},
+			Expected: []*visibleInterval{
+				{start: 0, stop: 77824, fileId: "4,0b3df938e301"},
+				{start: 77824, stop: 208896, fileId: "4,0b3f0c7202f0"},
+				{start: 208896, stop: 339968, fileId: "2,0b4031a72689"},
+				{start: 339968, stop: 471040, fileId: "3,0b416a557362"},
+				{start: 471040, stop: 472225, fileId: "6,0b3e0650019c"},
+			},
+		},
 	}
 
 	for i, testcase := range testcases {
@@ -161,6 +208,7 @@ func TestIntervalMerging(t *testing.T) {
 		if len(intervals) != len(testcase.Expected) {
 			t.Fatalf("failed to compact test case %d, len %d expected %d", i, len(intervals), len(testcase.Expected))
 		}
+
 	}
 
 }
@@ -314,3 +362,23 @@ func TestChunksReading(t *testing.T) {
 	}
 
 }
+
+func BenchmarkCompactFileChunks(b *testing.B) {
+
+	var chunks []*filer_pb.FileChunk
+
+	k := 1024
+
+	for n := 0; n < k; n++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), Mtime: int64(n),
+		})
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), Mtime: int64(n + k),
+		})
+	}
+
+	for n := 0; n < b.N; n++ {
+		CompactFileChunks(chunks)
+	}
+}

weed/filer2/filer.go

@@ -3,30 +3,39 @@ package filer2
 import (
 	"context"
 	"fmt"
+	"math"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/operation"
-	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"github.com/karlseguin/ccache"
-	"math"
+)
+
+var (
+	OS_UID = uint32(os.Getuid())
+	OS_GID = uint32(os.Getgid())
 )
 
 type Filer struct {
-	store          FilerStore
-	directoryCache *ccache.Cache
-	MasterClient   *wdclient.MasterClient
+	store              FilerStore
+	directoryCache     *ccache.Cache
+	MasterClient       *wdclient.MasterClient
+	fileIdDeletionChan chan string
 }
 
 func NewFiler(masters []string) *Filer {
-	return &Filer{
-		directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
-		MasterClient:   wdclient.NewMasterClient(context.Background(), "filer", masters),
+	f := &Filer{
+		directoryCache:     ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
+		MasterClient:       wdclient.NewMasterClient(context.Background(), "filer", masters),
+		fileIdDeletionChan: make(chan string, 4096),
 	}
+
+	go f.loopProcessingDeletion()
+
+	return f
 }
 
 func (f *Filer) SetStore(store FilerStore) {
@@ -47,6 +56,10 @@ func (fs *Filer) KeepConnectedToMaster() {
 
 func (f *Filer) CreateEntry(entry *Entry) error {
 
+	if string(entry.FullPath) == "/" {
+		return nil
+	}
+
 	dirParts := strings.Split(string(entry.FullPath), "/")
 
 	// fmt.Printf("directory parts: %+v\n", dirParts)
@@ -88,11 +101,13 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 			glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode)
 			mkdirErr := f.store.InsertEntry(dirEntry)
 			if mkdirErr != nil {
-				return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
+				if _, err := f.FindEntry(FullPath(dirPath)); err == ErrNotFound {
+					return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
+				}
+			} else {
+				f.NotifyUpdateEvent(nil, dirEntry, false)
 			}
 
-			f.NotifyUpdateEvent(nil, dirEntry)
-
 		} else if !dirEntry.IsDirectory() {
 			return fmt.Errorf("%s is a file", dirPath)
 		}
@@ -121,22 +136,51 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 
 	oldEntry, _ := f.FindEntry(entry.FullPath)
 
-	if err := f.store.InsertEntry(entry); err != nil {
-		return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
+	if oldEntry == nil {
+		if err := f.store.InsertEntry(entry); err != nil {
+			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
+		}
+	} else {
+		if err := f.UpdateEntry(oldEntry, entry); err != nil {
+			return fmt.Errorf("update entry %s: %v", entry.FullPath, err)
+		}
 	}
 
-	f.NotifyUpdateEvent(oldEntry, entry)
+	f.NotifyUpdateEvent(oldEntry, entry, true)
 
 	f.deleteChunksIfNotNew(oldEntry, entry)
 
 	return nil
 }
 
-func (f *Filer) UpdateEntry(entry *Entry) (err error) {
+func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
+	if oldEntry != nil {
+		if oldEntry.IsDirectory() && !entry.IsDirectory() {
+			return fmt.Errorf("existing %s is a directory", entry.FullPath)
+		}
+		if !oldEntry.IsDirectory() && entry.IsDirectory() {
+			return fmt.Errorf("existing %s is a file", entry.FullPath)
+		}
+	}
 	return f.store.UpdateEntry(entry)
 }
 
 func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
+
+	now := time.Now()
+
+	if string(p) == "/" {
+		return &Entry{
+			FullPath: p,
+			Attr: Attr{
+				Mtime:  now,
+				Crtime: now,
+				Mode:   os.ModeDir | 0755,
+				Uid:    OS_UID,
+				Gid:    OS_GID,
+			},
+		}, nil
+	}
 	return f.store.FindEntry(p)
 }
 
@@ -151,24 +195,40 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 		if isRecursive {
 			limit = math.MaxInt32
 		}
-		entries, err := f.ListDirectoryEntries(p, "", false, limit)
-		if err != nil {
-			return fmt.Errorf("list folder %s: %v", p, err)
-		}
-		if isRecursive {
-			for _, sub := range entries {
-				f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
+		lastFileName := ""
+		includeLastFile := false
+		for limit > 0 {
+			entries, err := f.ListDirectoryEntries(p, lastFileName, includeLastFile, 1024)
+			if err != nil {
+				return fmt.Errorf("list folder %s: %v", p, err)
 			}
-		} else {
-			if len(entries) > 0 {
-				return fmt.Errorf("folder %s is not empty", p)
+			if len(entries) == 0 {
+				break
+			} else {
+				if isRecursive {
+					for _, sub := range entries {
+						lastFileName = sub.Name()
+						f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
+						limit--
+						if limit <= 0 {
+							break
+						}
+					}
+				} else {
+					if len(entries) > 0 {
+						return fmt.Errorf("folder %s is not empty", p)
+					}
+				}
+				f.cacheDelDirectory(string(p))
+				if len(entries) < 1024 {
+					break
+				}
 			}
 		}
-		f.cacheDelDirectory(string(p))
 	}
 
 	if shouldDeleteChunks {
-		f.deleteChunks(entry.Chunks)
+		f.DeleteChunks(entry.Chunks)
 	}
 
 	if p == "/" {
@@ -176,7 +236,7 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 	}
 	glog.V(3).Infof("deleting entry %v", p)
 
-	f.NotifyUpdateEvent(entry, nil)
+	f.NotifyUpdateEvent(entry, nil, shouldDeleteChunks)
 
 	return f.store.DeleteEntry(p)
 }
@@ -220,45 +280,3 @@ func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) {
 
 	f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute)
 }
-
-func (f *Filer) deleteChunks(chunks []*filer_pb.FileChunk) {
-	for _, chunk := range chunks {
-		f.DeleteFileByFileId(chunk.FileId)
-	}
-}
-
-func (f *Filer) DeleteFileByFileId(fileId string) {
-	fileUrlOnVolume, err := f.MasterClient.LookupFileId(fileId)
-	if err != nil {
-		glog.V(0).Infof("can not find file %s: %v", fileId, err)
-	}
-	if err := operation.DeleteFromVolumeServer(fileUrlOnVolume, ""); err != nil {
-		glog.V(0).Infof("deleting file %s: %v", fileId, err)
-	}
-}
-
-func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
-
-	if oldEntry == nil {
-		return
-	}
-	if newEntry == nil {
-		f.deleteChunks(oldEntry.Chunks)
-	}
-
-	var toDelete []*filer_pb.FileChunk
-
-	for _, oldChunk := range oldEntry.Chunks {
-		found := false
-		for _, newChunk := range newEntry.Chunks {
-			if oldChunk.FileId == newChunk.FileId {
-				found = true
-				break
-			}
-		}
-		if !found {
-			toDelete = append(toDelete, oldChunk)
-		}
-	}
-	f.deleteChunks(toDelete)
-}

weed/filer2/filer_deletion.go

@@ -0,0 +1,88 @@
+package filer2
+
+import (
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+func (f *Filer) loopProcessingDeletion() {
+
+	ticker := time.NewTicker(5 * time.Second)
+
+	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
+		m := make(map[string]operation.LookupResult)
+		for _, vid := range vids {
+			locs := f.MasterClient.GetVidLocations(vid)
+			var locations []operation.Location
+			for _, loc := range locs {
+				locations = append(locations, operation.Location{
+					Url:       loc.Url,
+					PublicUrl: loc.PublicUrl,
+				})
+			}
+			m[vid] = operation.LookupResult{
+				VolumeId:  vid,
+				Locations: locations,
+			}
+		}
+		return m, nil
+	}
+
+	var fileIds []string
+	for {
+		select {
+		case fid := <-f.fileIdDeletionChan:
+			fileIds = append(fileIds, fid)
+			if len(fileIds) >= 4096 {
+				glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
+				operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+				fileIds = fileIds[:0]
+			}
+		case <-ticker.C:
+			if len(fileIds) > 0 {
+				glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
+				operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+				fileIds = fileIds[:0]
+			}
+		}
+	}
+}
+
+func (f *Filer) DeleteChunks(chunks []*filer_pb.FileChunk) {
+	for _, chunk := range chunks {
+		f.fileIdDeletionChan <- chunk.FileId
+	}
+}
+
+func (f *Filer) DeleteFileByFileId(fileId string) {
+	f.fileIdDeletionChan <- fileId
+}
+
+func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
+
+	if oldEntry == nil {
+		return
+	}
+	if newEntry == nil {
+		f.DeleteChunks(oldEntry.Chunks)
+	}
+
+	var toDelete []*filer_pb.FileChunk
+
+	for _, oldChunk := range oldEntry.Chunks {
+		found := false
+		for _, newChunk := range newEntry.Chunks {
+			if oldChunk.FileId == newChunk.FileId {
+				found = true
+				break
+			}
+		}
+		if !found {
+			toDelete = append(toDelete, oldChunk)
+		}
+	}
+	f.DeleteChunks(toDelete)
+}

weed/filer2/filer_notify.go

@@ -1,11 +1,12 @@
 package filer2
 
 import (
-	"github.com/chrislusf/seaweedfs/weed/msgqueue"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/notification"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
-func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry) {
+func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool) {
 	var key string
 	if oldEntry != nil {
 		key = string(oldEntry.FullPath)
@@ -15,28 +16,18 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry) {
 		return
 	}
 
-	if msgqueue.Queue != nil {
+	if notification.Queue != nil {
 
-		msgqueue.Queue.SendMessage(
+		glog.V(3).Infof("notifying entry update %v", key)
+
+		notification.Queue.SendMessage(
 			key,
 			&filer_pb.EventNotification{
-				OldEntry: toProtoEntry(oldEntry),
-				NewEntry: toProtoEntry(newEntry),
+				OldEntry:     oldEntry.ToProtoEntry(),
+				NewEntry:     newEntry.ToProtoEntry(),
+				DeleteChunks: deleteChunks,
 			},
 		)
 
 	}
-
-}
-
-func toProtoEntry(entry *Entry) *filer_pb.Entry {
-	if entry == nil {
-		return nil
-	}
-	return &filer_pb.Entry{
-		Name:        string(entry.FullPath),
-		IsDirectory: entry.IsDirectory(),
-		Attributes:  EntryAttributeToPb(entry),
-		Chunks:      entry.Chunks,
-	}
 }

weed/filer2/filer_notify_test.go

@@ -0,0 +1,51 @@
+package filer2
+
+import (
+	"testing"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/golang/protobuf/proto"
+)
+
+func TestProtoMarshalText(t *testing.T) {
+
+	oldEntry := &Entry{
+		FullPath: FullPath("/this/path/to"),
+		Attr: Attr{
+			Mtime:  time.Now(),
+			Mode:   0644,
+			Uid:    1,
+			Mime:   "text/json",
+			TtlSec: 25,
+		},
+		Chunks: []*filer_pb.FileChunk{
+			&filer_pb.FileChunk{
+				FileId:       "234,2423423422",
+				Offset:       234234,
+				Size:         234,
+				Mtime:        12312423,
+				ETag:         "2342342354",
+				SourceFileId: "23234,2342342342",
+			},
+		},
+	}
+
+	notification := &filer_pb.EventNotification{
+		OldEntry:     oldEntry.ToProtoEntry(),
+		NewEntry:     nil,
+		DeleteChunks: true,
+	}
+
+	text := proto.MarshalTextString(notification)
+
+	notification2 := &filer_pb.EventNotification{}
+	proto.UnmarshalText(text, notification2)
+
+	if notification2.OldEntry.Chunks[0].SourceFileId != notification.OldEntry.Chunks[0].SourceFileId {
+		t.Fatalf("marshal/unmarshal error: %s", text)
+	}
+
+	println(text)
+
+}

weed/filer2/leveldb/leveldb_store.go

@@ -33,11 +33,13 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration) (er
 }
 
 func (store *LevelDBStore) initialize(dir string) (err error) {
+	glog.Infof("filer store dir: %s", dir)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}
 
 	if store.db, err = leveldb.OpenFile(dir, nil); err != nil {
+		glog.Infof("filer store open dir %s: %v", dir, err)
 		return
 	}
 	return

weed/filer2/leveldb/leveldb_store_test.go

@@ -59,3 +59,25 @@ func TestCreateAndFind(t *testing.T) {
 	}
 
 }
+
+func TestEmptyRoot(t *testing.T) {
+	filer := filer2.NewFiler(nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	defer os.RemoveAll(dir)
+	store := &LevelDBStore{}
+	store.initialize(dir)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	// checking one upper directory
+	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	if err != nil {
+		t.Errorf("list entries: %v", err)
+		return
+	}
+	if len(entries) != 0 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}

weed/filesys/dir.go

@@ -31,8 +31,10 @@ var _ = fs.NodeSetattrer(&Dir{})
 
 func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
+	// https://github.com/bazil/fuse/issues/196
+	attr.Valid = time.Second
+
 	if dir.Path == "/" {
-		attr.Valid = time.Second
 		attr.Mode = os.ModeDir | 0777
 		return nil
 	}
@@ -54,19 +56,23 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
 	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		request := &filer_pb.GetEntryAttributesRequest{
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: parent,
 			Name:      name,
-			ParentDir: parent,
 		}
 
 		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
-		resp, err := client.GetEntryAttributes(context, request)
+		resp, err := client.LookupDirectoryEntry(context, request)
 		if err != nil {
 			glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
 			return err
 		}
 
-		dir.attributes = resp.Attributes
+		if resp.Entry != nil {
+			dir.attributes = resp.Entry.Attributes
+		}
+
+		dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
 
 		return nil
 	})
@@ -79,9 +85,6 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 	// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))
 
 	attr.Mode = os.FileMode(dir.attributes.FileMode) | os.ModeDir
-	if dir.Path == "/" && dir.attributes.FileMode == 0 {
-		attr.Valid = time.Second
-	}
 
 	attr.Mtime = time.Unix(dir.attributes.Mtime, 0)
 	attr.Ctime = time.Unix(dir.attributes.Crtime, 0)
@@ -91,55 +94,57 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 	return nil
 }
 
-func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk, attr *filer_pb.FuseAttributes) *File {
+func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
 	return &File{
-		Name:       name,
-		dir:        dir,
-		wfs:        dir.wfs,
-		attributes: attr,
-		Chunks:     chunks,
+		Name:  name,
+		dir:   dir,
+		wfs:   dir.wfs,
+		entry: entry,
 	}
 }
 
 func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
 	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
 
-	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		request := &filer_pb.CreateEntryRequest{
-			Directory: dir.Path,
-			Entry: &filer_pb.Entry{
-				Name:        req.Name,
-				IsDirectory: req.Mode&os.ModeDir > 0,
-				Attributes: &filer_pb.FuseAttributes{
-					Mtime:       time.Now().Unix(),
-					Crtime:      time.Now().Unix(),
-					FileMode:    uint32(req.Mode),
-					Uid:         req.Uid,
-					Gid:         req.Gid,
-					Collection:  dir.wfs.option.Collection,
-					Replication: dir.wfs.option.Replication,
-					TtlSec:      dir.wfs.option.TtlSec,
-				},
+	request := &filer_pb.CreateEntryRequest{
+		Directory: dir.Path,
+		Entry: &filer_pb.Entry{
+			Name:        req.Name,
+			IsDirectory: req.Mode&os.ModeDir > 0,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:       time.Now().Unix(),
+				Crtime:      time.Now().Unix(),
+				FileMode:    uint32(req.Mode),
+				Uid:         req.Uid,
+				Gid:         req.Gid,
+				Collection:  dir.wfs.option.Collection,
+				Replication: dir.wfs.option.Replication,
+				TtlSec:      dir.wfs.option.TtlSec,
 			},
+		},
+	}
+	glog.V(1).Infof("create: %v", request)
+
+	if request.Entry.IsDirectory {
+		if err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+			if _, err := client.CreateEntry(ctx, request); err != nil {
+				glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
+				return fuse.EIO
+			}
+			return nil
+		}); err != nil {
+			return nil, nil, err
 		}
-
-		glog.V(1).Infof("create: %v", request)
-		if _, err := client.CreateEntry(ctx, request); err != nil {
-			glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
-			return fuse.EIO
-		}
-
-		return nil
-	})
-
-	if err == nil {
-		file := dir.newFile(req.Name, nil, &filer_pb.FuseAttributes{})
-		file.isOpen = true
-		return file, dir.wfs.AcquireHandle(file, req.Uid, req.Gid), nil
 	}
 
-	return nil, nil, err
+	file := dir.newFile(req.Name, request.Entry)
+	if !request.Entry.IsDirectory {
+		file.isOpen = true
+	}
+	fh := dir.wfs.AcquireHandle(file, req.Uid, req.Gid)
+	fh.dirtyMetadata = true
+	return file, fh, nil
+
 }
 
 func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
@@ -181,30 +186,40 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err
 func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
 
 	var entry *filer_pb.Entry
-	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		request := &filer_pb.LookupDirectoryEntryRequest{
-			Directory: dir.Path,
-			Name:      req.Name,
-		}
+	item := dir.wfs.listDirectoryEntriesCache.Get(path.Join(dir.Path, req.Name))
+	if item != nil && !item.Expired() {
+		entry = item.Value().(*filer_pb.Entry)
+	}
 
-		glog.V(4).Infof("lookup directory entry: %v", request)
-		resp, err := client.LookupDirectoryEntry(ctx, request)
-		if err != nil {
-			// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
-			return fuse.ENOENT
-		}
+	if entry == nil {
+		err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		entry = resp.Entry
+			request := &filer_pb.LookupDirectoryEntryRequest{
+				Directory: dir.Path,
+				Name:      req.Name,
+			}
 
-		return nil
-	})
+			glog.V(4).Infof("lookup directory entry: %v", request)
+			resp, err := client.LookupDirectoryEntry(ctx, request)
+			if err != nil {
+				// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
+				return fuse.ENOENT
+			}
+
+			entry = resp.Entry
+
+			dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
+
+			return nil
+		})
+	}
 
 	if entry != nil {
 		if entry.IsDirectory {
 			node = &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs, attributes: entry.Attributes}
 		} else {
-			node = dir.newFile(req.Name, entry.Chunks, entry.Attributes)
+			node = dir.newFile(req.Name, entry)
 		}
 
 		resp.EntryValid = time.Duration(0)
@@ -224,27 +239,46 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		request := &filer_pb.ListEntriesRequest{
-			Directory: dir.Path,
-			Limit:     uint32(dir.wfs.option.DirListingLimit),
-		}
+		paginationLimit := 1024
+		remaining := dir.wfs.option.DirListingLimit
 
-		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("list %s: %v", dir.Path, err)
-			return fuse.EIO
-		}
+		lastEntryName := ""
 
-		for _, entry := range resp.Entries {
-			if entry.IsDirectory {
-				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
-				ret = append(ret, dirent)
-			} else {
-				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
-				ret = append(ret, dirent)
+		for remaining >= 0 {
+
+			request := &filer_pb.ListEntriesRequest{
+				Directory:         dir.Path,
+				StartFromFileName: lastEntryName,
+				Limit:             uint32(paginationLimit),
 			}
-			dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry, 300*time.Millisecond)
+
+			glog.V(4).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
+			if err != nil {
+				glog.V(0).Infof("list %s: %v", dir.Path, err)
+				return fuse.EIO
+			}
+
+			cacheTtl := estimatedCacheTtl(len(resp.Entries))
+
+			for _, entry := range resp.Entries {
+				if entry.IsDirectory {
+					dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
+					ret = append(ret, dirent)
+				} else {
+					dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
+					ret = append(ret, dirent)
+				}
+				dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
+				lastEntryName = entry.Name
+			}
+
+			remaining -= len(resp.Entries)
+
+			if len(resp.Entries) < paginationLimit {
+				break
+			}
+
 		}
 
 		return nil
@@ -260,7 +294,6 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
 			Name:         req.Name,
-			IsDirectory:  req.Dir,
 			IsDeleteData: true,
 		}
 
@@ -271,6 +304,8 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 			return fuse.EIO
 		}
 
+		dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
+
 		return nil
 	})
 
@@ -313,7 +348,27 @@ func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fus
 			return fuse.EIO
 		}
 
+		dir.wfs.listDirectoryEntriesCache.Delete(dir.Path)
+
 		return nil
 	})
 
 }
+
+func estimatedCacheTtl(numEntries int) time.Duration {
+	if numEntries < 100 {
+		// 30 ms per entry
+		return 3 * time.Second
+	}
+	if numEntries < 1000 {
+		// 10 ms per entry
+		return 10 * time.Second
+	}
+	if numEntries < 10000 {
+		// 10 ms per entry
+		return 100 * time.Second
+	}
+
+	// 2 ms per entry
+	return time.Duration(numEntries*2) * time.Millisecond
+}

weed/filesys/dir_rename.go

@@ -4,9 +4,11 @@ import (
 	"bazil.org/fuse"
 	"bazil.org/fuse/fs"
 	"context"
+	"math"
+	"path/filepath"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"path/filepath"
 )
 
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
@@ -40,21 +42,37 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParent string, entry *filer_pb.Entry, newParent, newName string) error {
 	if entry.IsDirectory {
 		currentDirPath := filepath.Join(oldParent, entry.Name)
-		request := &filer_pb.ListEntriesRequest{
-			Directory: currentDirPath,
-		}
 
-		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("list %s: %v", oldParent, err)
-			return fuse.EIO
-		}
-
-		for _, item := range resp.Entries {
-			err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
+		lastFileName := ""
+		includeLastFile := false
+		limit := math.MaxInt32
+		for limit > 0 {
+			request := &filer_pb.ListEntriesRequest{
+				Directory:          currentDirPath,
+				StartFromFileName:  lastFileName,
+				InclusiveStartFrom: includeLastFile,
+				Limit:              1024,
+			}
+			glog.V(4).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
 			if err != nil {
-				return err
+				glog.V(0).Infof("list %s: %v", oldParent, err)
+				return fuse.EIO
+			}
+			if len(resp.Entries) == 0 {
+				break
+			}
+
+			for _, item := range resp.Entries {
+				lastFileName = item.Name
+				err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
+				if err != nil {
+					return err
+				}
+				limit--
+			}
+			if len(resp.Entries) < 1024 {
+				break
 			}
 		}
 
@@ -84,7 +102,6 @@ func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParen
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    oldParent,
 			Name:         entry.Name,
-			IsDirectory:  entry.IsDirectory,
 			IsDeleteData: false,
 		}
 

weed/filesys/dirty_page.go

@@ -9,6 +9,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"sync"
 )
 
 type ContinuousDirtyPages struct {
@@ -17,6 +18,7 @@ type ContinuousDirtyPages struct {
 	Size    int64
 	Data    []byte
 	f       *File
+	lock    sync.Mutex
 }
 
 func newDirtyPages(file *File) *ContinuousDirtyPages {
@@ -26,37 +28,24 @@ func newDirtyPages(file *File) *ContinuousDirtyPages {
 	}
 }
 
+func (pages *ContinuousDirtyPages) InitializeToFile(file *File) *ContinuousDirtyPages {
+	if len(pages.Data) != int(file.wfs.option.ChunkSizeLimit) {
+		pages.Data = make([]byte, file.wfs.option.ChunkSizeLimit)
+	}
+	pages.f = file
+	return pages
+}
+
 func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
 
+	pages.lock.Lock()
+	defer pages.lock.Unlock()
+
 	var chunk *filer_pb.FileChunk
 
 	if len(data) > len(pages.Data) {
 		// this is more than what buffer can hold.
-
-		// flush existing
-		if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
-			if chunk != nil {
-				glog.V(4).Infof("%s/%s flush existing [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
-			}
-			chunks = append(chunks, chunk)
-		} else {
-			glog.V(0).Infof("%s/%s failed to flush1 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
-			return
-		}
-		pages.Size = 0
-
-		// flush the big page
-		if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil {
-			if chunk != nil {
-				glog.V(4).Infof("%s/%s flush big request [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
-				chunks = append(chunks, chunk)
-			}
-		} else {
-			glog.V(0).Infof("%s/%s failed to flush2 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
-			return
-		}
-
-		return
+		return pages.flushAndSave(ctx, offset, data)
 	}
 
 	if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) ||
@@ -77,25 +66,75 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da
 			return
 		}
 		pages.Offset = offset
-		pages.Size = int64(len(data))
 		copy(pages.Data, data)
+		pages.Size = int64(len(data))
 		return
 	}
 
-	copy(pages.Data[offset-pages.Offset:], data)
+	if offset != pages.Offset+pages.Size {
+		// when this happens, debug shows the data overlapping with existing data is empty
+		// the data is not just append
+		if offset == pages.Offset && int(pages.Size) < len(data) {
+			// glog.V(2).Infof("pages[%d,%d) pages.Data len=%v, data len=%d, pages.Size=%d", pages.Offset, pages.Offset+pages.Size, len(pages.Data), len(data), pages.Size)
+			copy(pages.Data[pages.Size:], data[pages.Size:])
+		} else {
+			if pages.Size != 0 {
+				glog.V(1).Infof("%s/%s add page: pages [%d, %d) write [%d, %d)", pages.f.dir.Path, pages.f.Name, pages.Offset, pages.Offset+pages.Size, offset, offset+int64(len(data)))
+			}
+			return pages.flushAndSave(ctx, offset, data)
+		}
+	} else {
+		copy(pages.Data[offset-pages.Offset:], data)
+	}
+
 	pages.Size = max(pages.Size, offset+int64(len(data))-pages.Offset)
 
 	return
 }
 
+func (pages *ContinuousDirtyPages) flushAndSave(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
+
+	var chunk *filer_pb.FileChunk
+
+	// flush existing
+	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
+		if chunk != nil {
+			glog.V(4).Infof("%s/%s flush existing [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
+			chunks = append(chunks, chunk)
+		}
+	} else {
+		glog.V(0).Infof("%s/%s failed to flush1 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
+		return
+	}
+	pages.Size = 0
+	pages.Offset = 0
+
+	// flush the new page
+	if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil {
+		if chunk != nil {
+			glog.V(4).Infof("%s/%s flush big request [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
+			chunks = append(chunks, chunk)
+		}
+	} else {
+		glog.V(0).Infof("%s/%s failed to flush2 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
+		return
+	}
+
+	return
+}
+
 func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *filer_pb.FileChunk, err error) {
 
+	pages.lock.Lock()
+	defer pages.lock.Unlock()
+
 	if pages.Size == 0 {
 		return nil, nil
 	}
 
 	if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil {
 		pages.Size = 0
+		pages.Offset = 0
 		if chunk != nil {
 			glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size))
 		}
@@ -104,15 +143,16 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f
 }
 
 func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Context) (*filer_pb.FileChunk, error) {
-	return pages.saveToStorage(ctx, pages.Data[:pages.Size], pages.Offset)
-}
-
-func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {
 
 	if pages.Size == 0 {
 		return nil, nil
 	}
 
+	return pages.saveToStorage(ctx, pages.Data[:pages.Size], pages.Offset)
+}
+
+func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {
+
 	var fileId, host string
 
 	if err := pages.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
@@ -139,7 +179,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 	}
 
 	fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
-	bufReader := bytes.NewReader(pages.Data[:pages.Size])
+	bufReader := bytes.NewReader(buf)
 	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, "")
 	if err != nil {
 		glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
@@ -155,6 +195,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 		Offset: offset,
 		Size:   uint64(len(buf)),
 		Mtime:  time.Now().UnixNano(),
+		ETag:   uploadResult.ETag,
 	}, nil
 
 }

weed/filesys/file.go

@@ -12,18 +12,19 @@ import (
 	"time"
 )
 
+const blockSize = 512
+
 var _ = fs.Node(&File{})
 var _ = fs.NodeOpener(&File{})
 var _ = fs.NodeFsyncer(&File{})
 var _ = fs.NodeSetattrer(&File{})
 
 type File struct {
-	Chunks     []*filer_pb.FileChunk
-	Name       string
-	dir        *Dir
-	wfs        *WFS
-	attributes *filer_pb.FuseAttributes
-	isOpen     bool
+	Name   string
+	dir    *Dir
+	wfs    *WFS
+	entry  *filer_pb.Entry
+	isOpen bool
 }
 
 func (file *File) fullpath() string {
@@ -32,46 +33,17 @@ func (file *File) fullpath() string {
 
 func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {
 
-	if file.attributes == nil || !file.isOpen {
-		item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
-		if item != nil && !item.Expired() {
-			entry := item.Value().(*filer_pb.Entry)
-			file.Chunks = entry.Chunks
-			file.attributes = entry.Attributes
-			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
-		} else {
-			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-				request := &filer_pb.GetEntryAttributesRequest{
-					Name:      file.Name,
-					ParentDir: file.dir.Path,
-				}
-
-				resp, err := client.GetEntryAttributes(ctx, request)
-				if err != nil {
-					glog.V(0).Infof("file attr read file %v: %v", request, err)
-					return err
-				}
-
-				file.attributes = resp.Attributes
-				file.Chunks = resp.Chunks
-
-				glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.attributes, filer2.TotalSize(file.Chunks))
-
-				return nil
-			})
-
-			if err != nil {
-				return err
-			}
-		}
+	if err := file.maybeLoadAttributes(ctx); err != nil {
+		return err
 	}
 
-	attr.Mode = os.FileMode(file.attributes.FileMode)
-	attr.Size = filer2.TotalSize(file.Chunks)
-	attr.Mtime = time.Unix(file.attributes.Mtime, 0)
-	attr.Gid = file.attributes.Gid
-	attr.Uid = file.attributes.Uid
+	attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
+	attr.Size = filer2.TotalSize(file.entry.Chunks)
+	attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0)
+	attr.Gid = file.entry.Attributes.Gid
+	attr.Uid = file.entry.Attributes.Uid
+	attr.Blocks = attr.Size/blockSize + 1
+	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)
 
 	return nil
 
@@ -95,41 +67,45 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op
 
 func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
 
-	glog.V(3).Infof("%v file setattr %+v, fh=%d", file.fullpath(), req, req.Handle)
+	if err := file.maybeLoadAttributes(ctx); err != nil {
+		return err
+	}
+
+	if file.isOpen {
+		return nil
+	}
+
+	glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
 	if req.Valid.Size() {
 
 		glog.V(3).Infof("%v file setattr set size=%v", file.fullpath(), req.Size)
 		if req.Size == 0 {
 			// fmt.Printf("truncate %v \n", fullPath)
-			file.Chunks = nil
+			file.entry.Chunks = nil
 		}
-		file.attributes.FileSize = req.Size
+		file.entry.Attributes.FileSize = req.Size
 	}
 	if req.Valid.Mode() {
-		file.attributes.FileMode = uint32(req.Mode)
+		file.entry.Attributes.FileMode = uint32(req.Mode)
 	}
 
 	if req.Valid.Uid() {
-		file.attributes.Uid = req.Uid
+		file.entry.Attributes.Uid = req.Uid
 	}
 
 	if req.Valid.Gid() {
-		file.attributes.Gid = req.Gid
+		file.entry.Attributes.Gid = req.Gid
 	}
 
 	if req.Valid.Mtime() {
-		file.attributes.Mtime = req.Mtime.Unix()
+		file.entry.Attributes.Mtime = req.Mtime.Unix()
 	}
 
 	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
 			Directory: file.dir.Path,
-			Entry: &filer_pb.Entry{
-				Name:       file.Name,
-				Attributes: file.attributes,
-				Chunks:     file.Chunks,
-			},
+			Entry:     file.entry,
 		}
 
 		glog.V(1).Infof("set attr file entry: %v", request)
@@ -151,3 +127,41 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
 
 	return nil
 }
+
+func (file *File) maybeLoadAttributes(ctx context.Context) error {
+	if file.entry == nil || !file.isOpen {
+		item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
+		if item != nil && !item.Expired() {
+			entry := item.Value().(*filer_pb.Entry)
+			file.entry = entry
+			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
+		} else {
+			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+				request := &filer_pb.LookupDirectoryEntryRequest{
+					Name:      file.Name,
+					Directory: file.dir.Path,
+				}
+
+				resp, err := client.LookupDirectoryEntry(ctx, request)
+				if err != nil {
+					glog.V(0).Infof("file attr read file %v: %v", request, err)
+					return err
+				}
+
+				file.entry = resp.Entry
+
+				glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))
+
+				file.wfs.listDirectoryEntriesCache.Set(file.fullpath(), file.entry, file.wfs.option.EntryCacheTtl)
+
+				return nil
+			})
+
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

weed/filesys/filehandle.go

@@ -12,15 +12,15 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 )
 
 type FileHandle struct {
 	// cache file has been written to
 	dirtyPages    *ContinuousDirtyPages
-	dirtyMetadata bool
 	contentType   string
-
-	handle uint64
+	dirtyMetadata bool
+	handle        uint64
 
 	f         *File
 	RequestId fuse.RequestID // unique ID for request
@@ -38,6 +38,16 @@ func newFileHandle(file *File, uid, gid uint32) *FileHandle {
 	}
 }
 
+func (fh *FileHandle) InitializeToFile(file *File, uid, gid uint32) *FileHandle {
+	newHandle := &FileHandle{
+		f:          file,
+		dirtyPages: fh.dirtyPages.InitializeToFile(file),
+		Uid:        uid,
+		Gid:        gid,
+	}
+	return newHandle
+}
+
 var _ = fs.Handle(&FileHandle{})
 
 // var _ = fs.HandleReadAller(&FileHandle{})
@@ -51,14 +61,14 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 	glog.V(4).Infof("%s read fh %d: [%d,%d)", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(req.Size))
 
 	// this value should come from the filer instead of the old f
-	if len(fh.f.Chunks) == 0 {
-		glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name)
-		return fmt.Errorf("empty file %v/%v", fh.f.dir.Path, fh.f.Name)
+	if len(fh.f.entry.Chunks) == 0 {
+		glog.V(1).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name)
+		return nil
 	}
 
 	buff := make([]byte, req.Size)
 
-	chunkViews := filer2.ViewFromChunks(fh.f.Chunks, req.Offset, req.Size)
+	chunkViews := filer2.ViewFromChunks(fh.f.entry.Chunks, req.Offset, req.Size)
 
 	var vids []string
 	for _, chunkView := range chunkViews {
@@ -108,7 +118,8 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 				fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
 				chunkView.Offset,
 				int(chunkView.Size),
-				buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)])
+				buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)],
+				!chunkView.IsFullChunk)
 
 			if err != nil {
 
@@ -140,6 +151,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 
 	chunks, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data)
 	if err != nil {
+		glog.Errorf("%+v/%v write fh %d: [%d,%d): %v", fh.f.dir.Path, fh.f.Name, fh.handle, req.Offset, req.Offset+int64(len(req.Data)), err)
 		return fmt.Errorf("write %s/%s at [%d,%d): %v", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data)), err)
 	}
 
@@ -151,7 +163,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 	}
 
 	for _, chunk := range chunks {
-		fh.f.Chunks = append(fh.f.Chunks, chunk)
+		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
 		glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
 		fh.dirtyMetadata = true
 	}
@@ -163,7 +175,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 
 	glog.V(4).Infof("%v release fh %d", fh.f.fullpath(), fh.handle)
 
-	fh.f.wfs.ReleaseHandle(fuse.HandleID(fh.handle))
+	fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
 
 	fh.f.isOpen = false
 
@@ -179,54 +191,43 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 
 	chunk, err := fh.dirtyPages.FlushToStorage(ctx)
 	if err != nil {
-		glog.V(0).Infof("flush %s/%s to %s [%d,%d): %v", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
-		return fmt.Errorf("flush %s/%s to %s [%d,%d): %v", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size), err)
+		glog.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
+		return fmt.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
 	}
 	if chunk != nil {
-		fh.f.Chunks = append(fh.f.Chunks, chunk)
-		fh.dirtyMetadata = true
+		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
 	}
 
 	if !fh.dirtyMetadata {
 		return nil
 	}
 
-	if len(fh.f.Chunks) == 0 {
-		glog.V(2).Infof("fh %s/%s flush skipping empty: %v", fh.f.dir.Path, fh.f.Name, req)
-		return nil
-	}
+	return fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-	err = fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
-
-		if fh.f.attributes != nil {
-			fh.f.attributes.Mime = fh.contentType
+		if fh.f.entry.Attributes != nil {
+			fh.f.entry.Attributes.Mime = fh.contentType
+			fh.f.entry.Attributes.Uid = req.Uid
+			fh.f.entry.Attributes.Gid = req.Gid
+			fh.f.entry.Attributes.Mtime = time.Now().Unix()
+			fh.f.entry.Attributes.Crtime = time.Now().Unix()
+			fh.f.entry.Attributes.FileMode = uint32(0770)
 		}
 
-		request := &filer_pb.UpdateEntryRequest{
+		request := &filer_pb.CreateEntryRequest{
 			Directory: fh.f.dir.Path,
-			Entry: &filer_pb.Entry{
-				Name:       fh.f.Name,
-				Attributes: fh.f.attributes,
-				Chunks:     fh.f.Chunks,
-			},
+			Entry:     fh.f.entry,
 		}
 
-		glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.Chunks))
-		for i, chunk := range fh.f.Chunks {
+		glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
+		for i, chunk := range fh.f.entry.Chunks {
 			glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
 		}
-		if _, err := client.UpdateEntry(ctx, request); err != nil {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
 
 		return nil
 	})
-
-	if err == nil {
-		fh.dirtyMetadata = false
-	}
-
-	return err
 }
 
 func volumeId(fileId string) string {

weed/filesys/wfs.go

@@ -1,14 +1,19 @@
 package filesys
 
 import (
+	"context"
+	"fmt"
+	"math"
+	"sync"
+	"time"
+
 	"bazil.org/fuse"
 	"bazil.org/fuse/fs"
-	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/karlseguin/ccache"
-	"sync"
+	"google.golang.org/grpc"
 )
 
 type Option struct {
@@ -20,8 +25,12 @@ type Option struct {
 	ChunkSizeLimit     int64
 	DataCenter         string
 	DirListingLimit    int
+	EntryCacheTtl      time.Duration
 }
 
+var _ = fs.FS(&WFS{})
+var _ = fs.FSStatfser(&WFS{})
+
 type WFS struct {
 	option                    *Option
 	listDirectoryEntriesCache *ccache.Cache
@@ -30,12 +39,18 @@ type WFS struct {
 	handles           []*FileHandle
 	pathToHandleIndex map[string]int
 	pathToHandleLock  sync.Mutex
+
+	stats statsCache
+}
+type statsCache struct {
+	filer_pb.StatisticsResponse
+	lastChecked int64 // unix time in seconds
 }
 
 func NewSeaweedFileSystem(option *Option) *WFS {
 	return &WFS{
-		option: option,
-		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)),
+		option:                    option,
+		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(int64(option.DirListingLimit) + 200).ItemsToPrune(100)),
 		pathToHandleIndex:         make(map[string]int),
 	}
 }
@@ -46,15 +61,11 @@ func (wfs *WFS) Root() (fs.Node, error) {
 
 func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(wfs.option.FilerGrpcAddress)
-	if err != nil {
-		return fmt.Errorf("fail to dial %s: %v", wfs.option.FilerGrpcAddress, err)
-	}
-	defer grpcConnection.Close()
+	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, wfs.option.FilerGrpcAddress)
 
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {
@@ -69,15 +80,14 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
 		return wfs.handles[index]
 	}
 
-	fileHandle = newFileHandle(file, uid, gid)
-
 	if found && wfs.handles[index] != nil {
 		glog.V(4).Infoln(fullpath, "reuse previous fileHandle id", index)
-		wfs.handles[index] = fileHandle
+		wfs.handles[index].InitializeToFile(file, uid, gid)
 		fileHandle.handle = uint64(index)
 		return
 	}
 
+	fileHandle = newFileHandle(file, uid, gid)
 	for i, h := range wfs.handles {
 		if h == nil {
 			wfs.handles[i] = fileHandle
@@ -96,14 +106,77 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
 	return
 }
 
-func (wfs *WFS) ReleaseHandle(handleId fuse.HandleID) {
+func (wfs *WFS) ReleaseHandle(fullpath string, handleId fuse.HandleID) {
 	wfs.pathToHandleLock.Lock()
 	defer wfs.pathToHandleLock.Unlock()
 
-	glog.V(4).Infoln("releasing handle id", handleId, "current handles lengh", len(wfs.handles))
+	glog.V(4).Infof("%s releasing handle id %d current handles length %d", fullpath, handleId, len(wfs.handles))
+	delete(wfs.pathToHandleIndex, fullpath)
 	if int(handleId) < len(wfs.handles) {
 		wfs.handles[int(handleId)] = nil
 	}
 
 	return
 }
+
+// Statfs is called to obtain file system metadata. Implements fuse.FSStatfser
+func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.StatfsResponse) error {
+
+	glog.V(4).Infof("reading fs stats: %+v", req)
+
+	if wfs.stats.lastChecked < time.Now().Unix()-20 {
+
+		err := wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+			request := &filer_pb.StatisticsRequest{
+				Collection:  wfs.option.Collection,
+				Replication: wfs.option.Replication,
+				Ttl:         fmt.Sprintf("%ds", wfs.option.TtlSec),
+			}
+
+			glog.V(4).Infof("reading filer stats: %+v", request)
+			resp, err := client.Statistics(ctx, request)
+			if err != nil {
+				glog.V(0).Infof("reading filer stats %v: %v", request, err)
+				return err
+			}
+			glog.V(4).Infof("read filer stats: %+v", resp)
+
+			wfs.stats.TotalSize = resp.TotalSize
+			wfs.stats.UsedSize = resp.UsedSize
+			wfs.stats.FileCount = resp.FileCount
+			wfs.stats.lastChecked = time.Now().Unix()
+
+			return nil
+		})
+		if err != nil {
+			glog.V(0).Infof("filer Statistics: %v", err)
+			return err
+		}
+	}
+
+	totalDiskSize := wfs.stats.TotalSize
+	usedDiskSize := wfs.stats.UsedSize
+	actualFileCount := wfs.stats.FileCount
+
+	// Compute the total number of available blocks
+	resp.Blocks = totalDiskSize / blockSize
+
+	// Compute the number of used blocks
+	numblocks := uint64(usedDiskSize / blockSize)
+
+	// Report the number of free and available blocks for the block size
+	resp.Bfree = resp.Blocks - numblocks
+	resp.Bavail = resp.Blocks - numblocks
+	resp.Bsize = uint32(blockSize)
+
+	// Report the total number of possible files in the file system (and those free)
+	resp.Files = math.MaxInt64
+	resp.Ffree = math.MaxInt64 - actualFileCount
+
+	// Report the maximum length of a name and the minimum fragment size
+	resp.Namelen = 1024
+	resp.Frsize = uint32(blockSize)
+
+	return nil
+}

weed/images/favicon.go

@@ -1,236 +0,0 @@
-// Code generated by go-bindata.
-// sources:
-// favicon/favicon.ico
-
-package images
-
-import (
-	"bytes"
-	"compress/gzip"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-)
-
-func bindataRead(data []byte, name string) ([]byte, error) {
-	gz, err := gzip.NewReader(bytes.NewBuffer(data))
-	if err != nil {
-		return nil, fmt.Errorf("Read %q: %v", name, err)
-	}
-
-	var buf bytes.Buffer
-	_, err = io.Copy(&buf, gz)
-	clErr := gz.Close()
-
-	if err != nil {
-		return nil, fmt.Errorf("Read %q: %v", name, err)
-	}
-	if clErr != nil {
-		return nil, err
-	}
-
-	return buf.Bytes(), nil
-}
-
-type asset struct {
-	bytes []byte
-	info  os.FileInfo
-}
-
-type bindataFileInfo struct {
-	name    string
-	size    int64
-	mode    os.FileMode
-	modTime time.Time
-}
-
-func (fi bindataFileInfo) Name() string {
-	return fi.name
-}
-func (fi bindataFileInfo) Size() int64 {
-	return fi.size
-}
-func (fi bindataFileInfo) Mode() os.FileMode {
-	return fi.mode
-}
-func (fi bindataFileInfo) ModTime() time.Time {
-	return fi.modTime
-}
-func (fi bindataFileInfo) IsDir() bool {
-	return false
-}
-func (fi bindataFileInfo) Sys() interface{} {
-	return nil
-}
-
-var _favicon = []byte("\x1f\x8b\x08\x00\x00\x09\x6e\x88\x00\xff\xec\x94\x5f\x48\x53\x51\x1c\xc7\xbf\x77\xce\xeb\xf1\xff\x9d\x53\x37\xe7\xbf\x4d\xcd\x2c\x08\xa2\x87\xea\xd1\x82\x9e\xa2\x88\x5e\x7a\x8a\x82\x1e\xf2\x21\x7a\x28\x42\xc9\x7a\xb0\x08\x22\x4d\x08\xfa\x43\x25\x61\x05\x4b\xac\xb7\x7a\x92\x8a\x82\xa2\x34\x59\x5c\x4d\x5b\x39\x2d\xfb\xb3\x9c\xe0\x52\xb7\x5a\x6e\x73\xeb\xdc\xbb\xed\xb6\x5d\xc7\x89\x7a\xa8\xa0\xfb\x81\x7b\xef\x39\xbf\xef\x3d\xf7\xfe\xce\x79\xf8\x00\x1c\x74\x10\x04\xe9\x49\xd0\x94\x09\xd4\x03\xb0\x5a\x63\xf3\x1e\x02\x74\xd2\x5a\x03\xad\x09\x52\x1d\xb1\xba\x0c\x87\x1f\x70\xb1\xab\xff\xd0\x2a\x0c\x9d\xde\x0e\xf1\xe4\x66\xbc\xba\xb2\x0f\xe1\xc0\x3c\x5e\x77\xef\x87\xe7\xe9\x4d\xcc\x3a\x1f\x21\x1a\x8d\x22\x1a\x89\xc0\x7d\xbf\x0b\x03\xcd\xab\x31\xde\x73\x18\x6f\x6e\xb5\x61\xea\xf1\x0d\x84\xbf\xf9\x31\x76\xfd\x20\xbe\x4e\xb9\x20\xb6\x6f\xc5\xb3\xd6\xb5\x78\x7e\x62\x13\x7d\xae\x83\xa3\xad\x11\x73\xae\x7e\xbc\x38\xb3\x03\x23\x67\x77\x62\x61\xd6\x03\x67\x57\x13\x1c\xc7\x37\x62\xa2\xf7\x28\x22\xe1\x10\x7c\x93\x22\x42\x7e\x2f\x82\x73\xd3\x18\x68\x59\x03\x97\xbd\x05\x43\x1d\xdb\xa0\xa1\xa1\xf1\xaf\x51\x5b\x97\x3c\xab\x29\x20\xe6\xa4\x69\x71\x35\x21\x56\x1b\x25\x3e\xaf\xac\x22\x31\x12\x2f\x94\x59\x48\x79\x05\x45\x7e\xb9\x84\xde\x4a\x4d\xca\xfa\x0c\x43\x11\xbd\xe7\x1a\xa5\x71\x3e\xa5\x80\x14\x0a\x89\x2c\xfe\x19\x92\x17\x9f\xf3\x94\x2c\x92\x9d\x93\x9b\xf4\xf3\x0c\x7d\x66\x4a\xa7\x9c\xee\xb7\x37\xf9\xcb\x48\x9e\xb5\xd2\xab\x11\x49\x9e\xd5\x27\x37\x03\xc5\xb3\x8e\x63\x1b\xe0\x79\xd2\x8b\x4f\x0f\xaf\xca\x2e\x95\x1c\x2b\xf9\x75\x7e\x7c\x50\x76\xe0\xe8\xf9\x5d\xb2\x2f\xbd\xc3\x77\x11\x59\x0c\x61\xf2\x76\x3b\x66\xc4\x3e\x0c\x1e\x59\x8f\xc0\xf4\x84\xec\xd6\x0f\x7d\xe7\xe0\x7f\x3f\x22\x7b\x33\xf4\xe5\xb3\xec\x54\xc9\x9d\x8b\xc1\x00\x75\x73\x2b\x86\xa9\xcb\xdf\xdd\xe9\x94\x1d\xed\xb2\x37\x53\x4f\xdb\xe1\x7b\x2b\xe2\xe3\xbd\x4b\x78\x79\x71\x0f\x82\xbe\x19\x38\x2f\xef\xa5\xdf\xee\xc0\xe8\x85\xdd\x58\xf0\xba\xe1\x7e\xd0\x8d\xb1\x6b\x07\x64\x9f\x8b\xa7\xb6\xfc\xb9\xc3\xd3\xd0\xd0\xf8\x7b\xac\xb4\x31\xe3\x86\x15\x15\x8c\x54\xbf\x9c\xe7\x19\x39\x97\xc5\xb3\xf2\x65\x44\x32\x79\xbd\x31\x81\x2a\xae\xcb\xe3\x53\x49\x8d\x6b\x79\x35\xaa\xf5\xb6\x1a\x76\x0e\x43\xb5\x54\xb5\xe6\x24\x58\xd2\xa0\xa9\x8a\xd9\x3f\xa5\xd2\x9c\x2e\x2f\xb7\x28\x43\x8b\x90\x26\xcf\x34\x97\x29\xe3\x42\x4b\x9a\x9c\xe7\x75\x26\xc6\x5f\x69\xce\x1b\x0d\x4b\xca\xca\x41\x16\x4b\xfb\x2a\x29\x55\xe7\xea\xf3\x2a\xfa\x49\x2e\x18\x98\x79\x41\xa1\x6a\xbd\x72\x90\xb9\x34\xcd\xcb\x67\xf6\x4f\xb2\x99\xfb\xcb\xe2\x18\x31\x74\x19\x7a\x56\x0c\xe6\xe2\xff\x97\xef\x01\x00\x00\xff\xff\x3e\xd4\x17\xe7\x36\x0e\x00\x00")
-
-func faviconBytes() ([]byte, error) {
-	return bindataRead(
-		_favicon,
-		"favicon/favicon.ico",
-	)
-}
-
-func favicon() (*asset, error) {
-	bytes, err := faviconBytes()
-	if err != nil {
-		return nil, err
-	}
-
-	info := bindataFileInfo{name: "favicon/favicon.ico", size: 3638, mode: os.FileMode(420), modTime: time.Unix(1460621502, 0)}
-	a := &asset{bytes: bytes, info: info}
-	return a, nil
-}
-
-// Asset loads and returns the asset for the given name.
-// It returns an error if the asset could not be found or
-// could not be loaded.
-func Asset(name string) ([]byte, error) {
-	cannonicalName := strings.Replace(name, "\\", "/", -1)
-	if f, ok := _bindata[cannonicalName]; ok {
-		a, err := f()
-		if err != nil {
-			return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
-		}
-		return a.bytes, nil
-	}
-	return nil, fmt.Errorf("Asset %s not found", name)
-}
-
-// MustAsset is like Asset but panics when Asset would return an error.
-// It simplifies safe initialization of global variables.
-func MustAsset(name string) []byte {
-	a, err := Asset(name)
-	if err != nil {
-		panic("asset: Asset(" + name + "): " + err.Error())
-	}
-
-	return a
-}
-
-// AssetInfo loads and returns the asset info for the given name.
-// It returns an error if the asset could not be found or
-// could not be loaded.
-func AssetInfo(name string) (os.FileInfo, error) {
-	cannonicalName := strings.Replace(name, "\\", "/", -1)
-	if f, ok := _bindata[cannonicalName]; ok {
-		a, err := f()
-		if err != nil {
-			return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
-		}
-		return a.info, nil
-	}
-	return nil, fmt.Errorf("AssetInfo %s not found", name)
-}
-
-// AssetNames returns the names of the assets.
-func AssetNames() []string {
-	names := make([]string, 0, len(_bindata))
-	for name := range _bindata {
-		names = append(names, name)
-	}
-	return names
-}
-
-// _bindata is a table, holding each asset generator, mapped to its name.
-var _bindata = map[string]func() (*asset, error){
-	"favicon/favicon.ico": favicon,
-}
-
-// AssetDir returns the file names below a certain
-// directory embedded in the file by go-bindata.
-// For example if you run go-bindata on data/... and data contains the
-// following hierarchy:
-//     data/
-//       foo.txt
-//       img/
-//         a.png
-//         b.png
-// then AssetDir("data") would return []string{"foo.txt", "img"}
-// AssetDir("data/img") would return []string{"a.png", "b.png"}
-// AssetDir("foo.txt") and AssetDir("notexist") would return an error
-// AssetDir("") will return []string{"data"}.
-func AssetDir(name string) ([]string, error) {
-	node := _bintree
-	if len(name) != 0 {
-		cannonicalName := strings.Replace(name, "\\", "/", -1)
-		pathList := strings.Split(cannonicalName, "/")
-		for _, p := range pathList {
-			node = node.Children[p]
-			if node == nil {
-				return nil, fmt.Errorf("Asset %s not found", name)
-			}
-		}
-	}
-	if node.Func != nil {
-		return nil, fmt.Errorf("Asset %s not found", name)
-	}
-	rv := make([]string, 0, len(node.Children))
-	for childName := range node.Children {
-		rv = append(rv, childName)
-	}
-	return rv, nil
-}
-
-type bintree struct {
-	Func     func() (*asset, error)
-	Children map[string]*bintree
-}
-
-var _bintree = &bintree{nil, map[string]*bintree{
-	"favicon": &bintree{nil, map[string]*bintree{
-		"favicon.ico": &bintree{favicon, map[string]*bintree{}},
-	}},
-}}
-
-// RestoreAsset restores an asset under the given directory
-func RestoreAsset(dir, name string) error {
-	data, err := Asset(name)
-	if err != nil {
-		return err
-	}
-	info, err := AssetInfo(name)
-	if err != nil {
-		return err
-	}
-	err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
-	if err != nil {
-		return err
-	}
-	err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
-	if err != nil {
-		return err
-	}
-	err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-// RestoreAssets restores an asset under the given directory recursively
-func RestoreAssets(dir, name string) error {
-	children, err := AssetDir(name)
-	// File
-	if err != nil {
-		return RestoreAsset(dir, name)
-	}
-	// Dir
-	for _, child := range children {
-		err = RestoreAssets(dir, filepath.Join(name, child))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func _filePath(dir, name string) string {
-	cannonicalName := strings.Replace(name, "\\", "/", -1)
-	return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
-}

weed/msgqueue/configuration.go

@@ -1,33 +0,0 @@
-package msgqueue
-
-import (
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/spf13/viper"
-)
-
-var (
-	MessageQueues []MessageQueue
-
-	Queue MessageQueue
-)
-
-func LoadConfiguration(config *viper.Viper) {
-
-	if config == nil {
-		return
-	}
-
-	for _, store := range MessageQueues {
-		if config.GetBool(store.GetName() + ".enabled") {
-			viperSub := config.Sub(store.GetName())
-			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize store for %s: %+v",
-					store.GetName(), err)
-			}
-			Queue = store
-			glog.V(0).Infof("Configure message queue for %s", store.GetName())
-			return
-		}
-	}
-
-}

weed/msgqueue/message_queue.go

@@ -1,14 +0,0 @@
-package msgqueue
-
-import (
-	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/golang/protobuf/proto"
-)
-
-type MessageQueue interface {
-	// GetName gets the name to locate the configuration in message_queue.toml file
-	GetName() string
-	// Initialize initializes the file store
-	Initialize(configuration util.Configuration) error
-	SendMessage(key string, message proto.Message) error
-}

weed/notification/aws_sqs/aws_sqs_pub.go

@@ -0,0 +1,92 @@
+package aws_sqs
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/notification"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+)
+
+func init() {
+	notification.MessageQueues = append(notification.MessageQueues, &AwsSqsPub{})
+}
+
+type AwsSqsPub struct {
+	svc      *sqs.SQS
+	queueUrl string
+}
+
+func (k *AwsSqsPub) GetName() string {
+	return "aws_sqs"
+}
+
+func (k *AwsSqsPub) Initialize(configuration util.Configuration) (err error) {
+	glog.V(0).Infof("filer.notification.aws_sqs.region: %v", configuration.GetString("region"))
+	glog.V(0).Infof("filer.notification.aws_sqs.sqs_queue_name: %v", configuration.GetString("sqs_queue_name"))
+	return k.initialize(
+		configuration.GetString("aws_access_key_id"),
+		configuration.GetString("aws_secret_access_key"),
+		configuration.GetString("region"),
+		configuration.GetString("sqs_queue_name"),
+	)
+}
+
+func (k *AwsSqsPub) initialize(awsAccessKeyId, aswSecretAccessKey, region, queueName string) (err error) {
+
+	config := &aws.Config{
+		Region: aws.String(region),
+	}
+	if awsAccessKeyId != "" && aswSecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, aswSecretAccessKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return fmt.Errorf("create aws session: %v", err)
+	}
+	k.svc = sqs.New(sess)
+
+	result, err := k.svc.GetQueueUrl(&sqs.GetQueueUrlInput{
+		QueueName: aws.String(queueName),
+	})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sqs.ErrCodeQueueDoesNotExist {
+			return fmt.Errorf("unable to find queue %s", queueName)
+		}
+		return fmt.Errorf("get queue %s url: %v", queueName, err)
+	}
+
+	k.queueUrl = *result.QueueUrl
+
+	return nil
+}
+
+func (k *AwsSqsPub) SendMessage(key string, message proto.Message) (err error) {
+
+	text := proto.MarshalTextString(message)
+
+	_, err = k.svc.SendMessage(&sqs.SendMessageInput{
+		DelaySeconds: aws.Int64(10),
+		MessageAttributes: map[string]*sqs.MessageAttributeValue{
+			"key": &sqs.MessageAttributeValue{
+				DataType:    aws.String("String"),
+				StringValue: aws.String(key),
+			},
+		},
+		MessageBody: aws.String(text),
+		QueueUrl:    &k.queueUrl,
+	})
+
+	if err != nil {
+		return fmt.Errorf("send message to sqs %s: %v", k.queueUrl, err)
+	}
+
+	return nil
+}

weed/notification/configuration.go

@@ -0,0 +1,58 @@
+package notification
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"github.com/spf13/viper"
+)
+
+type MessageQueue interface {
+	// GetName gets the name to locate the configuration in filer.toml file
+	GetName() string
+	// Initialize initializes the file store
+	Initialize(configuration util.Configuration) error
+	SendMessage(key string, message proto.Message) error
+}
+
+var (
+	MessageQueues []MessageQueue
+
+	Queue MessageQueue
+)
+
+func LoadConfiguration(config *viper.Viper) {
+
+	if config == nil {
+		return
+	}
+
+	validateOneEnabledQueue(config)
+
+	for _, queue := range MessageQueues {
+		if config.GetBool(queue.GetName() + ".enabled") {
+			viperSub := config.Sub(queue.GetName())
+			if err := queue.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize notification for %s: %+v",
+					queue.GetName(), err)
+			}
+			Queue = queue
+			glog.V(0).Infof("Configure notification message queue for %s", queue.GetName())
+			return
+		}
+	}
+
+}
+
+func validateOneEnabledQueue(config *viper.Viper) {
+	enabledQueue := ""
+	for _, queue := range MessageQueues {
+		if config.GetBool(queue.GetName() + ".enabled") {
+			if enabledQueue == "" {
+				enabledQueue = queue.GetName()
+			} else {
+				glog.Fatalf("Notification message queue is enabled for both %s and %s", enabledQueue, queue.GetName())
+			}
+		}
+	}
+}

weed/notification/google_pub_sub/google_pub_sub.go

@@ -0,0 +1,89 @@
+package google_pub_sub
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"cloud.google.com/go/pubsub"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/notification"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/golang/protobuf/proto"
+	"google.golang.org/api/option"
+)
+
+func init() {
+	notification.MessageQueues = append(notification.MessageQueues, &GooglePubSub{})
+}
+
+type GooglePubSub struct {
+	topic *pubsub.Topic
+}
+
+func (k *GooglePubSub) GetName() string {
+	return "google_pub_sub"
+}
+
+func (k *GooglePubSub) Initialize(configuration util.Configuration) (err error) {
+	glog.V(0).Infof("notification.google_pub_sub.project_id: %v", configuration.GetString("project_id"))
+	glog.V(0).Infof("notification.google_pub_sub.topic: %v", configuration.GetString("topic"))
+	return k.initialize(
+		configuration.GetString("google_application_credentials"),
+		configuration.GetString("project_id"),
+		configuration.GetString("topic"),
+	)
+}
+
+func (k *GooglePubSub) initialize(google_application_credentials, projectId, topicName string) (err error) {
+
+	ctx := context.Background()
+	// Creates a client.
+	if google_application_credentials == "" {
+		var found bool
+		google_application_credentials, found = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS")
+		if !found {
+			glog.Fatalf("need to specific GOOGLE_APPLICATION_CREDENTIALS env variable or google_application_credentials in filer.toml")
+		}
+	}
+
+	client, err := pubsub.NewClient(ctx, projectId, option.WithCredentialsFile(google_application_credentials))
+	if err != nil {
+		glog.Fatalf("Failed to create client: %v", err)
+	}
+
+	k.topic = client.Topic(topicName)
+	if exists, err := k.topic.Exists(ctx); err == nil {
+		if !exists {
+			k.topic, err = client.CreateTopic(ctx, topicName)
+			if err != nil {
+				glog.Fatalf("Failed to create topic %s: %v", topicName, err)
+			}
+		}
+	} else {
+		glog.Fatalf("Failed to check topic %s: %v", topicName, err)
+	}
+
+	return nil
+}
+
+func (k *GooglePubSub) SendMessage(key string, message proto.Message) (err error) {
+
+	bytes, err := proto.Marshal(message)
+	if err != nil {
+		return
+	}
+
+	ctx := context.Background()
+	result := k.topic.Publish(ctx, &pubsub.Message{
+		Data:       bytes,
+		Attributes: map[string]string{"key": key},
+	})
+
+	_, err = result.Get(ctx)
+	if err != nil {
+		return fmt.Errorf("send message to google pub sub %s: %v", k.topic.String(), err)
+	}
+
+	return nil
+}

weed/notification/kafka/kafka_queue.go

@@ -3,13 +3,13 @@ package kafka
 import (
 	"github.com/Shopify/sarama"
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/msgqueue"
+	"github.com/chrislusf/seaweedfs/weed/notification"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/golang/protobuf/proto"
 )
 
 func init() {
-	msgqueue.MessageQueues = append(msgqueue.MessageQueues, &KafkaQueue{})
+	notification.MessageQueues = append(notification.MessageQueues, &KafkaQueue{})
 }
 
 type KafkaQueue struct {
@@ -22,8 +22,8 @@ func (k *KafkaQueue) GetName() string {
 }
 
 func (k *KafkaQueue) Initialize(configuration util.Configuration) (err error) {
-	glog.V(0).Infof("filer.msgqueue.kafka.hosts: %v\n", configuration.GetStringSlice("hosts"))
-	glog.V(0).Infof("filer.msgqueue.kafka.topic: %v\n", configuration.GetString("topic"))
+	glog.V(0).Infof("filer.notification.kafka.hosts: %v\n", configuration.GetStringSlice("hosts"))
+	glog.V(0).Infof("filer.notification.kafka.topic: %v\n", configuration.GetString("topic"))
 	return k.initialize(
 		configuration.GetStringSlice("hosts"),
 		configuration.GetString("topic"),
@@ -37,6 +37,9 @@ func (k *KafkaQueue) initialize(hosts []string, topic string) (err error) {
 	config.Producer.Return.Successes = true
 	config.Producer.Return.Errors = true
 	k.producer, err = sarama.NewAsyncProducer(hosts, config)
+	if err != nil {
+		return err
+	}
 	k.topic = topic
 	go k.handleSuccess()
 	go k.handleError()

weed/notification/log/log_queue.go

@@ -2,13 +2,13 @@ package kafka
 
 import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/msgqueue"
+	"github.com/chrislusf/seaweedfs/weed/notification"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/golang/protobuf/proto"
 )
 
 func init() {
-	msgqueue.MessageQueues = append(msgqueue.MessageQueues, &LogQueue{})
+	notification.MessageQueues = append(notification.MessageQueues, &LogQueue{})
 }
 
 type LogQueue struct {

weed/operation/assign_file_id.go

@@ -1,13 +1,10 @@
 package operation
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
-	"net/url"
-	"strconv"
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
 
 type VolumeAssignRequest struct {
@@ -29,52 +26,54 @@ type AssignResult struct {
 }
 
 func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
+
 	var requests []*VolumeAssignRequest
 	requests = append(requests, primaryRequest)
 	requests = append(requests, alternativeRequests...)
 
 	var lastError error
+	ret := &AssignResult{}
+
 	for i, request := range requests {
 		if request == nil {
 			continue
 		}
-		values := make(url.Values)
-		values.Add("count", strconv.FormatUint(request.Count, 10))
-		if request.Replication != "" {
-			values.Add("replication", request.Replication)
-		}
-		if request.Collection != "" {
-			values.Add("collection", request.Collection)
-		}
-		if request.Ttl != "" {
-			values.Add("ttl", request.Ttl)
-		}
-		if request.DataCenter != "" {
-			values.Add("dataCenter", request.DataCenter)
-		}
-		if request.Rack != "" {
-			values.Add("rack", request.Rack)
-		}
-		if request.DataNode != "" {
-			values.Add("dataNode", request.DataNode)
+
+		lastError = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+			req := &master_pb.AssignRequest{
+				Count:       primaryRequest.Count,
+				Replication: primaryRequest.Replication,
+				Collection:  primaryRequest.Collection,
+				Ttl:         primaryRequest.Ttl,
+				DataCenter:  primaryRequest.DataCenter,
+				Rack:        primaryRequest.Rack,
+				DataNode:    primaryRequest.DataNode,
+			}
+			resp, grpcErr := masterClient.Assign(context.Background(), req)
+			if grpcErr != nil {
+				return grpcErr
+			}
+
+			ret.Count = resp.Count
+			ret.Fid = resp.Fid
+			ret.Url = resp.Url
+			ret.PublicUrl = resp.PublicUrl
+			ret.Error = resp.Error
+
+			return nil
+
+		})
+
+		if lastError != nil {
+			continue
 		}
 
-		postUrl := fmt.Sprintf("http://%s/dir/assign", server)
-		jsonBlob, err := util.Post(postUrl, values)
-		glog.V(2).Infof("assign %d result from %s %+v : %s", i, postUrl, values, string(jsonBlob))
-		if err != nil {
-			return nil, err
-		}
-		var ret AssignResult
-		err = json.Unmarshal(jsonBlob, &ret)
-		if err != nil {
-			return nil, fmt.Errorf("/dir/assign result JSON unmarshal error:%v, json:%s", err, string(jsonBlob))
-		}
 		if ret.Count <= 0 {
 			lastError = fmt.Errorf("assign failure %d: %v", i+1, ret.Error)
 			continue
 		}
-		return &ret, nil
+
 	}
-	return nil, lastError
+
+	return ret, lastError
 }

weed/operation/chunked_file.go

@@ -70,16 +70,22 @@ func (cm *ChunkManifest) Marshal() ([]byte, error) {
 }
 
 func (cm *ChunkManifest) DeleteChunks(master string) error {
-	deleteError := 0
+	var fileIds []string
 	for _, ci := range cm.Chunks {
-		if e := DeleteFile(master, ci.Fid, ""); e != nil {
-			deleteError++
-			glog.V(0).Infof("Delete %s error: %v, master: %s", ci.Fid, e, master)
+		fileIds = append(fileIds, ci.Fid)
+	}
+	results, err := DeleteFiles(master, fileIds)
+	if err != nil {
+		glog.V(0).Infof("delete %+v: %v", fileIds, err)
+		return fmt.Errorf("chunk delete: %v", err)
+	}
+	for _, result := range results {
+		if result.Error != "" {
+			glog.V(0).Infof("delete file %+v: %v", result.FileId, result.Error)
+			return fmt.Errorf("chunk delete %v: %v", result.FileId, result.Error)
 		}
 	}
-	if deleteError > 0 {
-		return errors.New("Not all chunks deleted.")
-	}
+
 	return nil
 }
 

weed/operation/compress.go

@@ -8,21 +8,41 @@ import (
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"golang.org/x/tools/godoc/util"
 )
 
 /*
 * Default more not to gzip since gzip can be done on client side.
  */
-func IsGzippable(ext, mtype string) bool {
+func IsGzippable(ext, mtype string, data []byte) bool {
+
+	// text
 	if strings.HasPrefix(mtype, "text/") {
 		return true
 	}
+
+	// images
+	switch ext {
+	case ".svg", ".bmp":
+		return true
+	}
+	if strings.HasPrefix(mtype, "image/") {
+		return false
+	}
+
+	// by file name extention
 	switch ext {
 	case ".zip", ".rar", ".gz", ".bz2", ".xz":
 		return false
 	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
 		return true
+	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
+		return true
+	case ".png", ".jpg", ".jpeg", "":
+		return true
 	}
+
+	// by mime type
 	if strings.HasPrefix(mtype, "application/") {
 		if strings.HasSuffix(mtype, "xml") {
 			return true
@@ -31,7 +51,10 @@ func IsGzippable(ext, mtype string) bool {
 			return true
 		}
 	}
-	return false
+
+	isMostlyText := util.IsText(data)
+
+	return isMostlyText
 }
 
 func GzipData(input []byte) ([]byte, error) {

weed/operation/delete_content.go

@@ -1,17 +1,14 @@
 package operation
 
 import (
-	"encoding/json"
+	"context"
 	"errors"
 	"fmt"
-	"net/url"
+	"net/http"
 	"strings"
 	"sync"
 
-	"net/http"
-
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 )
 
 type DeleteResult struct {
@@ -21,26 +18,6 @@ type DeleteResult struct {
 	Error  string `json:"error,omitempty"`
 }
 
-func DeleteFromVolumeServer(fileUrlOnVolume string, jwt security.EncodedJwt) error {
-	err := util.Delete(fileUrlOnVolume, jwt)
-	if err != nil {
-		return fmt.Errorf("Failed to delete %s:%v", fileUrlOnVolume, err)
-	}
-	return nil
-}
-
-func DeleteFile(master string, fileId string, jwt security.EncodedJwt) error {
-	fileUrl, err := LookupFileId(master, fileId)
-	if err != nil {
-		return fmt.Errorf("Failed to lookup %s:%v", fileId, err)
-	}
-	err = util.Delete(fileUrl, jwt)
-	if err != nil {
-		return fmt.Errorf("Failed to delete %s:%v", fileUrl, err)
-	}
-	return nil
-}
-
 func ParseFileId(fid string) (vid string, key_cookie string, err error) {
 	commaIndex := strings.Index(fid, ",")
 	if commaIndex <= 0 {
@@ -49,20 +26,28 @@ func ParseFileId(fid string) (vid string, key_cookie string, err error) {
 	return fid[:commaIndex], fid[commaIndex+1:], nil
 }
 
-type DeleteFilesResult struct {
-	Errors  []string
-	Results []DeleteResult
+// DeleteFiles batch deletes a list of fileIds
+func DeleteFiles(master string, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
+
+	lookupFunc := func(vids []string) (map[string]LookupResult, error) {
+		return LookupVolumeIds(master, vids)
+	}
+
+	return DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+
 }
 
-func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
+func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []string) (map[string]LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
+
+	var ret []*volume_server_pb.DeleteResult
+
 	vid_to_fileIds := make(map[string][]string)
-	ret := &DeleteFilesResult{}
 	var vids []string
 	for _, fileId := range fileIds {
 		vid, _, err := ParseFileId(fileId)
 		if err != nil {
-			ret.Results = append(ret.Results, DeleteResult{
-				Fid:    vid,
+			ret = append(ret, &volume_server_pb.DeleteResult{
+				FileId: vid,
 				Status: http.StatusBadRequest,
 				Error:  err.Error()},
 			)
@@ -75,7 +60,7 @@ func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
 		vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
 	}
 
-	lookupResults, err := LookupVolumeIds(master, vids)
+	lookupResults, err := lookupFunc(vids)
 	if err != nil {
 		return ret, err
 	}
@@ -83,7 +68,11 @@ func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
 	server_to_fileIds := make(map[string][]string)
 	for vid, result := range lookupResults {
 		if result.Error != "" {
-			ret.Errors = append(ret.Errors, result.Error)
+			ret = append(ret, &volume_server_pb.DeleteResult{
+				FileId: vid,
+				Status: http.StatusBadRequest,
+				Error:  err.Error()},
+			)
 			continue
 		}
 		for _, location := range result.Locations {
@@ -101,25 +90,52 @@ func DeleteFiles(master string, fileIds []string) (*DeleteFilesResult, error) {
 		wg.Add(1)
 		go func(server string, fidList []string) {
 			defer wg.Done()
-			values := make(url.Values)
-			for _, fid := range fidList {
-				values.Add("fid", fid)
+
+			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, fidList); deleteErr != nil {
+				err = deleteErr
+			} else {
+				ret = append(ret, deleteResults...)
 			}
-			jsonBlob, err := util.Post("http://"+server+"/delete", values)
-			if err != nil {
-				ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
-				return
-			}
-			var result []DeleteResult
-			err = json.Unmarshal(jsonBlob, &result)
-			if err != nil {
-				ret.Errors = append(ret.Errors, err.Error()+" "+string(jsonBlob))
-				return
-			}
-			ret.Results = append(ret.Results, result...)
+
 		}(server, fidList)
 	}
 	wg.Wait()
 
-	return ret, nil
+	return ret, err
+}
+
+// DeleteFilesAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
+func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
+
+	err = WithVolumeServerClient(volumeServer, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+
+		req := &volume_server_pb.BatchDeleteRequest{
+			FileIds: fileIds,
+		}
+
+		resp, err := volumeServerClient.BatchDelete(context.Background(), req)
+
+		// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)
+
+		if err != nil {
+			return err
+		}
+
+		ret = append(ret, resp.Results...)
+
+		return nil
+	})
+
+	if err != nil {
+		return
+	}
+
+	for _, result := range ret {
+		if result.Error != "" && result.Error != "Not Found" {
+			return nil, fmt.Errorf("delete fileId %s: %v", result.FileId, result.Error)
+		}
+	}
+
+	return
+
 }

weed/operation/grpc_client.go

@@ -0,0 +1,52 @@
+package operation
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+var (
+	grpcClients     = make(map[string]*grpc.ClientConn)
+	grpcClientsLock sync.Mutex
+)
+
+func WithVolumeServerClient(volumeServer string, fn func(volume_server_pb.VolumeServerClient) error) error {
+
+	grpcAddress, err := toVolumeServerGrpcAddress(volumeServer)
+	if err != nil {
+		return err
+	}
+
+	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+		client := volume_server_pb.NewVolumeServerClient(grpcConnection)
+		return fn(client)
+	}, grpcAddress)
+
+}
+
+func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err error) {
+	sepIndex := strings.LastIndex(volumeServer, ":")
+	port, err := strconv.Atoi(volumeServer[sepIndex+1:])
+	if err != nil {
+		glog.Errorf("failed to parse volume server address: %v", volumeServer)
+		return "", err
+	}
+	return fmt.Sprintf("%s:%d", volumeServer[0:sepIndex], port+10000), nil
+}
+
+func withMasterServerClient(masterServer string, fn func(masterClient master_pb.SeaweedClient) error) error {
+
+	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+		client := master_pb.NewSeaweedClient(grpcConnection)
+		return fn(client)
+	}, masterServer)
+
+}

weed/operation/lookup.go

@@ -1,6 +1,7 @@
 package operation
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -9,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -95,24 +97,41 @@ func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, err
 	}
 
 	//only query unknown_vids
-	values := make(url.Values)
-	for _, vid := range unknown_vids {
-		values.Add("volumeId", vid)
-	}
-	jsonBlob, err := util.Post("http://"+server+"/vol/lookup", values)
+
+	err := withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+		req := &master_pb.LookupVolumeRequest{
+			VolumeIds: unknown_vids,
+		}
+		resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
+		if grpcErr != nil {
+			return grpcErr
+		}
+
+		//set newly checked vids to cache
+		for _, vidLocations := range resp.VolumeIdLocations {
+			var locations []Location
+			for _, loc := range vidLocations.Locations {
+				locations = append(locations, Location{
+					Url:       loc.Url,
+					PublicUrl: loc.PublicUrl,
+				})
+			}
+			if vidLocations.Error != "" {
+				vc.Set(vidLocations.VolumeId, locations, 10*time.Minute)
+			}
+			ret[vidLocations.VolumeId] = LookupResult{
+				VolumeId:  vidLocations.VolumeId,
+				Locations: locations,
+				Error:     vidLocations.Error,
+			}
+		}
+
+		return nil
+	})
+
 	if err != nil {
 		return nil, err
 	}
-	err = json.Unmarshal(jsonBlob, &ret)
-	if err != nil {
-		return nil, errors.New(err.Error() + " " + string(jsonBlob))
-	}
-
-	//set newly checked vids to cache
-	for _, vid := range unknown_vids {
-		locations := ret[vid].Locations
-		vc.Set(vid, locations, 10*time.Minute)
-	}
 
 	return ret, nil
 }

weed/operation/stats.go

@@ -0,0 +1,24 @@
+package operation
+
+import (
+	"context"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+)
+
+func Statistics(server string, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
+
+	err = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+		grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req)
+		if grpcErr != nil {
+			return grpcErr
+		}
+
+		resp = grpcResponse
+
+		return nil
+
+	})
+
+	return
+}

weed/operation/submit.go

@@ -18,7 +18,6 @@ type FilePart struct {
 	Reader      io.Reader
 	FileName    string
 	FileSize    int64
-	IsGzipped   bool
 	MimeType    string
 	ModTime     int64 //in seconds
 	Replication string
@@ -103,7 +102,6 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
 	ret.ModTime = fi.ModTime().UTC().Unix()
 	ret.FileSize = fi.Size()
 	ext := strings.ToLower(path.Ext(fullPathFilename))
-	ret.IsGzipped = ext == ".gz"
 	ret.FileName = fi.Name()
 	if ext != "" {
 		ret.MimeType = mime.TypeByExtension(ext)
@@ -193,7 +191,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 			cm.DeleteChunks(master)
 		}
 	} else {
-		ret, e := Upload(fileUrl, baseName, fi.Reader, fi.IsGzipped, fi.MimeType, nil, jwt)
+		ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt)
 		if e != nil {
 			return 0, e
 		}
@@ -203,7 +201,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 }
 
 func upload_one_chunk(filename string, reader io.Reader, master,
-	fileUrl string, jwt security.EncodedJwt,
+fileUrl string, jwt security.EncodedJwt,
 ) (size uint32, e error) {
 	glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
 	uploadResult, uploadError := Upload(fileUrl, filename, reader, false,

weed/operation/sync_volume.go

@@ -1,55 +1,60 @@
 package operation
 
 import (
-	"encoding/json"
+	"context"
 	"fmt"
-	"net/url"
+	"io"
 
-	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-type SyncVolumeResponse struct {
-	Replication     string `json:"Replication,omitempty"`
-	Ttl             string `json:"Ttl,omitempty"`
-	TailOffset      uint64 `json:"TailOffset,omitempty"`
-	CompactRevision uint16 `json:"CompactRevision,omitempty"`
-	IdxFileSize     uint64 `json:"IdxFileSize,omitempty"`
-	Error           string `json:"error,omitempty"`
-}
+func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
 
-func GetVolumeSyncStatus(server string, vid string) (*SyncVolumeResponse, error) {
-	values := make(url.Values)
-	values.Add("volume", vid)
-	jsonBlob, err := util.Post("http://"+server+"/admin/sync/status", values)
-	glog.V(2).Info("sync volume result :", string(jsonBlob))
-	if err != nil {
-		return nil, err
-	}
-	var ret SyncVolumeResponse
-	err = json.Unmarshal(jsonBlob, &ret)
-	if err != nil {
-		return nil, err
-	}
-	if ret.Error != "" {
-		return nil, fmt.Errorf("Volume %s get sync status error: %s", vid, ret.Error)
-	}
-	return &ret, nil
+	WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
+		resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
+			VolumdId: vid,
+		})
+		return nil
+	})
+
+	return
 }
+
+func GetVolumeIdxEntries(server string, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
+
+	return WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
+		stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
+			VolumdId: vid,
+		})
+		if err != nil {
+			return err
+		}
+
+		var indexFileContent []byte
+
+		for {
+			resp, err := stream.Recv()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return fmt.Errorf("read index entries: %v", err)
+			}
+			indexFileContent = append(indexFileContent, resp.IndexFileContent...)
+		}
+
+		dataSize := len(indexFileContent)
+
+		for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
+			line := indexFileContent[idx : idx+NeedleEntrySize]
+			key := BytesToNeedleId(line[:NeedleIdSize])
+			offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
+			size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
+			eachEntryFn(key, offset, size)
+		}
 
-func GetVolumeIdxEntries(server string, vid string, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
-	values := make(url.Values)
-	values.Add("volume", vid)
-	line := make([]byte, NeedleEntrySize)
-	err := util.GetBufferStream("http://"+server+"/admin/sync/index", values, line, func(bytes []byte) {
-		key := BytesToNeedleId(line[:NeedleIdSize])
-		offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
-		size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
-		eachEntryFn(key, offset, size)
+		return nil
 	})
-	if err != nil {
-		return err
-	}
-	return nil
 }

weed/operation/upload_content.go

@@ -22,6 +22,7 @@ type UploadResult struct {
 	Name  string `json:"name,omitempty"`
 	Size  uint32 `json:"size,omitempty"`
 	Error string `json:"error,omitempty"`
+	ETag  string `json:"error,omitempty"`
 }
 
 var (
@@ -36,6 +37,7 @@ func init() {
 
 var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
+// Upload sends a POST request to a volume server to upload the content
 func Upload(uploadUrl string, filename string, reader io.Reader, isGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (*UploadResult, error) {
 	return upload_content(uploadUrl, func(w io.Writer) (err error) {
 		_, err = io.Copy(w, reader)
@@ -90,6 +92,13 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 		return nil, post_err
 	}
 	defer resp.Body.Close()
+
+	if resp.StatusCode < http.StatusOK ||
+		resp.StatusCode > http.StatusIMUsed {
+		return nil, errors.New(http.StatusText(resp.StatusCode))
+	}
+
+	etag := getEtag(resp)
 	resp_body, ra_err := ioutil.ReadAll(resp.Body)
 	if ra_err != nil {
 		return nil, ra_err
@@ -103,5 +112,14 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	if ret.Error != "" {
 		return nil, errors.New(ret.Error)
 	}
+	ret.ETag = etag
 	return &ret, nil
 }
+
+func getEtag(r *http.Response) (etag string) {
+	etag = r.Header.Get("ETag")
+	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
+		etag = etag[1 : len(etag)-1]
+	}
+	return
+}

weed/pb/Makefile

@@ -3,5 +3,8 @@ all: gen
 .PHONY : gen
 
 gen:
-	protoc seaweed.proto --go_out=plugins=grpc:./master_pb
+	protoc master.proto --go_out=plugins=grpc:./master_pb
+	protoc volume_server.proto --go_out=plugins=grpc:./volume_server_pb
 	protoc filer.proto --go_out=plugins=grpc:./filer_pb
+	# protoc filer.proto --java_out=../../other/java/client/src/main/java
+	cp filer.proto ../../other/java/client/src/main/proto

weed/pb/filer.proto

@@ -2,6 +2,9 @@ syntax = "proto3";
 
 package filer_pb;
 
+option java_package = "seaweedfs.client";
+option java_outer_classname = "FilerProto";
+
 //////////////////////////////////////////////////
 
 service SeaweedFiler {
@@ -12,9 +15,6 @@ service SeaweedFiler {
     rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) {
     }
 
-    rpc GetEntryAttributes (GetEntryAttributesRequest) returns (GetEntryAttributesResponse) {
-    }
-
     rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
     }
 
@@ -33,6 +33,9 @@ service SeaweedFiler {
     rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
     }
 
+    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+    }
+
 }
 
 //////////////////////////////////////////////////
@@ -63,11 +66,13 @@ message Entry {
     bool is_directory = 2;
     repeated FileChunk chunks = 3;
     FuseAttributes attributes = 4;
+    map<string, bytes> extended = 5;
 }
 
 message EventNotification {
-    Entry old_entry = 2;
-    Entry new_entry = 3;
+    Entry old_entry = 1;
+    Entry new_entry = 2;
+    bool delete_chunks = 3;
 }
 
 message FileChunk {
@@ -75,38 +80,23 @@ message FileChunk {
     int64 offset = 2;
     uint64 size = 3;
     int64 mtime = 4;
+    string e_tag = 5;
+    string source_file_id = 6;
 }
 
 message FuseAttributes {
     uint64 file_size = 1;
-    int64 mtime = 2;
+    int64 mtime = 2; // unix time in seconds
     uint32 file_mode = 3;
     uint32 uid = 4;
     uint32 gid = 5;
-    int64 crtime = 6;
+    int64 crtime = 6; // unix time in seconds
     string mime = 7;
     string replication = 8;
     string collection = 9;
     int32 ttl_sec = 10;
-}
-
-message GetEntryAttributesRequest {
-    string name = 1;
-    string parent_dir = 2;
-    string file_id = 3;
-}
-
-message GetEntryAttributesResponse {
-    FuseAttributes attributes = 1;
-    repeated FileChunk chunks = 2;
-}
-
-message GetFileContentRequest {
-    string file_id = 1;
-}
-
-message GetFileContentResponse {
-    bytes content = 1;
+    string user_name = 11; // for hdfs
+    repeated string group_name = 12; // for hdfs
 }
 
 message CreateEntryRequest {
@@ -127,7 +117,7 @@ message UpdateEntryResponse {
 message DeleteEntryRequest {
     string directory = 1;
     string name = 2;
-    bool is_directory = 3;
+    // bool is_directory = 3;
     bool is_delete_data = 4;
     bool is_recursive = 5;
 }
@@ -172,3 +162,17 @@ message DeleteCollectionRequest {
 
 message DeleteCollectionResponse {
 }
+
+message StatisticsRequest {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+}
+message StatisticsResponse {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+    uint64 total_size = 4;
+    uint64 used_size = 5;
+    uint64 file_count = 6;
+}

weed/pb/filer_pb/filer.pb.go

@@ -17,10 +17,6 @@ It has these top-level messages:
 	EventNotification
 	FileChunk
 	FuseAttributes
-	GetEntryAttributesRequest
-	GetEntryAttributesResponse
-	GetFileContentRequest
-	GetFileContentResponse
 	CreateEntryRequest
 	CreateEntryResponse
 	UpdateEntryRequest
@@ -35,6 +31,8 @@ It has these top-level messages:
 	LookupVolumeResponse
 	DeleteCollectionRequest
 	DeleteCollectionResponse
+	StatisticsRequest
+	StatisticsResponse
 */
 package filer_pb
 
@@ -163,10 +161,11 @@ func (m *ListEntriesResponse) GetEntries() []*Entry {
 }
 
 type Entry struct {
-	Name        string          `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
-	IsDirectory bool            `protobuf:"varint,2,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"`
-	Chunks      []*FileChunk    `protobuf:"bytes,3,rep,name=chunks" json:"chunks,omitempty"`
-	Attributes  *FuseAttributes `protobuf:"bytes,4,opt,name=attributes" json:"attributes,omitempty"`
+	Name        string            `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	IsDirectory bool              `protobuf:"varint,2,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"`
+	Chunks      []*FileChunk      `protobuf:"bytes,3,rep,name=chunks" json:"chunks,omitempty"`
+	Attributes  *FuseAttributes   `protobuf:"bytes,4,opt,name=attributes" json:"attributes,omitempty"`
+	Extended    map[string][]byte `protobuf:"bytes,5,rep,name=extended" json:"extended,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"`
 }
 
 func (m *Entry) Reset()                    { *m = Entry{} }
@@ -202,10 +201,17 @@ func (m *Entry) GetAttributes() *FuseAttributes {
 	return nil
 }
 
+func (m *Entry) GetExtended() map[string][]byte {
+	if m != nil {
+		return m.Extended
+	}
+	return nil
+}
+
 type EventNotification struct {
-	Name     string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
-	OldEntry *Entry `protobuf:"bytes,2,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"`
-	NewEntry *Entry `protobuf:"bytes,3,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"`
+	OldEntry     *Entry `protobuf:"bytes,1,opt,name=old_entry,json=oldEntry" json:"old_entry,omitempty"`
+	NewEntry     *Entry `protobuf:"bytes,2,opt,name=new_entry,json=newEntry" json:"new_entry,omitempty"`
+	DeleteChunks bool   `protobuf:"varint,3,opt,name=delete_chunks,json=deleteChunks" json:"delete_chunks,omitempty"`
 }
 
 func (m *EventNotification) Reset()                    { *m = EventNotification{} }
@@ -213,13 +219,6 @@ func (m *EventNotification) String() string            { return proto.CompactTex
 func (*EventNotification) ProtoMessage()               {}
 func (*EventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
 
-func (m *EventNotification) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
 func (m *EventNotification) GetOldEntry() *Entry {
 	if m != nil {
 		return m.OldEntry
@@ -234,11 +233,20 @@ func (m *EventNotification) GetNewEntry() *Entry {
 	return nil
 }
 
+func (m *EventNotification) GetDeleteChunks() bool {
+	if m != nil {
+		return m.DeleteChunks
+	}
+	return false
+}
+
 type FileChunk struct {
-	FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
-	Offset int64  `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
-	Size   uint64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
-	Mtime  int64  `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
+	FileId       string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
+	Offset       int64  `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
+	Size         uint64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
+	Mtime        int64  `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
+	ETag         string `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
+	SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
 }
 
 func (m *FileChunk) Reset()                    { *m = FileChunk{} }
@@ -274,17 +282,33 @@ func (m *FileChunk) GetMtime() int64 {
 	return 0
 }
 
+func (m *FileChunk) GetETag() string {
+	if m != nil {
+		return m.ETag
+	}
+	return ""
+}
+
+func (m *FileChunk) GetSourceFileId() string {
+	if m != nil {
+		return m.SourceFileId
+	}
+	return ""
+}
+
 type FuseAttributes struct {
-	FileSize    uint64 `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
-	Mtime       int64  `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
-	FileMode    uint32 `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"`
-	Uid         uint32 `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"`
-	Gid         uint32 `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"`
-	Crtime      int64  `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"`
-	Mime        string `protobuf:"bytes,7,opt,name=mime" json:"mime,omitempty"`
-	Replication string `protobuf:"bytes,8,opt,name=replication" json:"replication,omitempty"`
-	Collection  string `protobuf:"bytes,9,opt,name=collection" json:"collection,omitempty"`
-	TtlSec      int32  `protobuf:"varint,10,opt,name=ttl_sec,json=ttlSec" json:"ttl_sec,omitempty"`
+	FileSize    uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
+	Mtime       int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
+	FileMode    uint32   `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"`
+	Uid         uint32   `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"`
+	Gid         uint32   `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"`
+	Crtime      int64    `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"`
+	Mime        string   `protobuf:"bytes,7,opt,name=mime" json:"mime,omitempty"`
+	Replication string   `protobuf:"bytes,8,opt,name=replication" json:"replication,omitempty"`
+	Collection  string   `protobuf:"bytes,9,opt,name=collection" json:"collection,omitempty"`
+	TtlSec      int32    `protobuf:"varint,10,opt,name=ttl_sec,json=ttlSec" json:"ttl_sec,omitempty"`
+	UserName    string   `protobuf:"bytes,11,opt,name=user_name,json=userName" json:"user_name,omitempty"`
+	GroupName   []string `protobuf:"bytes,12,rep,name=group_name,json=groupName" json:"group_name,omitempty"`
 }
 
 func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
@@ -362,90 +386,16 @@ func (m *FuseAttributes) GetTtlSec() int32 {
 	return 0
 }
 
-type GetEntryAttributesRequest struct {
-	Name      string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
-	ParentDir string `protobuf:"bytes,2,opt,name=parent_dir,json=parentDir" json:"parent_dir,omitempty"`
-	FileId    string `protobuf:"bytes,3,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
-}
-
-func (m *GetEntryAttributesRequest) Reset()                    { *m = GetEntryAttributesRequest{} }
-func (m *GetEntryAttributesRequest) String() string            { return proto.CompactTextString(m) }
-func (*GetEntryAttributesRequest) ProtoMessage()               {}
-func (*GetEntryAttributesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
-
-func (m *GetEntryAttributesRequest) GetName() string {
+func (m *FuseAttributes) GetUserName() string {
 	if m != nil {
-		return m.Name
+		return m.UserName
 	}
 	return ""
 }
 
-func (m *GetEntryAttributesRequest) GetParentDir() string {
+func (m *FuseAttributes) GetGroupName() []string {
 	if m != nil {
-		return m.ParentDir
-	}
-	return ""
-}
-
-func (m *GetEntryAttributesRequest) GetFileId() string {
-	if m != nil {
-		return m.FileId
-	}
-	return ""
-}
-
-type GetEntryAttributesResponse struct {
-	Attributes *FuseAttributes `protobuf:"bytes,1,opt,name=attributes" json:"attributes,omitempty"`
-	Chunks     []*FileChunk    `protobuf:"bytes,2,rep,name=chunks" json:"chunks,omitempty"`
-}
-
-func (m *GetEntryAttributesResponse) Reset()                    { *m = GetEntryAttributesResponse{} }
-func (m *GetEntryAttributesResponse) String() string            { return proto.CompactTextString(m) }
-func (*GetEntryAttributesResponse) ProtoMessage()               {}
-func (*GetEntryAttributesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
-
-func (m *GetEntryAttributesResponse) GetAttributes() *FuseAttributes {
-	if m != nil {
-		return m.Attributes
-	}
-	return nil
-}
-
-func (m *GetEntryAttributesResponse) GetChunks() []*FileChunk {
-	if m != nil {
-		return m.Chunks
-	}
-	return nil
-}
-
-type GetFileContentRequest struct {
-	FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
-}
-
-func (m *GetFileContentRequest) Reset()                    { *m = GetFileContentRequest{} }
-func (m *GetFileContentRequest) String() string            { return proto.CompactTextString(m) }
-func (*GetFileContentRequest) ProtoMessage()               {}
-func (*GetFileContentRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
-
-func (m *GetFileContentRequest) GetFileId() string {
-	if m != nil {
-		return m.FileId
-	}
-	return ""
-}
-
-type GetFileContentResponse struct {
-	Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"`
-}
-
-func (m *GetFileContentResponse) Reset()                    { *m = GetFileContentResponse{} }
-func (m *GetFileContentResponse) String() string            { return proto.CompactTextString(m) }
-func (*GetFileContentResponse) ProtoMessage()               {}
-func (*GetFileContentResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
-
-func (m *GetFileContentResponse) GetContent() []byte {
-	if m != nil {
-		return m.Content
+		return m.GroupName
 	}
 	return nil
 }
@@ -458,7 +408,7 @@ type CreateEntryRequest struct {
 func (m *CreateEntryRequest) Reset()                    { *m = CreateEntryRequest{} }
 func (m *CreateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryRequest) ProtoMessage()               {}
-func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
 
 func (m *CreateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -480,7 +430,7 @@ type CreateEntryResponse struct {
 func (m *CreateEntryResponse) Reset()                    { *m = CreateEntryResponse{} }
 func (m *CreateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryResponse) ProtoMessage()               {}
-func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
 
 type UpdateEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@@ -490,7 +440,7 @@ type UpdateEntryRequest struct {
 func (m *UpdateEntryRequest) Reset()                    { *m = UpdateEntryRequest{} }
 func (m *UpdateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryRequest) ProtoMessage()               {}
-func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
 
 func (m *UpdateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -512,20 +462,20 @@ type UpdateEntryResponse struct {
 func (m *UpdateEntryResponse) Reset()                    { *m = UpdateEntryResponse{} }
 func (m *UpdateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryResponse) ProtoMessage()               {}
-func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
 
 type DeleteEntryRequest struct {
-	Directory    string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
-	Name         string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
-	IsDirectory  bool   `protobuf:"varint,3,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"`
-	IsDeleteData bool   `protobuf:"varint,4,opt,name=is_delete_data,json=isDeleteData" json:"is_delete_data,omitempty"`
-	IsRecursive  bool   `protobuf:"varint,5,opt,name=is_recursive,json=isRecursive" json:"is_recursive,omitempty"`
+	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
+	Name      string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
+	// bool is_directory = 3;
+	IsDeleteData bool `protobuf:"varint,4,opt,name=is_delete_data,json=isDeleteData" json:"is_delete_data,omitempty"`
+	IsRecursive  bool `protobuf:"varint,5,opt,name=is_recursive,json=isRecursive" json:"is_recursive,omitempty"`
 }
 
 func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
 func (m *DeleteEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryRequest) ProtoMessage()               {}
-func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
 
 func (m *DeleteEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -541,13 +491,6 @@ func (m *DeleteEntryRequest) GetName() string {
 	return ""
 }
 
-func (m *DeleteEntryRequest) GetIsDirectory() bool {
-	if m != nil {
-		return m.IsDirectory
-	}
-	return false
-}
-
 func (m *DeleteEntryRequest) GetIsDeleteData() bool {
 	if m != nil {
 		return m.IsDeleteData
@@ -568,7 +511,7 @@ type DeleteEntryResponse struct {
 func (m *DeleteEntryResponse) Reset()                    { *m = DeleteEntryResponse{} }
 func (m *DeleteEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryResponse) ProtoMessage()               {}
-func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
 
 type AssignVolumeRequest struct {
 	Count       int32  `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
@@ -581,7 +524,7 @@ type AssignVolumeRequest struct {
 func (m *AssignVolumeRequest) Reset()                    { *m = AssignVolumeRequest{} }
 func (m *AssignVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeRequest) ProtoMessage()               {}
-func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
 
 func (m *AssignVolumeRequest) GetCount() int32 {
 	if m != nil {
@@ -628,7 +571,7 @@ type AssignVolumeResponse struct {
 func (m *AssignVolumeResponse) Reset()                    { *m = AssignVolumeResponse{} }
 func (m *AssignVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeResponse) ProtoMessage()               {}
-func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
 
 func (m *AssignVolumeResponse) GetFileId() string {
 	if m != nil {
@@ -665,7 +608,7 @@ type LookupVolumeRequest struct {
 func (m *LookupVolumeRequest) Reset()                    { *m = LookupVolumeRequest{} }
 func (m *LookupVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeRequest) ProtoMessage()               {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
 
 func (m *LookupVolumeRequest) GetVolumeIds() []string {
 	if m != nil {
@@ -681,7 +624,7 @@ type Locations struct {
 func (m *Locations) Reset()                    { *m = Locations{} }
 func (m *Locations) String() string            { return proto.CompactTextString(m) }
 func (*Locations) ProtoMessage()               {}
-func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
 
 func (m *Locations) GetLocations() []*Location {
 	if m != nil {
@@ -698,7 +641,7 @@ type Location struct {
 func (m *Location) Reset()                    { *m = Location{} }
 func (m *Location) String() string            { return proto.CompactTextString(m) }
 func (*Location) ProtoMessage()               {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
 
 func (m *Location) GetUrl() string {
 	if m != nil {
@@ -721,7 +664,7 @@ type LookupVolumeResponse struct {
 func (m *LookupVolumeResponse) Reset()                    { *m = LookupVolumeResponse{} }
 func (m *LookupVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse) ProtoMessage()               {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
 
 func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations {
 	if m != nil {
@@ -737,7 +680,7 @@ type DeleteCollectionRequest struct {
 func (m *DeleteCollectionRequest) Reset()                    { *m = DeleteCollectionRequest{} }
 func (m *DeleteCollectionRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionRequest) ProtoMessage()               {}
-func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
 
 func (m *DeleteCollectionRequest) GetCollection() string {
 	if m != nil {
@@ -752,7 +695,95 @@ type DeleteCollectionResponse struct {
 func (m *DeleteCollectionResponse) Reset()                    { *m = DeleteCollectionResponse{} }
 func (m *DeleteCollectionResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionResponse) ProtoMessage()               {}
-func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+
+type StatisticsRequest struct {
+	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
+	Collection  string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
+	Ttl         string `protobuf:"bytes,3,opt,name=ttl" json:"ttl,omitempty"`
+}
+
+func (m *StatisticsRequest) Reset()                    { *m = StatisticsRequest{} }
+func (m *StatisticsRequest) String() string            { return proto.CompactTextString(m) }
+func (*StatisticsRequest) ProtoMessage()               {}
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+
+func (m *StatisticsRequest) GetReplication() string {
+	if m != nil {
+		return m.Replication
+	}
+	return ""
+}
+
+func (m *StatisticsRequest) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+func (m *StatisticsRequest) GetTtl() string {
+	if m != nil {
+		return m.Ttl
+	}
+	return ""
+}
+
+type StatisticsResponse struct {
+	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
+	Collection  string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
+	Ttl         string `protobuf:"bytes,3,opt,name=ttl" json:"ttl,omitempty"`
+	TotalSize   uint64 `protobuf:"varint,4,opt,name=total_size,json=totalSize" json:"total_size,omitempty"`
+	UsedSize    uint64 `protobuf:"varint,5,opt,name=used_size,json=usedSize" json:"used_size,omitempty"`
+	FileCount   uint64 `protobuf:"varint,6,opt,name=file_count,json=fileCount" json:"file_count,omitempty"`
+}
+
+func (m *StatisticsResponse) Reset()                    { *m = StatisticsResponse{} }
+func (m *StatisticsResponse) String() string            { return proto.CompactTextString(m) }
+func (*StatisticsResponse) ProtoMessage()               {}
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+
+func (m *StatisticsResponse) GetReplication() string {
+	if m != nil {
+		return m.Replication
+	}
+	return ""
+}
+
+func (m *StatisticsResponse) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+func (m *StatisticsResponse) GetTtl() string {
+	if m != nil {
+		return m.Ttl
+	}
+	return ""
+}
+
+func (m *StatisticsResponse) GetTotalSize() uint64 {
+	if m != nil {
+		return m.TotalSize
+	}
+	return 0
+}
+
+func (m *StatisticsResponse) GetUsedSize() uint64 {
+	if m != nil {
+		return m.UsedSize
+	}
+	return 0
+}
+
+func (m *StatisticsResponse) GetFileCount() uint64 {
+	if m != nil {
+		return m.FileCount
+	}
+	return 0
+}
 
 func init() {
 	proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest")
@@ -763,10 +794,6 @@ func init() {
 	proto.RegisterType((*EventNotification)(nil), "filer_pb.EventNotification")
 	proto.RegisterType((*FileChunk)(nil), "filer_pb.FileChunk")
 	proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes")
-	proto.RegisterType((*GetEntryAttributesRequest)(nil), "filer_pb.GetEntryAttributesRequest")
-	proto.RegisterType((*GetEntryAttributesResponse)(nil), "filer_pb.GetEntryAttributesResponse")
-	proto.RegisterType((*GetFileContentRequest)(nil), "filer_pb.GetFileContentRequest")
-	proto.RegisterType((*GetFileContentResponse)(nil), "filer_pb.GetFileContentResponse")
 	proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest")
 	proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse")
 	proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest")
@@ -781,6 +808,8 @@ func init() {
 	proto.RegisterType((*LookupVolumeResponse)(nil), "filer_pb.LookupVolumeResponse")
 	proto.RegisterType((*DeleteCollectionRequest)(nil), "filer_pb.DeleteCollectionRequest")
 	proto.RegisterType((*DeleteCollectionResponse)(nil), "filer_pb.DeleteCollectionResponse")
+	proto.RegisterType((*StatisticsRequest)(nil), "filer_pb.StatisticsRequest")
+	proto.RegisterType((*StatisticsResponse)(nil), "filer_pb.StatisticsResponse")
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -796,13 +825,13 @@ const _ = grpc.SupportPackageIsVersion4
 type SeaweedFilerClient interface {
 	LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error)
 	ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error)
-	GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error)
 	CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
 	UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
 	DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
 	AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
 	LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error)
 	DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
+	Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
 }
 
 type seaweedFilerClient struct {
@@ -831,15 +860,6 @@ func (c *seaweedFilerClient) ListEntries(ctx context.Context, in *ListEntriesReq
 	return out, nil
 }
 
-func (c *seaweedFilerClient) GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) {
-	out := new(GetEntryAttributesResponse)
-	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetEntryAttributes", in, out, c.cc, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
 func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) {
 	out := new(CreateEntryResponse)
 	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/CreateEntry", in, out, c.cc, opts...)
@@ -894,18 +914,27 @@ func (c *seaweedFilerClient) DeleteCollection(ctx context.Context, in *DeleteCol
 	return out, nil
 }
 
+func (c *seaweedFilerClient) Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) {
+	out := new(StatisticsResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/Statistics", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for SeaweedFiler service
 
 type SeaweedFilerServer interface {
 	LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error)
 	ListEntries(context.Context, *ListEntriesRequest) (*ListEntriesResponse, error)
-	GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error)
 	CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
 	UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
 	DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
 	AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
 	LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error)
 	DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
+	Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
 }
 
 func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) {
@@ -948,24 +977,6 @@ func _SeaweedFiler_ListEntries_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
-func _SeaweedFiler_GetEntryAttributes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(GetEntryAttributesRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(SeaweedFilerServer).GetEntryAttributes(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/filer_pb.SeaweedFiler/GetEntryAttributes",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SeaweedFilerServer).GetEntryAttributes(ctx, req.(*GetEntryAttributesRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
 func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(CreateEntryRequest)
 	if err := dec(in); err != nil {
@@ -1074,6 +1085,24 @@ func _SeaweedFiler_DeleteCollection_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedFiler_Statistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(StatisticsRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).Statistics(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/Statistics",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).Statistics(ctx, req.(*StatisticsRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "filer_pb.SeaweedFiler",
 	HandlerType: (*SeaweedFilerServer)(nil),
@@ -1086,10 +1115,6 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			MethodName: "ListEntries",
 			Handler:    _SeaweedFiler_ListEntries_Handler,
 		},
-		{
-			MethodName: "GetEntryAttributes",
-			Handler:    _SeaweedFiler_GetEntryAttributes_Handler,
-		},
 		{
 			MethodName: "CreateEntry",
 			Handler:    _SeaweedFiler_CreateEntry_Handler,
@@ -1114,6 +1139,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			MethodName: "DeleteCollection",
 			Handler:    _SeaweedFiler_DeleteCollection_Handler,
 		},
+		{
+			MethodName: "Statistics",
+			Handler:    _SeaweedFiler_Statistics_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "filer.proto",
@@ -1122,78 +1151,85 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1154 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xcd, 0x6e, 0xdb, 0x46,
-	0x10, 0x36, 0x25, 0x4b, 0x36, 0x47, 0x72, 0x1a, 0xaf, 0x9c, 0x84, 0x55, 0xec, 0x54, 0xd9, 0x26,
-	0x85, 0x83, 0x1a, 0x86, 0xe1, 0xf6, 0x90, 0xb6, 0x28, 0xd0, 0xc0, 0x76, 0x82, 0x00, 0x4e, 0x02,
-	0xd0, 0x75, 0x81, 0xa2, 0x07, 0x81, 0x26, 0x47, 0xee, 0xc2, 0x14, 0xa9, 0x72, 0x97, 0x76, 0xdd,
-	0x6b, 0x8f, 0x3d, 0xf5, 0x21, 0x8a, 0xde, 0xfb, 0x00, 0xbd, 0xf4, 0xc5, 0x8a, 0xfd, 0xa1, 0xb4,
-	0x14, 0x29, 0x27, 0x39, 0xe4, 0xb6, 0x3b, 0x33, 0x3b, 0x33, 0xdf, 0xec, 0xcc, 0xc7, 0x25, 0x74,
-	0x46, 0x2c, 0xc6, 0x6c, 0x77, 0x92, 0xa5, 0x22, 0x25, 0xab, 0x6a, 0x33, 0x9c, 0x9c, 0xd1, 0x37,
-	0x70, 0xff, 0x38, 0x4d, 0x2f, 0xf2, 0xc9, 0x21, 0xcb, 0x30, 0x14, 0x69, 0x76, 0x7d, 0x94, 0x88,
-	0xec, 0xda, 0xc7, 0x5f, 0x72, 0xe4, 0x82, 0x6c, 0x82, 0x1b, 0x15, 0x0a, 0xcf, 0x19, 0x38, 0xdb,
-	0xae, 0x3f, 0x13, 0x10, 0x02, 0xcb, 0x49, 0x30, 0x46, 0xaf, 0xa1, 0x14, 0x6a, 0x4d, 0x8f, 0x60,
-	0xb3, 0xde, 0x21, 0x9f, 0xa4, 0x09, 0x47, 0xf2, 0x18, 0x5a, 0x28, 0x05, 0xca, 0x5b, 0x67, 0xff,
-	0xa3, 0xdd, 0x22, 0x95, 0x5d, 0x6d, 0xa7, 0xb5, 0xf4, 0x5f, 0x07, 0xc8, 0x31, 0xe3, 0x42, 0x0a,
-	0x19, 0xf2, 0x77, 0xcb, 0xe7, 0x2e, 0xb4, 0x27, 0x19, 0x8e, 0xd8, 0xaf, 0x26, 0x23, 0xb3, 0x23,
-	0x3b, 0xb0, 0xce, 0x45, 0x90, 0x89, 0xe7, 0x59, 0x3a, 0x7e, 0xce, 0x62, 0x7c, 0x2d, 0x93, 0x6e,
-	0x2a, 0x93, 0xaa, 0x82, 0xec, 0x02, 0x61, 0x49, 0x18, 0xe7, 0x9c, 0x5d, 0xe2, 0x49, 0xa1, 0xf5,
-	0x96, 0x07, 0xce, 0xf6, 0xaa, 0x5f, 0xa3, 0x21, 0x1b, 0xd0, 0x8a, 0xd9, 0x98, 0x09, 0xaf, 0x35,
-	0x70, 0xb6, 0xd7, 0x7c, 0xbd, 0xa1, 0xdf, 0x41, 0xaf, 0x94, 0xbf, 0x81, 0xff, 0x04, 0x56, 0x50,
-	0x8b, 0x3c, 0x67, 0xd0, 0xac, 0x2b, 0x40, 0xa1, 0xa7, 0x7f, 0x39, 0xd0, 0x52, 0xa2, 0x69, 0x9d,
-	0x9d, 0x59, 0x9d, 0xc9, 0x43, 0xe8, 0x32, 0x3e, 0x9c, 0x15, 0xa3, 0xa1, 0xf2, 0xeb, 0x30, 0x3e,
-	0xad, 0x3b, 0xf9, 0x1c, 0xda, 0xe1, 0xcf, 0x79, 0x72, 0xc1, 0xbd, 0xa6, 0x0a, 0xd5, 0x9b, 0x85,
-	0x92, 0x60, 0x0f, 0xa4, 0xce, 0x37, 0x26, 0xe4, 0x29, 0x40, 0x20, 0x44, 0xc6, 0xce, 0x72, 0x81,
-	0x5c, 0xa1, 0xed, 0xec, 0x7b, 0xd6, 0x81, 0x9c, 0xe3, 0xb3, 0xa9, 0xde, 0xb7, 0x6c, 0xe9, 0xef,
-	0x0e, 0xac, 0x1f, 0x5d, 0x62, 0x22, 0x5e, 0xa7, 0x82, 0x8d, 0x58, 0x18, 0x08, 0x96, 0x26, 0xb5,
-	0x39, 0xef, 0x80, 0x9b, 0xc6, 0xd1, 0x50, 0xdf, 0x7f, 0xa3, 0xfe, 0xfe, 0x57, 0xd3, 0x38, 0xd2,
-	0xa8, 0x77, 0xc0, 0x4d, 0xf0, 0xca, 0x58, 0x37, 0x17, 0x58, 0x27, 0x78, 0xa5, 0x56, 0x74, 0x04,
-	0xee, 0x14, 0x14, 0xb9, 0x07, 0x2b, 0xd2, 0x70, 0xc8, 0x22, 0x13, 0xbf, 0x2d, 0xb7, 0x2f, 0x23,
-	0xd9, 0x21, 0xe9, 0x68, 0xc4, 0x51, 0xa8, 0xf0, 0x4d, 0xdf, 0xec, 0x64, 0xb6, 0x9c, 0xfd, 0xa6,
-	0x9b, 0x62, 0xd9, 0x57, 0x6b, 0x79, 0xaf, 0x63, 0xc1, 0xc6, 0xa8, 0x8a, 0xd1, 0xf4, 0xf5, 0x86,
-	0xfe, 0xd1, 0x80, 0x5b, 0xe5, 0x62, 0x90, 0xfb, 0xe0, 0xaa, 0x68, 0xca, 0x83, 0xa3, 0x3c, 0xa8,
-	0x01, 0x3b, 0x29, 0x79, 0x69, 0x58, 0x5e, 0xa6, 0x47, 0xc6, 0x69, 0xa4, 0x83, 0xae, 0xe9, 0x23,
-	0xaf, 0xd2, 0x08, 0xc9, 0x6d, 0x68, 0xe6, 0x2c, 0x52, 0x61, 0xd7, 0x7c, 0xb9, 0x94, 0x92, 0x73,
-	0x16, 0x99, 0x06, 0x93, 0x4b, 0x09, 0x24, 0xcc, 0x94, 0xdf, 0xb6, 0x06, 0xa2, 0x77, 0x12, 0xc8,
-	0x58, 0x4a, 0x57, 0x74, 0xd9, 0xe5, 0x9a, 0x0c, 0xa0, 0x93, 0xe1, 0x24, 0x36, 0x37, 0xe3, 0xad,
-	0x2a, 0x95, 0x2d, 0x22, 0x0f, 0x00, 0xc2, 0x34, 0x8e, 0x31, 0x54, 0x06, 0xae, 0x32, 0xb0, 0x24,
-	0xb2, 0x9e, 0x42, 0xc4, 0x43, 0x8e, 0xa1, 0x07, 0x03, 0x67, 0xbb, 0xe5, 0xb7, 0x85, 0x88, 0x4f,
-	0x30, 0xa4, 0xe7, 0xf0, 0xf1, 0x0b, 0x54, 0x4d, 0x7e, 0x6d, 0x75, 0x87, 0x19, 0xd6, 0xba, 0x16,
-	0xd8, 0x02, 0x98, 0x04, 0x19, 0x26, 0x42, 0xb6, 0xae, 0x19, 0x53, 0x57, 0x4b, 0x0e, 0x59, 0x66,
-	0x5f, 0x5c, 0xd3, 0xbe, 0x38, 0xd9, 0x64, 0xfd, 0xba, 0x48, 0x66, 0xac, 0xca, 0xdd, 0xeb, 0xbc,
-	0x7b, 0xf7, 0x5a, 0x43, 0xd2, 0x78, 0xeb, 0x90, 0xd0, 0x3d, 0xb8, 0xf3, 0x02, 0x85, 0x92, 0xa7,
-	0x89, 0xc0, 0x44, 0x14, 0x50, 0x17, 0x35, 0x1c, 0xdd, 0x87, 0xbb, 0xf3, 0x27, 0x4c, 0xca, 0x1e,
-	0xac, 0x84, 0x5a, 0xa4, 0x8e, 0x74, 0xfd, 0x62, 0x4b, 0x7f, 0x04, 0x72, 0x90, 0x61, 0x20, 0xf0,
-	0x3d, 0xa8, 0x78, 0x4a, 0xab, 0x8d, 0x1b, 0x69, 0xf5, 0x0e, 0xf4, 0x4a, 0xae, 0x75, 0x2e, 0x32,
-	0xe2, 0xe9, 0x24, 0xfa, 0x50, 0x11, 0x4b, 0xae, 0x4d, 0xc4, 0x7f, 0x1c, 0x20, 0x87, 0x18, 0xe3,
-	0x7b, 0x85, 0xac, 0xf9, 0xde, 0x54, 0x78, 0xb0, 0x59, 0xe5, 0xc1, 0x47, 0x70, 0x4b, 0x9a, 0xa8,
-	0x68, 0xc3, 0x28, 0x10, 0x81, 0x21, 0xf3, 0x2e, 0xe3, 0x3a, 0x85, 0xc3, 0x40, 0x04, 0xc6, 0x51,
-	0x86, 0x61, 0x9e, 0x49, 0x7e, 0x57, 0xc3, 0xa6, 0x1c, 0xf9, 0x85, 0x48, 0x62, 0x29, 0xe5, 0x6c,
-	0xb0, 0xfc, 0xed, 0x40, 0xef, 0x19, 0xe7, 0xec, 0x3c, 0xf9, 0x21, 0x8d, 0xf3, 0x31, 0x16, 0x60,
-	0x36, 0xa0, 0x15, 0xa6, 0xb9, 0xb9, 0xdf, 0x96, 0xaf, 0x37, 0x73, 0xb3, 0xd6, 0xa8, 0xcc, 0xda,
-	0xdc, 0xb4, 0x36, 0xab, 0xd3, 0x6a, 0x4d, 0xe3, 0xb2, 0x3d, 0x8d, 0xe4, 0x13, 0xe8, 0x48, 0x78,
-	0xc3, 0x10, 0x13, 0x81, 0x99, 0x42, 0xe0, 0xfa, 0x20, 0x45, 0x07, 0x4a, 0x42, 0x2f, 0x61, 0xa3,
-	0x9c, 0xa8, 0xe9, 0xc5, 0x85, 0x7c, 0x29, 0xa9, 0x28, 0x8b, 0x4d, 0x96, 0x72, 0xa9, 0x06, 0x38,
-	0x3f, 0x8b, 0x59, 0x38, 0x94, 0x8a, 0xa6, 0x19, 0x60, 0x25, 0x39, 0xcd, 0xe2, 0x19, 0xe6, 0x65,
-	0x0b, 0x33, 0xfd, 0x12, 0x7a, 0xfa, 0x51, 0x50, 0x2e, 0xd0, 0x16, 0xc0, 0xa5, 0x12, 0x0c, 0x59,
-	0xa4, 0xbf, 0x87, 0xae, 0xef, 0x6a, 0xc9, 0xcb, 0x88, 0xd3, 0x6f, 0xc1, 0x3d, 0x4e, 0x35, 0x66,
-	0x4e, 0xf6, 0xc0, 0x8d, 0x8b, 0x8d, 0xf9, 0x74, 0x92, 0x59, 0xcb, 0x15, 0x76, 0xfe, 0xcc, 0x88,
-	0x7e, 0x03, 0xab, 0x85, 0xb8, 0xc0, 0xe1, 0x2c, 0xc2, 0xd1, 0x98, 0xc3, 0x41, 0xff, 0x73, 0x60,
-	0xa3, 0x9c, 0xb2, 0x29, 0xd5, 0x29, 0xac, 0x4d, 0x43, 0x0c, 0xc7, 0xc1, 0xc4, 0xe4, 0xb2, 0x67,
-	0xe7, 0x52, 0x3d, 0x36, 0x4d, 0x90, 0xbf, 0x0a, 0x26, 0xba, 0x7b, 0xba, 0xb1, 0x25, 0xea, 0x7f,
-	0x0f, 0xeb, 0x15, 0x13, 0x99, 0xf5, 0x05, 0x16, 0x73, 0x20, 0x97, 0xe4, 0x09, 0xb4, 0x2e, 0x83,
-	0x38, 0x47, 0x33, 0x74, 0xbd, 0x6a, 0x05, 0xb8, 0xaf, 0x2d, 0xbe, 0x6e, 0x3c, 0x75, 0xe8, 0x57,
-	0x70, 0x4f, 0x37, 0xec, 0xc1, 0xb4, 0xbf, 0x8a, 0xda, 0x97, 0xdb, 0xd0, 0x99, 0x6f, 0x43, 0xda,
-	0x07, 0xaf, 0x7a, 0x54, 0x83, 0xd9, 0xff, 0xb3, 0x0d, 0xdd, 0x13, 0x0c, 0xae, 0x10, 0x23, 0xc9,
-	0x6c, 0x19, 0x39, 0x2f, 0x8a, 0x55, 0x7e, 0xf4, 0x91, 0xc7, 0xf3, 0x55, 0xa9, 0x7d, 0x65, 0xf6,
-	0x3f, 0x7b, 0x9b, 0x99, 0x19, 0xb4, 0x25, 0x72, 0x0c, 0x1d, 0xeb, 0x55, 0x45, 0x36, 0xad, 0x83,
-	0x95, 0xc7, 0x62, 0x7f, 0x6b, 0x81, 0x76, 0xea, 0x2d, 0x00, 0x52, 0xfd, 0xa6, 0x90, 0x4f, 0x67,
-	0xc7, 0x16, 0x7e, 0xdb, 0xfa, 0x8f, 0x6e, 0x36, 0xb2, 0x13, 0xb6, 0x08, 0xd7, 0x4e, 0xb8, 0x4a,
-	0xf1, 0x76, 0xc2, 0x75, 0x2c, 0xad, 0xbc, 0x59, 0x64, 0x6a, 0x7b, 0xab, 0xd2, 0xb7, 0xed, 0xad,
-	0x8e, 0x81, 0x95, 0x37, 0x8b, 0xce, 0x6c, 0x6f, 0x55, 0x66, 0xb6, 0xbd, 0xd5, 0x71, 0xe0, 0x12,
-	0x79, 0x03, 0x5d, 0x9b, 0x5b, 0x88, 0x75, 0xa0, 0x86, 0x1c, 0xfb, 0x0f, 0x16, 0xa9, 0x6d, 0x87,
-	0xf6, 0x28, 0xd9, 0x0e, 0x6b, 0xc8, 0xc4, 0x76, 0x58, 0x37, 0x81, 0x74, 0x89, 0xfc, 0x04, 0xb7,
-	0xe7, 0x5b, 0x9a, 0x3c, 0x9c, 0x87, 0x55, 0x99, 0x94, 0x3e, 0xbd, 0xc9, 0xa4, 0x70, 0x7e, 0xd6,
-	0x56, 0x7f, 0x56, 0x5f, 0xfc, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x41, 0x0a, 0x10, 0x9d, 0x68, 0x0d,
-	0x00, 0x00,
+	// 1267 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xcd, 0x6f, 0xdc, 0x44,
+	0x14, 0x8f, 0xf7, 0x2b, 0xeb, 0xb7, 0xbb, 0x25, 0x99, 0x0d, 0xd4, 0xda, 0x7c, 0xb0, 0x35, 0x14,
+	0xa5, 0x22, 0x5a, 0x55, 0x81, 0x43, 0x4b, 0x85, 0x44, 0x9b, 0x0f, 0x29, 0x52, 0xda, 0x22, 0xa7,
+	0x41, 0x42, 0x1c, 0x2c, 0xc7, 0x9e, 0x5d, 0x46, 0xf1, 0xda, 0xc6, 0x33, 0x4e, 0x5a, 0xfe, 0x09,
+	0x2e, 0x5c, 0x39, 0x70, 0xe2, 0xbf, 0xe0, 0xc2, 0x81, 0xbf, 0x89, 0x1b, 0x9a, 0x0f, 0x7b, 0xc7,
+	0x6b, 0x27, 0x05, 0x55, 0xbd, 0xcd, 0xfc, 0xde, 0x9b, 0xf7, 0x35, 0xbf, 0x79, 0xcf, 0x86, 0xde,
+	0x94, 0x84, 0x38, 0x9d, 0x24, 0x69, 0xcc, 0x62, 0xd4, 0x15, 0x1b, 0x37, 0xb9, 0xb0, 0x5f, 0xc2,
+	0xe6, 0x69, 0x1c, 0x5f, 0x66, 0xc9, 0x21, 0x49, 0xb1, 0xcf, 0xe2, 0xf4, 0xcd, 0x51, 0xc4, 0xd2,
+	0x37, 0x0e, 0xfe, 0x29, 0xc3, 0x94, 0xa1, 0x2d, 0x30, 0x83, 0x5c, 0x60, 0x19, 0x63, 0x63, 0xd7,
+	0x74, 0x16, 0x00, 0x42, 0xd0, 0x8a, 0xbc, 0x39, 0xb6, 0x1a, 0x42, 0x20, 0xd6, 0xf6, 0x11, 0x6c,
+	0xd5, 0x1b, 0xa4, 0x49, 0x1c, 0x51, 0x8c, 0xee, 0x43, 0x1b, 0x73, 0x40, 0x58, 0xeb, 0xed, 0x7f,
+	0x30, 0xc9, 0x43, 0x99, 0x48, 0x3d, 0x29, 0xb5, 0xff, 0x34, 0x00, 0x9d, 0x12, 0xca, 0x38, 0x48,
+	0x30, 0xfd, 0x6f, 0xf1, 0x7c, 0x04, 0x9d, 0x24, 0xc5, 0x53, 0xf2, 0x5a, 0x45, 0xa4, 0x76, 0x68,
+	0x0f, 0xd6, 0x29, 0xf3, 0x52, 0x76, 0x9c, 0xc6, 0xf3, 0x63, 0x12, 0xe2, 0x17, 0x3c, 0xe8, 0xa6,
+	0x50, 0xa9, 0x0a, 0xd0, 0x04, 0x10, 0x89, 0xfc, 0x30, 0xa3, 0xe4, 0x0a, 0x9f, 0xe5, 0x52, 0xab,
+	0x35, 0x36, 0x76, 0xbb, 0x4e, 0x8d, 0x04, 0x6d, 0x40, 0x3b, 0x24, 0x73, 0xc2, 0xac, 0xf6, 0xd8,
+	0xd8, 0x1d, 0x38, 0x72, 0x63, 0x7f, 0x03, 0xc3, 0x52, 0xfc, 0x2a, 0xfd, 0x07, 0xb0, 0x8a, 0x25,
+	0x64, 0x19, 0xe3, 0x66, 0x5d, 0x01, 0x72, 0xb9, 0xfd, 0x5b, 0x03, 0xda, 0x02, 0x2a, 0xea, 0x6c,
+	0x2c, 0xea, 0x8c, 0xee, 0x41, 0x9f, 0x50, 0x77, 0x51, 0x8c, 0x86, 0x88, 0xaf, 0x47, 0x68, 0x51,
+	0x77, 0xf4, 0x39, 0x74, 0xfc, 0x1f, 0xb3, 0xe8, 0x92, 0x5a, 0x4d, 0xe1, 0x6a, 0xb8, 0x70, 0xc5,
+	0x93, 0x3d, 0xe0, 0x32, 0x47, 0xa9, 0xa0, 0x47, 0x00, 0x1e, 0x63, 0x29, 0xb9, 0xc8, 0x18, 0xa6,
+	0x22, 0xdb, 0xde, 0xbe, 0xa5, 0x1d, 0xc8, 0x28, 0x7e, 0x5a, 0xc8, 0x1d, 0x4d, 0x17, 0x3d, 0x86,
+	0x2e, 0x7e, 0xcd, 0x70, 0x14, 0xe0, 0xc0, 0x6a, 0x0b, 0x47, 0xdb, 0x4b, 0x39, 0x4d, 0x8e, 0x94,
+	0x5c, 0x66, 0x58, 0xa8, 0x8f, 0x9e, 0xc0, 0xa0, 0x24, 0x42, 0x6b, 0xd0, 0xbc, 0xc4, 0xf9, 0xcd,
+	0xf2, 0x25, 0xaf, 0xee, 0x95, 0x17, 0x66, 0x92, 0x64, 0x7d, 0x47, 0x6e, 0xbe, 0x6a, 0x3c, 0x32,
+	0xec, 0x5f, 0x0d, 0x58, 0x3f, 0xba, 0xc2, 0x11, 0x7b, 0x11, 0x33, 0x32, 0x25, 0xbe, 0xc7, 0x48,
+	0x1c, 0xa1, 0x3d, 0x30, 0xe3, 0x30, 0x70, 0x6f, 0xe5, 0x58, 0x37, 0x0e, 0x95, 0xbf, 0x3d, 0x30,
+	0x23, 0x7c, 0xad, 0xb4, 0x1b, 0x37, 0x68, 0x47, 0xf8, 0x5a, 0x6a, 0x7f, 0x02, 0x83, 0x00, 0x87,
+	0x98, 0x61, 0xb7, 0xa8, 0x2b, 0x2f, 0x7a, 0x5f, 0x82, 0xa2, 0x9e, 0xd4, 0xfe, 0xdd, 0x00, 0xb3,
+	0x28, 0x2f, 0xba, 0x0b, 0xab, 0xdc, 0x9c, 0x4b, 0x02, 0x95, 0x54, 0x87, 0x6f, 0x4f, 0x02, 0xce,
+	0xd5, 0x78, 0x3a, 0xa5, 0x98, 0x09, 0xb7, 0x4d, 0x47, 0xed, 0xf8, 0x5d, 0x53, 0xf2, 0xb3, 0xa4,
+	0x67, 0xcb, 0x11, 0x6b, 0x5e, 0x83, 0x39, 0x23, 0x73, 0x2c, 0xae, 0xa5, 0xe9, 0xc8, 0x0d, 0x1a,
+	0x42, 0x1b, 0xbb, 0xcc, 0x9b, 0x09, 0xde, 0x99, 0x4e, 0x0b, 0xbf, 0xf2, 0x66, 0xe8, 0x53, 0xb8,
+	0x43, 0xe3, 0x2c, 0xf5, 0xb1, 0x9b, 0xbb, 0xed, 0x08, 0x69, 0x5f, 0xa2, 0xc7, 0xc2, 0xb9, 0xfd,
+	0x77, 0x03, 0xee, 0x94, 0x6f, 0x14, 0x6d, 0x82, 0x29, 0x4e, 0x08, 0xe7, 0x86, 0x70, 0x2e, 0xba,
+	0xc4, 0x59, 0x29, 0x80, 0x86, 0x1e, 0x40, 0x7e, 0x64, 0x1e, 0x07, 0x32, 0xde, 0x81, 0x3c, 0xf2,
+	0x3c, 0x0e, 0x30, 0xbf, 0xc9, 0x8c, 0x04, 0x22, 0xe2, 0x81, 0xc3, 0x97, 0x1c, 0x99, 0x91, 0x40,
+	0xbd, 0x12, 0xbe, 0xe4, 0x35, 0xf0, 0x53, 0x61, 0xb7, 0x23, 0x6b, 0x20, 0x77, 0xbc, 0x06, 0x73,
+	0x8e, 0xae, 0xca, 0xc4, 0xf8, 0x1a, 0x8d, 0xa1, 0x97, 0xe2, 0x24, 0x54, 0xd7, 0x6c, 0x75, 0x85,
+	0x48, 0x87, 0xd0, 0x0e, 0x80, 0x1f, 0x87, 0x21, 0xf6, 0x85, 0x82, 0x29, 0x14, 0x34, 0x84, 0x5f,
+	0x05, 0x63, 0xa1, 0x4b, 0xb1, 0x6f, 0xc1, 0xd8, 0xd8, 0x6d, 0x3b, 0x1d, 0xc6, 0xc2, 0x33, 0xec,
+	0xf3, 0x3c, 0x32, 0x8a, 0x53, 0x57, 0xbc, 0xb1, 0x9e, 0x38, 0xd7, 0xe5, 0x80, 0xe8, 0x06, 0xdb,
+	0x00, 0xb3, 0x34, 0xce, 0x12, 0x29, 0xed, 0x8f, 0x9b, 0xbc, 0xe5, 0x08, 0x84, 0x8b, 0xed, 0xef,
+	0x01, 0x1d, 0xa4, 0xd8, 0x63, 0xf8, 0x7f, 0xb4, 0xcd, 0xa2, 0x05, 0x36, 0x6e, 0x6d, 0x81, 0x1f,
+	0xc2, 0xb0, 0x64, 0x5a, 0x76, 0x10, 0xee, 0xf1, 0x3c, 0x09, 0xde, 0x97, 0xc7, 0x92, 0x69, 0xe5,
+	0xf1, 0x17, 0x03, 0xd0, 0xa1, 0xa0, 0xf8, 0xbb, 0xcd, 0x06, 0x4e, 0x4e, 0xde, 0xb3, 0xe4, 0x13,
+	0x0a, 0x3c, 0xe6, 0xa9, 0xae, 0xda, 0x27, 0x54, 0xda, 0x3f, 0xf4, 0x98, 0xa7, 0x3a, 0x5b, 0x8a,
+	0xfd, 0x2c, 0xe5, 0x8d, 0x56, 0x10, 0x46, 0x74, 0x36, 0x27, 0x87, 0x78, 0xa0, 0xa5, 0x80, 0x54,
+	0xa0, 0x7f, 0x18, 0x30, 0x7c, 0x4a, 0x29, 0x99, 0x45, 0xdf, 0xc5, 0x61, 0x36, 0xc7, 0x79, 0xa4,
+	0x1b, 0xd0, 0xf6, 0xe3, 0x2c, 0x62, 0x22, 0xca, 0xb6, 0x23, 0x37, 0x4b, 0x7c, 0x69, 0x54, 0xf8,
+	0xb2, 0xc4, 0xb8, 0x66, 0x95, 0x71, 0x1a, 0xa3, 0x5a, 0x25, 0x46, 0x7d, 0x0c, 0x3d, 0x9e, 0x9e,
+	0xeb, 0xe3, 0x88, 0xe1, 0x54, 0x3d, 0x50, 0xe0, 0xd0, 0x81, 0x40, 0xec, 0x2b, 0xd8, 0x28, 0x07,
+	0xaa, 0xc6, 0xc3, 0x8d, 0xed, 0x82, 0x3f, 0xa7, 0x34, 0x54, 0x51, 0xf2, 0x25, 0x27, 0x66, 0x92,
+	0x5d, 0x84, 0xc4, 0x77, 0xb9, 0x40, 0x46, 0x67, 0x4a, 0xe4, 0x3c, 0x0d, 0x17, 0x39, 0xb7, 0xb4,
+	0x9c, 0xed, 0x2f, 0x61, 0x28, 0xa7, 0x73, 0xb9, 0x40, 0xdb, 0x00, 0x57, 0x02, 0x70, 0x49, 0x20,
+	0x07, 0x93, 0xe9, 0x98, 0x12, 0x39, 0x09, 0xa8, 0xfd, 0x35, 0x98, 0xa7, 0xb1, 0xcc, 0x99, 0xa2,
+	0x87, 0x60, 0x86, 0xf9, 0x46, 0xcd, 0x30, 0xb4, 0xe0, 0x53, 0xae, 0xe7, 0x2c, 0x94, 0xec, 0x27,
+	0xd0, 0xcd, 0xe1, 0x3c, 0x0f, 0xe3, 0xa6, 0x3c, 0x1a, 0x4b, 0x79, 0xd8, 0x7f, 0x19, 0xb0, 0x51,
+	0x0e, 0x59, 0x95, 0xea, 0x1c, 0x06, 0x85, 0x0b, 0x77, 0xee, 0x25, 0x2a, 0x96, 0x87, 0x7a, 0x2c,
+	0xd5, 0x63, 0x45, 0x80, 0xf4, 0xb9, 0x97, 0x48, 0xf6, 0xf4, 0x43, 0x0d, 0x1a, 0xbd, 0x82, 0xf5,
+	0x8a, 0x4a, 0xcd, 0x58, 0x7a, 0xa0, 0x8f, 0xa5, 0xd2, 0x68, 0x2d, 0x4e, 0xeb, 0xb3, 0xea, 0x31,
+	0xdc, 0x95, 0x84, 0x3d, 0x28, 0xf8, 0x95, 0xd7, 0xbe, 0x4c, 0x43, 0x63, 0x99, 0x86, 0xf6, 0x08,
+	0xac, 0xea, 0x51, 0x45, 0xf8, 0x19, 0xac, 0x9f, 0x31, 0x8f, 0x11, 0xca, 0x88, 0x5f, 0x7c, 0x23,
+	0x2d, 0xf1, 0xd6, 0x78, 0x5b, 0xa7, 0xac, 0x32, 0x7f, 0x0d, 0x9a, 0x8c, 0xe5, 0x9c, 0xe2, 0x4b,
+	0x7e, 0x0b, 0x48, 0xf7, 0xa4, 0xee, 0xe0, 0x3d, 0xb8, 0xe2, 0x7c, 0x60, 0x31, 0xf3, 0x42, 0x39,
+	0x89, 0x5a, 0x62, 0x12, 0x99, 0x02, 0x11, 0xa3, 0x48, 0x36, 0xeb, 0x40, 0x4a, 0xdb, 0x72, 0x4e,
+	0x71, 0x40, 0x08, 0xb7, 0x01, 0xc4, 0xf3, 0x91, 0xcc, 0xef, 0xc8, 0xb3, 0x1c, 0x39, 0xe0, 0xc0,
+	0xfe, 0x3f, 0x6d, 0xe8, 0x9f, 0x61, 0xef, 0x1a, 0xe3, 0x80, 0x0f, 0xc2, 0x14, 0xcd, 0x72, 0x6e,
+	0x95, 0x3f, 0x56, 0xd1, 0xfd, 0x65, 0x12, 0xd5, 0x7e, 0x1d, 0x8f, 0x3e, 0x7b, 0x9b, 0x9a, 0xba,
+	0xa6, 0x15, 0x74, 0x0a, 0x3d, 0xed, 0x6b, 0x10, 0x6d, 0x69, 0x07, 0x2b, 0x1f, 0xb9, 0xa3, 0xed,
+	0x1b, 0xa4, 0xba, 0x35, 0x6d, 0x32, 0xe8, 0xd6, 0xaa, 0xb3, 0x48, 0xb7, 0x56, 0x37, 0x4e, 0x84,
+	0x35, 0xad, 0xeb, 0xeb, 0xd6, 0xaa, 0x73, 0x46, 0xb7, 0x56, 0x37, 0x2a, 0x84, 0x35, 0xad, 0x35,
+	0xeb, 0xd6, 0xaa, 0x23, 0x44, 0xb7, 0x56, 0xd7, 0xcf, 0x57, 0xd0, 0x4b, 0xe8, 0xeb, 0x7d, 0x12,
+	0x69, 0x07, 0x6a, 0x1a, 0xfd, 0x68, 0xe7, 0x26, 0xb1, 0x6e, 0x50, 0x6f, 0x0b, 0xba, 0xc1, 0x9a,
+	0xc6, 0xa8, 0x1b, 0xac, 0xeb, 0x26, 0xf6, 0x0a, 0xfa, 0x01, 0xd6, 0x96, 0x9f, 0x27, 0xba, 0xb7,
+	0x9c, 0x56, 0xe5, 0xd5, 0x8f, 0xec, 0xdb, 0x54, 0x0a, 0xe3, 0x27, 0x00, 0x8b, 0x57, 0x87, 0x36,
+	0x17, 0x67, 0x2a, 0xaf, 0x7e, 0xb4, 0x55, 0x2f, 0xcc, 0x4d, 0x3d, 0xdb, 0x81, 0x35, 0x2a, 0xa9,
+	0x3f, 0xa5, 0x13, 0x3f, 0x24, 0x38, 0x62, 0xcf, 0x40, 0xbc, 0x82, 0x6f, 0xf9, 0x2f, 0xe1, 0x45,
+	0x47, 0xfc, 0x19, 0x7e, 0xf1, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x22, 0x62, 0x04, 0x66, 0x28,
+	0x0e, 0x00, 0x00,
 }

weed/pb/master.proto

@@ -9,6 +9,12 @@ service Seaweed {
     }
     rpc KeepConnected (stream ClientListenRequest) returns (stream VolumeLocation) {
     }
+    rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) {
+    }
+    rpc Assign (AssignRequest) returns (AssignResponse) {
+    }
+    rpc Statistics (StatisticsRequest) returns (StatisticsResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -23,6 +29,9 @@ message Heartbeat {
     string rack = 7;
     uint32 admin_port = 8;
     repeated VolumeInformationMessage volumes = 9;
+    // delta volume ids
+    repeated uint32 new_vids = 10;
+    repeated uint32 deleted_vids = 11;
 }
 
 message HeartbeatResponse {
@@ -66,3 +75,52 @@ message VolumeLocation {
     repeated uint32 new_vids = 3;
     repeated uint32 deleted_vids = 4;
 }
+
+message LookupVolumeRequest {
+    repeated string volume_ids = 1;
+    string collection = 2; // optional, a bit faster if provided.
+}
+message LookupVolumeResponse {
+    message VolumeIdLocation {
+        string volume_id = 1;
+        repeated Location locations = 2;
+        string error = 3;
+    }
+    repeated VolumeIdLocation volume_id_locations = 1;
+}
+
+message Location {
+    string url = 1;
+    string public_url = 2;
+}
+
+message AssignRequest {
+    uint64 count = 1;
+    string replication = 2;
+    string collection = 3;
+    string ttl = 4;
+    string data_center = 5;
+    string rack = 6;
+    string data_node = 7;
+}
+message AssignResponse {
+    string fid = 1;
+    string url = 2;
+    string public_url = 3;
+    uint64 count = 4;
+    string error = 5;
+}
+
+message StatisticsRequest {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+}
+message StatisticsResponse {
+    string replication = 1;
+    string collection = 2;
+    string ttl = 3;
+    uint64 total_size = 4;
+    uint64 used_size = 5;
+    uint64 file_count = 6;
+}

weed/pb/master_pb/seaweed.pb.go

@@ -1,585 +0,0 @@
-// Code generated by protoc-gen-go.
-// source: seaweed.proto
-// DO NOT EDIT!
-
-/*
-Package master_pb is a generated protocol buffer package.
-
-It is generated from these files:
-	seaweed.proto
-
-It has these top-level messages:
-	Heartbeat
-	HeartbeatResponse
-	VolumeInformationMessage
-	Empty
-	SuperBlockExtra
-	ClientListenRequest
-	VolumeLocation
-*/
-package master_pb
-
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
-
-import (
-	context "golang.org/x/net/context"
-	grpc "google.golang.org/grpc"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
-
-type Heartbeat struct {
-	Ip             string                      `protobuf:"bytes,1,opt,name=ip" json:"ip,omitempty"`
-	Port           uint32                      `protobuf:"varint,2,opt,name=port" json:"port,omitempty"`
-	PublicUrl      string                      `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"`
-	MaxVolumeCount uint32                      `protobuf:"varint,4,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"`
-	MaxFileKey     uint64                      `protobuf:"varint,5,opt,name=max_file_key,json=maxFileKey" json:"max_file_key,omitempty"`
-	DataCenter     string                      `protobuf:"bytes,6,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"`
-	Rack           string                      `protobuf:"bytes,7,opt,name=rack" json:"rack,omitempty"`
-	AdminPort      uint32                      `protobuf:"varint,8,opt,name=admin_port,json=adminPort" json:"admin_port,omitempty"`
-	Volumes        []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"`
-}
-
-func (m *Heartbeat) Reset()                    { *m = Heartbeat{} }
-func (m *Heartbeat) String() string            { return proto.CompactTextString(m) }
-func (*Heartbeat) ProtoMessage()               {}
-func (*Heartbeat) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
-
-func (m *Heartbeat) GetIp() string {
-	if m != nil {
-		return m.Ip
-	}
-	return ""
-}
-
-func (m *Heartbeat) GetPort() uint32 {
-	if m != nil {
-		return m.Port
-	}
-	return 0
-}
-
-func (m *Heartbeat) GetPublicUrl() string {
-	if m != nil {
-		return m.PublicUrl
-	}
-	return ""
-}
-
-func (m *Heartbeat) GetMaxVolumeCount() uint32 {
-	if m != nil {
-		return m.MaxVolumeCount
-	}
-	return 0
-}
-
-func (m *Heartbeat) GetMaxFileKey() uint64 {
-	if m != nil {
-		return m.MaxFileKey
-	}
-	return 0
-}
-
-func (m *Heartbeat) GetDataCenter() string {
-	if m != nil {
-		return m.DataCenter
-	}
-	return ""
-}
-
-func (m *Heartbeat) GetRack() string {
-	if m != nil {
-		return m.Rack
-	}
-	return ""
-}
-
-func (m *Heartbeat) GetAdminPort() uint32 {
-	if m != nil {
-		return m.AdminPort
-	}
-	return 0
-}
-
-func (m *Heartbeat) GetVolumes() []*VolumeInformationMessage {
-	if m != nil {
-		return m.Volumes
-	}
-	return nil
-}
-
-type HeartbeatResponse struct {
-	VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"`
-	SecretKey       string `protobuf:"bytes,2,opt,name=secretKey" json:"secretKey,omitempty"`
-	Leader          string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"`
-}
-
-func (m *HeartbeatResponse) Reset()                    { *m = HeartbeatResponse{} }
-func (m *HeartbeatResponse) String() string            { return proto.CompactTextString(m) }
-func (*HeartbeatResponse) ProtoMessage()               {}
-func (*HeartbeatResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
-
-func (m *HeartbeatResponse) GetVolumeSizeLimit() uint64 {
-	if m != nil {
-		return m.VolumeSizeLimit
-	}
-	return 0
-}
-
-func (m *HeartbeatResponse) GetSecretKey() string {
-	if m != nil {
-		return m.SecretKey
-	}
-	return ""
-}
-
-func (m *HeartbeatResponse) GetLeader() string {
-	if m != nil {
-		return m.Leader
-	}
-	return ""
-}
-
-type VolumeInformationMessage struct {
-	Id               uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
-	Size             uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
-	Collection       string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
-	FileCount        uint64 `protobuf:"varint,4,opt,name=file_count,json=fileCount" json:"file_count,omitempty"`
-	DeleteCount      uint64 `protobuf:"varint,5,opt,name=delete_count,json=deleteCount" json:"delete_count,omitempty"`
-	DeletedByteCount uint64 `protobuf:"varint,6,opt,name=deleted_byte_count,json=deletedByteCount" json:"deleted_byte_count,omitempty"`
-	ReadOnly         bool   `protobuf:"varint,7,opt,name=read_only,json=readOnly" json:"read_only,omitempty"`
-	ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"`
-	Version          uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"`
-	Ttl              uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"`
-}
-
-func (m *VolumeInformationMessage) Reset()                    { *m = VolumeInformationMessage{} }
-func (m *VolumeInformationMessage) String() string            { return proto.CompactTextString(m) }
-func (*VolumeInformationMessage) ProtoMessage()               {}
-func (*VolumeInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
-
-func (m *VolumeInformationMessage) GetId() uint32 {
-	if m != nil {
-		return m.Id
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetSize() uint64 {
-	if m != nil {
-		return m.Size
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetCollection() string {
-	if m != nil {
-		return m.Collection
-	}
-	return ""
-}
-
-func (m *VolumeInformationMessage) GetFileCount() uint64 {
-	if m != nil {
-		return m.FileCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetDeleteCount() uint64 {
-	if m != nil {
-		return m.DeleteCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 {
-	if m != nil {
-		return m.DeletedByteCount
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetReadOnly() bool {
-	if m != nil {
-		return m.ReadOnly
-	}
-	return false
-}
-
-func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 {
-	if m != nil {
-		return m.ReplicaPlacement
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetVersion() uint32 {
-	if m != nil {
-		return m.Version
-	}
-	return 0
-}
-
-func (m *VolumeInformationMessage) GetTtl() uint32 {
-	if m != nil {
-		return m.Ttl
-	}
-	return 0
-}
-
-type Empty struct {
-}
-
-func (m *Empty) Reset()                    { *m = Empty{} }
-func (m *Empty) String() string            { return proto.CompactTextString(m) }
-func (*Empty) ProtoMessage()               {}
-func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
-
-type SuperBlockExtra struct {
-	ErasureCoding *SuperBlockExtra_ErasureCoding `protobuf:"bytes,1,opt,name=erasure_coding,json=erasureCoding" json:"erasure_coding,omitempty"`
-}
-
-func (m *SuperBlockExtra) Reset()                    { *m = SuperBlockExtra{} }
-func (m *SuperBlockExtra) String() string            { return proto.CompactTextString(m) }
-func (*SuperBlockExtra) ProtoMessage()               {}
-func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
-
-func (m *SuperBlockExtra) GetErasureCoding() *SuperBlockExtra_ErasureCoding {
-	if m != nil {
-		return m.ErasureCoding
-	}
-	return nil
-}
-
-type SuperBlockExtra_ErasureCoding struct {
-	Data      uint32   `protobuf:"varint,1,opt,name=data" json:"data,omitempty"`
-	Parity    uint32   `protobuf:"varint,2,opt,name=parity" json:"parity,omitempty"`
-	VolumeIds []uint32 `protobuf:"varint,3,rep,packed,name=volume_ids,json=volumeIds" json:"volume_ids,omitempty"`
-}
-
-func (m *SuperBlockExtra_ErasureCoding) Reset()         { *m = SuperBlockExtra_ErasureCoding{} }
-func (m *SuperBlockExtra_ErasureCoding) String() string { return proto.CompactTextString(m) }
-func (*SuperBlockExtra_ErasureCoding) ProtoMessage()    {}
-func (*SuperBlockExtra_ErasureCoding) Descriptor() ([]byte, []int) {
-	return fileDescriptor0, []int{4, 0}
-}
-
-func (m *SuperBlockExtra_ErasureCoding) GetData() uint32 {
-	if m != nil {
-		return m.Data
-	}
-	return 0
-}
-
-func (m *SuperBlockExtra_ErasureCoding) GetParity() uint32 {
-	if m != nil {
-		return m.Parity
-	}
-	return 0
-}
-
-func (m *SuperBlockExtra_ErasureCoding) GetVolumeIds() []uint32 {
-	if m != nil {
-		return m.VolumeIds
-	}
-	return nil
-}
-
-type ClientListenRequest struct {
-	Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
-}
-
-func (m *ClientListenRequest) Reset()                    { *m = ClientListenRequest{} }
-func (m *ClientListenRequest) String() string            { return proto.CompactTextString(m) }
-func (*ClientListenRequest) ProtoMessage()               {}
-func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
-
-func (m *ClientListenRequest) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-type VolumeLocation struct {
-	Url         string   `protobuf:"bytes,1,opt,name=url" json:"url,omitempty"`
-	PublicUrl   string   `protobuf:"bytes,2,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"`
-	NewVids     []uint32 `protobuf:"varint,3,rep,packed,name=new_vids,json=newVids" json:"new_vids,omitempty"`
-	DeletedVids []uint32 `protobuf:"varint,4,rep,packed,name=deleted_vids,json=deletedVids" json:"deleted_vids,omitempty"`
-}
-
-func (m *VolumeLocation) Reset()                    { *m = VolumeLocation{} }
-func (m *VolumeLocation) String() string            { return proto.CompactTextString(m) }
-func (*VolumeLocation) ProtoMessage()               {}
-func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
-
-func (m *VolumeLocation) GetUrl() string {
-	if m != nil {
-		return m.Url
-	}
-	return ""
-}
-
-func (m *VolumeLocation) GetPublicUrl() string {
-	if m != nil {
-		return m.PublicUrl
-	}
-	return ""
-}
-
-func (m *VolumeLocation) GetNewVids() []uint32 {
-	if m != nil {
-		return m.NewVids
-	}
-	return nil
-}
-
-func (m *VolumeLocation) GetDeletedVids() []uint32 {
-	if m != nil {
-		return m.DeletedVids
-	}
-	return nil
-}
-
-func init() {
-	proto.RegisterType((*Heartbeat)(nil), "master_pb.Heartbeat")
-	proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse")
-	proto.RegisterType((*VolumeInformationMessage)(nil), "master_pb.VolumeInformationMessage")
-	proto.RegisterType((*Empty)(nil), "master_pb.Empty")
-	proto.RegisterType((*SuperBlockExtra)(nil), "master_pb.SuperBlockExtra")
-	proto.RegisterType((*SuperBlockExtra_ErasureCoding)(nil), "master_pb.SuperBlockExtra.ErasureCoding")
-	proto.RegisterType((*ClientListenRequest)(nil), "master_pb.ClientListenRequest")
-	proto.RegisterType((*VolumeLocation)(nil), "master_pb.VolumeLocation")
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// Client API for Seaweed service
-
-type SeaweedClient interface {
-	SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error)
-	KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error)
-}
-
-type seaweedClient struct {
-	cc *grpc.ClientConn
-}
-
-func NewSeaweedClient(cc *grpc.ClientConn) SeaweedClient {
-	return &seaweedClient{cc}
-}
-
-func (c *seaweedClient) SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) {
-	stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[0], c.cc, "/master_pb.Seaweed/SendHeartbeat", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &seaweedSendHeartbeatClient{stream}
-	return x, nil
-}
-
-type Seaweed_SendHeartbeatClient interface {
-	Send(*Heartbeat) error
-	Recv() (*HeartbeatResponse, error)
-	grpc.ClientStream
-}
-
-type seaweedSendHeartbeatClient struct {
-	grpc.ClientStream
-}
-
-func (x *seaweedSendHeartbeatClient) Send(m *Heartbeat) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *seaweedSendHeartbeatClient) Recv() (*HeartbeatResponse, error) {
-	m := new(HeartbeatResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *seaweedClient) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) {
-	stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[1], c.cc, "/master_pb.Seaweed/KeepConnected", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &seaweedKeepConnectedClient{stream}
-	return x, nil
-}
-
-type Seaweed_KeepConnectedClient interface {
-	Send(*ClientListenRequest) error
-	Recv() (*VolumeLocation, error)
-	grpc.ClientStream
-}
-
-type seaweedKeepConnectedClient struct {
-	grpc.ClientStream
-}
-
-func (x *seaweedKeepConnectedClient) Send(m *ClientListenRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *seaweedKeepConnectedClient) Recv() (*VolumeLocation, error) {
-	m := new(VolumeLocation)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-// Server API for Seaweed service
-
-type SeaweedServer interface {
-	SendHeartbeat(Seaweed_SendHeartbeatServer) error
-	KeepConnected(Seaweed_KeepConnectedServer) error
-}
-
-func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) {
-	s.RegisterService(&_Seaweed_serviceDesc, srv)
-}
-
-func _Seaweed_SendHeartbeat_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(SeaweedServer).SendHeartbeat(&seaweedSendHeartbeatServer{stream})
-}
-
-type Seaweed_SendHeartbeatServer interface {
-	Send(*HeartbeatResponse) error
-	Recv() (*Heartbeat, error)
-	grpc.ServerStream
-}
-
-type seaweedSendHeartbeatServer struct {
-	grpc.ServerStream
-}
-
-func (x *seaweedSendHeartbeatServer) Send(m *HeartbeatResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) {
-	m := new(Heartbeat)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _Seaweed_KeepConnected_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(SeaweedServer).KeepConnected(&seaweedKeepConnectedServer{stream})
-}
-
-type Seaweed_KeepConnectedServer interface {
-	Send(*VolumeLocation) error
-	Recv() (*ClientListenRequest, error)
-	grpc.ServerStream
-}
-
-type seaweedKeepConnectedServer struct {
-	grpc.ServerStream
-}
-
-func (x *seaweedKeepConnectedServer) Send(m *VolumeLocation) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *seaweedKeepConnectedServer) Recv() (*ClientListenRequest, error) {
-	m := new(ClientListenRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-var _Seaweed_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "master_pb.Seaweed",
-	HandlerType: (*SeaweedServer)(nil),
-	Methods:     []grpc.MethodDesc{},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "SendHeartbeat",
-			Handler:       _Seaweed_SendHeartbeat_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "KeepConnected",
-			Handler:       _Seaweed_KeepConnected_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "seaweed.proto",
-}
-
-func init() { proto.RegisterFile("seaweed.proto", fileDescriptor0) }
-
-var fileDescriptor0 = []byte{
-	// 708 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x54, 0x41, 0x6f, 0xf3, 0x44,
-	0x10, 0xc5, 0x49, 0xbe, 0x24, 0x9e, 0x7c, 0xc9, 0x97, 0x2e, 0x08, 0xb9, 0xa5, 0x94, 0x60, 0x2e,
-	0x46, 0xa0, 0x08, 0x95, 0x33, 0x97, 0x46, 0x45, 0x54, 0x2d, 0x6a, 0xe5, 0x88, 0x1e, 0xb8, 0x58,
-	0x1b, 0xef, 0xb4, 0x5a, 0x75, 0xbd, 0x36, 0xbb, 0x9b, 0x36, 0xee, 0x85, 0x7f, 0xc3, 0x8d, 0xbf,
-	0xc0, 0x89, 0x1f, 0x86, 0x76, 0xd7, 0x4e, 0x42, 0x28, 0xb7, 0xd9, 0x37, 0x6f, 0xbc, 0xe3, 0xf7,
-	0x66, 0x16, 0xc6, 0x1a, 0xe9, 0x0b, 0x22, 0x9b, 0x57, 0xaa, 0x34, 0x25, 0x09, 0x0b, 0xaa, 0x0d,
-	0xaa, 0xac, 0x5a, 0xc5, 0x7f, 0x76, 0x20, 0xfc, 0x09, 0xa9, 0x32, 0x2b, 0xa4, 0x86, 0x4c, 0xa0,
-	0xc3, 0xab, 0x28, 0x98, 0x05, 0x49, 0x98, 0x76, 0x78, 0x45, 0x08, 0xf4, 0xaa, 0x52, 0x99, 0xa8,
-	0x33, 0x0b, 0x92, 0x71, 0xea, 0x62, 0xf2, 0x39, 0x40, 0xb5, 0x5e, 0x09, 0x9e, 0x67, 0x6b, 0x25,
-	0xa2, 0xae, 0xe3, 0x86, 0x1e, 0xf9, 0x45, 0x09, 0x92, 0xc0, 0xb4, 0xa0, 0x9b, 0xec, 0xb9, 0x14,
-	0xeb, 0x02, 0xb3, 0xbc, 0x5c, 0x4b, 0x13, 0xf5, 0x5c, 0xf9, 0xa4, 0xa0, 0x9b, 0x7b, 0x07, 0x2f,
-	0x2c, 0x4a, 0x66, 0xf0, 0xde, 0x32, 0x1f, 0xb8, 0xc0, 0xec, 0x09, 0xeb, 0xe8, 0xdd, 0x2c, 0x48,
-	0x7a, 0x29, 0x14, 0x74, 0xf3, 0x23, 0x17, 0x78, 0x8d, 0x35, 0xf9, 0x02, 0x46, 0x8c, 0x1a, 0x9a,
-	0xe5, 0x28, 0x0d, 0xaa, 0xa8, 0xef, 0xee, 0x02, 0x0b, 0x2d, 0x1c, 0x62, 0xfb, 0x53, 0x34, 0x7f,
-	0x8a, 0x06, 0x2e, 0xe3, 0x62, 0xdb, 0x1f, 0x65, 0x05, 0x97, 0x99, 0xeb, 0x7c, 0xe8, 0xae, 0x0e,
-	0x1d, 0x72, 0x67, 0xdb, 0xff, 0x01, 0x06, 0xbe, 0x37, 0x1d, 0x85, 0xb3, 0x6e, 0x32, 0x3a, 0xff,
-	0x6a, 0xbe, 0x55, 0x63, 0xee, 0xdb, 0xbb, 0x92, 0x0f, 0xa5, 0x2a, 0xa8, 0xe1, 0xa5, 0xfc, 0x19,
-	0xb5, 0xa6, 0x8f, 0x98, 0xb6, 0x35, 0xb1, 0x86, 0xa3, 0xad, 0x5c, 0x29, 0xea, 0xaa, 0x94, 0x1a,
-	0x49, 0x02, 0x1f, 0x7c, 0x7e, 0xc9, 0x5f, 0xf1, 0x86, 0x17, 0xdc, 0x38, 0x0d, 0x7b, 0xe9, 0x21,
-	0x4c, 0x4e, 0x21, 0xd4, 0x98, 0x2b, 0x34, 0xd7, 0x58, 0x3b, 0x55, 0xc3, 0x74, 0x07, 0x90, 0x4f,
-	0xa1, 0x2f, 0x90, 0x32, 0x54, 0x8d, 0xac, 0xcd, 0x29, 0xfe, 0xbb, 0x03, 0xd1, 0xff, 0xb5, 0xe6,
-	0x3c, 0x63, 0xee, 0xbe, 0x71, 0xda, 0xe1, 0xcc, 0x6a, 0xa2, 0xf9, 0x2b, 0xba, 0xaf, 0xf7, 0x52,
-	0x17, 0x93, 0x33, 0x80, 0xbc, 0x14, 0x02, 0x73, 0x5b, 0xd8, 0x7c, 0x7c, 0x0f, 0xb1, 0x9a, 0x39,
-	0x1b, 0x76, 0x76, 0xf5, 0xd2, 0xd0, 0x22, 0xde, 0xa9, 0x2f, 0xe1, 0x3d, 0x43, 0x81, 0xa6, 0x25,
-	0x78, 0xa7, 0x46, 0x1e, 0xf3, 0x94, 0x6f, 0x81, 0xf8, 0x23, 0xcb, 0x56, 0xf5, 0x96, 0xd8, 0x77,
-	0xc4, 0x69, 0x93, 0xb9, 0xa8, 0x5b, 0xf6, 0x67, 0x10, 0x2a, 0xa4, 0x2c, 0x2b, 0xa5, 0xa8, 0x9d,
-	0x79, 0xc3, 0x74, 0x68, 0x81, 0x5b, 0x29, 0x6a, 0xf2, 0x0d, 0x1c, 0x29, 0xac, 0x04, 0xcf, 0x69,
-	0x56, 0x09, 0x9a, 0x63, 0x81, 0xb2, 0xf5, 0x71, 0xda, 0x24, 0xee, 0x5a, 0x9c, 0x44, 0x30, 0x78,
-	0x46, 0xa5, 0xed, 0x6f, 0x85, 0x8e, 0xd2, 0x1e, 0xc9, 0x14, 0xba, 0xc6, 0x88, 0x08, 0x1c, 0x6a,
-	0xc3, 0x78, 0x00, 0xef, 0x2e, 0x8b, 0xca, 0xd4, 0xf1, 0x5f, 0x01, 0x7c, 0x58, 0xae, 0x2b, 0x54,
-	0x17, 0xa2, 0xcc, 0x9f, 0x2e, 0x37, 0x46, 0x51, 0x72, 0x0b, 0x13, 0x54, 0x54, 0xaf, 0x95, 0xed,
-	0x9d, 0x71, 0xf9, 0xe8, 0x24, 0x1d, 0x9d, 0x27, 0x7b, 0xe3, 0x71, 0x50, 0x33, 0xbf, 0xf4, 0x05,
-	0x0b, 0xc7, 0x4f, 0xc7, 0xb8, 0x7f, 0x3c, 0xf9, 0x15, 0xc6, 0xff, 0xca, 0x5b, 0x63, 0xec, 0xe8,
-	0x36, 0x56, 0xb9, 0xd8, 0x3a, 0x5e, 0x51, 0xc5, 0x4d, 0xdd, 0xac, 0x58, 0x73, 0xb2, 0x86, 0x34,
-	0x1b, 0xc4, 0x99, 0x8e, 0xba, 0xb3, 0xae, 0x1d, 0x62, 0x8f, 0x5c, 0x31, 0x1d, 0x7f, 0x0d, 0x1f,
-	0x2f, 0x04, 0x47, 0x69, 0x6e, 0xb8, 0x36, 0x28, 0x53, 0xfc, 0x6d, 0x8d, 0xda, 0xd8, 0x1b, 0x24,
-	0x2d, 0xb0, 0x59, 0x60, 0x17, 0xc7, 0xbf, 0xc3, 0xc4, 0x8f, 0xce, 0x4d, 0x99, 0xbb, 0xb9, 0xb1,
-	0xc2, 0xd8, 0xcd, 0xf5, 0x24, 0x1b, 0x1e, 0xac, 0x74, 0xe7, 0x70, 0xa5, 0x8f, 0x61, 0x28, 0xf1,
-	0x25, 0x7b, 0xde, 0xb5, 0x32, 0x90, 0xf8, 0x72, 0xcf, 0x99, 0xde, 0x4d, 0x06, 0xf3, 0xe9, 0x9e,
-	0x4b, 0x37, 0x93, 0xc1, 0x2c, 0xe5, 0xfc, 0x8f, 0x00, 0x06, 0x4b, 0xff, 0xfc, 0x90, 0x2b, 0x18,
-	0x2f, 0x51, 0xb2, 0xdd, 0x83, 0xf3, 0xc9, 0x9e, 0xba, 0x5b, 0xf4, 0xe4, 0xf4, 0x2d, 0xb4, 0xdd,
-	0xb6, 0xf8, 0xa3, 0x24, 0xf8, 0x2e, 0x20, 0x77, 0x30, 0xbe, 0x46, 0xac, 0x16, 0xa5, 0x94, 0x98,
-	0x1b, 0x64, 0xe4, 0x6c, 0xaf, 0xe8, 0x0d, 0x71, 0x4e, 0x8e, 0xff, 0xb3, 0xe7, 0xad, 0x22, 0xfe,
-	0x8b, 0xab, 0xbe, 0x7b, 0x1c, 0xbf, 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, 0xfd, 0x83, 0xd9, 0x6b,
-	0x2d, 0x05, 0x00, 0x00,
-}

weed/pb/volume_server.proto

@@ -0,0 +1,165 @@
+syntax = "proto3";
+
+package volume_server_pb;
+
+//////////////////////////////////////////////////
+
+service VolumeServer {
+    //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
+    rpc BatchDelete (BatchDeleteRequest) returns (BatchDeleteResponse) {
+    }
+    rpc VacuumVolumeCheck (VacuumVolumeCheckRequest) returns (VacuumVolumeCheckResponse) {
+    }
+    rpc VacuumVolumeCompact (VacuumVolumeCompactRequest) returns (VacuumVolumeCompactResponse) {
+    }
+    rpc VacuumVolumeCommit (VacuumVolumeCommitRequest) returns (VacuumVolumeCommitResponse) {
+    }
+    rpc VacuumVolumeCleanup (VacuumVolumeCleanupRequest) returns (VacuumVolumeCleanupResponse) {
+    }
+
+    rpc DeleteCollection (DeleteCollectionRequest) returns (DeleteCollectionResponse) {
+    }
+    rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) {
+    }
+
+    rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
+    }
+    rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) {
+    }
+    rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) {
+    }
+
+    rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
+    }
+    rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
+    }
+
+    // rpc VolumeUiPage (VolumeUiPageRequest) returns (VolumeUiPageResponse) {}
+
+}
+
+//////////////////////////////////////////////////
+
+message BatchDeleteRequest {
+    repeated string file_ids = 1;
+}
+
+message BatchDeleteResponse {
+    repeated DeleteResult results = 1;
+}
+message DeleteResult {
+    string file_id = 1;
+    int32 status = 2;
+    string error = 3;
+    uint32 size = 4;
+}
+
+message Empty {
+}
+
+message VacuumVolumeCheckRequest {
+    uint32 volumd_id = 1;
+}
+message VacuumVolumeCheckResponse {
+    double garbage_ratio = 1;
+}
+
+message VacuumVolumeCompactRequest {
+    uint32 volumd_id = 1;
+    int64 preallocate = 2;
+}
+message VacuumVolumeCompactResponse {
+}
+
+message VacuumVolumeCommitRequest {
+    uint32 volumd_id = 1;
+}
+message VacuumVolumeCommitResponse {
+}
+
+message VacuumVolumeCleanupRequest {
+    uint32 volumd_id = 1;
+}
+message VacuumVolumeCleanupResponse {
+}
+
+message DeleteCollectionRequest {
+    string collection = 1;
+}
+message DeleteCollectionResponse {
+}
+
+message AssignVolumeRequest {
+    uint32 volumd_id = 1;
+    string collection = 2;
+    int64 preallocate = 3;
+    string replication = 4;
+    string ttl = 5;
+}
+message AssignVolumeResponse {
+}
+
+message VolumeSyncStatusRequest {
+    uint32 volumd_id = 1;
+}
+message VolumeSyncStatusResponse {
+    uint32 volumd_id = 1;
+    string collection = 2;
+    string replication = 4;
+    string ttl = 5;
+    uint64 tail_offset = 6;
+    uint32 compact_revision = 7;
+    uint64 idx_file_size = 8;
+}
+
+message VolumeSyncIndexRequest {
+    uint32 volumd_id = 1;
+}
+message VolumeSyncIndexResponse {
+    bytes index_file_content = 1;
+}
+
+message VolumeSyncDataRequest {
+    uint32 volumd_id = 1;
+    uint32 revision = 2;
+    uint32 offset = 3;
+    uint32 size = 4;
+    string needle_id = 5;
+}
+message VolumeSyncDataResponse {
+    bytes file_content = 1;
+}
+
+message VolumeMountRequest {
+    uint32 volumd_id = 1;
+}
+message VolumeMountResponse {
+}
+
+message VolumeUnmountRequest {
+    uint32 volumd_id = 1;
+}
+message VolumeUnmountResponse {
+}
+
+message VolumeUiPageRequest {
+}
+message VolumeUiPageResponse {
+}
+
+message DiskStatus {
+    string dir = 1;
+    uint64 all = 2;
+    uint64 used = 3;
+    uint64 free = 4;
+}
+
+message MemStatus {
+    int32 goroutines = 1;
+    uint64 all = 2;
+    uint64 used = 3;
+    uint64 free = 4;
+    uint64 self = 5;
+    uint64 heap = 6;
+    uint64 stack = 7;
+}

weed/replication/replicator.go

@@ -0,0 +1,61 @@
+package replication
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/replication/sink"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+type Replicator struct {
+	sink   sink.ReplicationSink
+	source *source.FilerSource
+}
+
+func NewReplicator(sourceConfig util.Configuration, dataSink sink.ReplicationSink) *Replicator {
+
+	source := &source.FilerSource{}
+	source.Initialize(sourceConfig)
+
+	dataSink.SetSourceFiler(source)
+
+	return &Replicator{
+		sink:   dataSink,
+		source: source,
+	}
+}
+
+func (r *Replicator) Replicate(key string, message *filer_pb.EventNotification) error {
+	if !strings.HasPrefix(key, r.source.Dir) {
+		glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
+		return nil
+	}
+	newKey := filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):])
+	glog.V(3).Infof("replicate %s => %s", key, newKey)
+	key = newKey
+	if message.OldEntry != nil && message.NewEntry == nil {
+		glog.V(4).Infof("deleting %v", key)
+		return r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks)
+	}
+	if message.OldEntry == nil && message.NewEntry != nil {
+		glog.V(4).Infof("creating %v", key)
+		return r.sink.CreateEntry(key, message.NewEntry)
+	}
+	if message.OldEntry == nil && message.NewEntry == nil {
+		glog.V(0).Infof("weird message %+v", message)
+		return nil
+	}
+
+	foundExisting, err := r.sink.UpdateEntry(key, message.OldEntry, message.NewEntry, message.DeleteChunks)
+	if foundExisting {
+		glog.V(4).Infof("updated %v", key)
+		return err
+	}
+
+	glog.V(4).Infof("creating missing %v", key)
+	return r.sink.CreateEntry(key, message.NewEntry)
+}

weed/replication/sink/azuresink/azure_sink.go

@@ -0,0 +1,150 @@
+package azuresink
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/replication/sink"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+type AzureSink struct {
+	containerURL azblob.ContainerURL
+	container    string
+	dir          string
+	filerSource  *source.FilerSource
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &AzureSink{})
+}
+
+func (g *AzureSink) GetName() string {
+	return "azure"
+}
+
+func (g *AzureSink) GetSinkToDirectory() string {
+	return g.dir
+}
+
+func (g *AzureSink) Initialize(configuration util.Configuration) error {
+	return g.initialize(
+		configuration.GetString("account_name"),
+		configuration.GetString("account_key"),
+		configuration.GetString("container"),
+		configuration.GetString("directory"),
+	)
+}
+
+func (g *AzureSink) SetSourceFiler(s *source.FilerSource) {
+	g.filerSource = s
+}
+
+func (g *AzureSink) initialize(accountName, accountKey, container, dir string) error {
+	g.container = container
+	g.dir = dir
+
+	// Use your Storage account's name and key to create a credential object.
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		glog.Fatalf("failed to create Azure credential with account name:%s key:%s", accountName, accountKey)
+	}
+
+	// Create a request pipeline that is used to process HTTP(S) requests and responses.
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+
+	// Create an ServiceURL object that wraps the service URL and a request pipeline.
+	u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", accountName))
+	serviceURL := azblob.NewServiceURL(*u, p)
+
+	g.containerURL = serviceURL.NewContainerURL(g.container)
+
+	return nil
+}
+
+func (g *AzureSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
+
+	key = cleanKey(key)
+
+	if isDirectory {
+		key = key + "/"
+	}
+
+	ctx := context.Background()
+
+	if _, err := g.containerURL.NewBlobURL(key).Delete(ctx,
+		azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil {
+		return fmt.Errorf("azure delete %s/%s: %v", g.container, key, err)
+	}
+
+	return nil
+
+}
+
+func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
+
+	key = cleanKey(key)
+
+	if entry.IsDirectory {
+		return nil
+	}
+
+	totalSize := filer2.TotalSize(entry.Chunks)
+	chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int(totalSize))
+
+	ctx := context.Background()
+
+	// Create a URL that references a to-be-created blob in your
+	// Azure Storage account's container.
+	appendBlobURL := g.containerURL.NewAppendBlobURL(key)
+
+	_, err := appendBlobURL.Create(ctx, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{})
+	if err != nil {
+		return err
+	}
+
+	for _, chunk := range chunkViews {
+
+		fileUrl, err := g.filerSource.LookupFileId(chunk.FileId)
+		if err != nil {
+			return err
+		}
+
+		var writeErr error
+		_, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) {
+			_, writeErr = appendBlobURL.AppendBlock(ctx, bytes.NewReader(data), azblob.AppendBlobAccessConditions{}, nil)
+		})
+
+		if readErr != nil {
+			return readErr
+		}
+		if writeErr != nil {
+			return writeErr
+		}
+
+	}
+
+	return nil
+
+}
+
+func (g *AzureSink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+	key = cleanKey(key)
+	// TODO improve efficiency
+	return false, nil
+}
+
+func cleanKey(key string) string {
+	if strings.HasPrefix(key, "/") {
+		key = key[1:]
+	}
+	return key
+}

weed/replication/sink/b2sink/b2_sink.go

@@ -0,0 +1,144 @@
+package B2Sink
+
+import (
+	"context"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/replication/sink"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/kurin/blazer/b2"
+)
+
+type B2Sink struct {
+	client      *b2.Client
+	bucket      string
+	dir         string
+	filerSource *source.FilerSource
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &B2Sink{})
+}
+
+func (g *B2Sink) GetName() string {
+	return "backblaze"
+}
+
+func (g *B2Sink) GetSinkToDirectory() string {
+	return g.dir
+}
+
+func (g *B2Sink) Initialize(configuration util.Configuration) error {
+	return g.initialize(
+		configuration.GetString("b2_account_id"),
+		configuration.GetString("b2_master_application_key"),
+		configuration.GetString("bucket"),
+		configuration.GetString("directory"),
+	)
+}
+
+func (g *B2Sink) SetSourceFiler(s *source.FilerSource) {
+	g.filerSource = s
+}
+
+func (g *B2Sink) initialize(accountId, accountKey, bucket, dir string) error {
+	ctx := context.Background()
+	client, err := b2.NewClient(ctx, accountId, accountKey)
+	if err != nil {
+		return err
+	}
+
+	g.client = client
+	g.bucket = bucket
+	g.dir = dir
+
+	return nil
+}
+
+func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool) error {
+
+	key = cleanKey(key)
+
+	if isDirectory {
+		key = key + "/"
+	}
+
+	ctx := context.Background()
+
+	bucket, err := g.client.Bucket(ctx, g.bucket)
+	if err != nil {
+		return err
+	}
+
+	targetObject := bucket.Object(key)
+
+	return targetObject.Delete(ctx)
+
+}
+
+func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
+
+	key = cleanKey(key)
+
+	if entry.IsDirectory {
+		return nil
+	}
+
+	totalSize := filer2.TotalSize(entry.Chunks)
+	chunkViews := filer2.ViewFromChunks(entry.Chunks, 0, int(totalSize))
+
+	ctx := context.Background()
+
+	bucket, err := g.client.Bucket(ctx, g.bucket)
+	if err != nil {
+		return err
+	}
+
+	targetObject := bucket.Object(key)
+	writer := targetObject.NewWriter(ctx)
+
+	for _, chunk := range chunkViews {
+
+		fileUrl, err := g.filerSource.LookupFileId(chunk.FileId)
+		if err != nil {
+			return err
+		}
+
+		var writeErr error
+		_, readErr := util.ReadUrlAsStream(fileUrl, chunk.Offset, int(chunk.Size), func(data []byte) {
+			_, err := writer.Write(data)
+			if err != nil {
+				writeErr = err
+			}
+		})
+
+		if readErr != nil {
+			return readErr
+		}
+		if writeErr != nil {
+			return writeErr
+		}
+
+	}
+
+	return writer.Close()
+
+}
+
+func (g *B2Sink) UpdateEntry(key string, oldEntry, newEntry *filer_pb.Entry, deleteIncludeChunks bool) (foundExistingEntry bool, err error) {
+
+	key = cleanKey(key)
+
+	// TODO improve efficiency
+	return false, nil
+}
+
+func cleanKey(key string) string {
+	if strings.HasPrefix(key, "/") {
+		key = key[1:]
+	}
+	return key
+}

weed/replication/sink/filersink/fetch_write.go

@@ -0,0 +1,123 @@
+package filersink
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk) (replicatedChunks []*filer_pb.FileChunk, err error) {
+	if len(sourceChunks) == 0 {
+		return
+	}
+	var wg sync.WaitGroup
+	for _, sourceChunk := range sourceChunks {
+		wg.Add(1)
+		go func(chunk *filer_pb.FileChunk) {
+			defer wg.Done()
+			replicatedChunk, e := fs.replicateOneChunk(chunk)
+			if e != nil {
+				err = e
+			}
+			replicatedChunks = append(replicatedChunks, replicatedChunk)
+		}(sourceChunk)
+	}
+	wg.Wait()
+
+	return
+}
+
+func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk) (*filer_pb.FileChunk, error) {
+
+	fileId, err := fs.fetchAndWrite(sourceChunk)
+	if err != nil {
+		return nil, fmt.Errorf("copy %s: %v", sourceChunk.FileId, err)
+	}
+
+	return &filer_pb.FileChunk{
+		FileId:       fileId,
+		Offset:       sourceChunk.Offset,
+		Size:         sourceChunk.Size,
+		Mtime:        sourceChunk.Mtime,
+		ETag:         sourceChunk.ETag,
+		SourceFileId: sourceChunk.FileId,
+	}, nil
+}
+
+func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId string, err error) {
+
+	filename, header, readCloser, err := fs.filerSource.ReadPart(sourceChunk.FileId)
+	if err != nil {
+		return "", fmt.Errorf("read part %s: %v", sourceChunk.FileId, err)
+	}
+	defer readCloser.Close()
+
+	var host string
+
+	if err := fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.AssignVolumeRequest{
+			Count:       1,
+			Replication: fs.replication,
+			Collection:  fs.collection,
+			TtlSec:      fs.ttlSec,
+			DataCenter:  fs.dataCenter,
+		}
+
+		resp, err := client.AssignVolume(context.Background(), request)
+		if err != nil {
+			glog.V(0).Infof("assign volume failure %v: %v", request, err)
+			return err
+		}
+
+		fileId, host = resp.FileId, resp.Url
+
+		return nil
+	}); err != nil {
+		return "", fmt.Errorf("filerGrpcAddress assign volume: %v", err)
+	}
+
+	fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
+
+	glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header)
+
+	uploadResult, err := operation.Upload(fileUrl, filename, readCloser,
+		"gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, "")
+	if err != nil {
+		glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
+		return "", fmt.Errorf("upload data: %v", err)
+	}
+	if uploadResult.Error != "" {
+		glog.V(0).Infof("upload failure %v to %s: %v", filename, fileUrl, err)
+		return "", fmt.Errorf("upload result: %v", uploadResult.Error)
+	}
+
+	return
+}
+
+func (fs *FilerSink) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+
+	grpcConnection, err := util.GrpcDial(fs.grpcAddress)
+	if err != nil {
+		return fmt.Errorf("fail to dial %s: %v", fs.grpcAddress, err)
+	}
+	defer grpcConnection.Close()
+
+	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+
+	return fn(client)
+}
+
+func volumeId(fileId string) string {
+	lastCommaIndex := strings.LastIndex(fileId, ",")
+	if lastCommaIndex > 0 {
+		return fileId[:lastCommaIndex]
+	}
+	return fileId
+}