1.23

due to https://github.com/soheilhy/cmux/issues/64

fix https://github.com/chrislusf/seaweedfs/issues/820
fix https://github.com/chrislusf/seaweedfs/issues/840
fix https://github.com/chrislusf/seaweedfs/issues/841

.gitignore

@@ -79,3 +79,4 @@ test_data
 build
 target
 *.class
+other/java/hdfs/dependency-reduced-pom.xml

Dockerfile.go_build

@@ -1,6 +0,0 @@
-FROM cydev/go
-RUN go get github.com/chrislusf/seaweedfs/weed
-EXPOSE 8080
-EXPOSE 9333
-VOLUME /data
-ENTRYPOINT ["weed"]

README.md

@@ -1,24 +1,26 @@
 # SeaweedFS
 
+
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
+[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=604800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
 
 <h2 align="center">Supporting SeaweedFS</h2>
 
-SeaweedFS is Apache-licensed open source project, independent project with its ongoing development made 
-possible entirely thanks to the support by these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
-If you'd like to grow SeaweedFS even stronger, please consider to 
-<a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a>.
+SeaweedFS is an independent Apache-licensed open source project with its ongoing development made 
+possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
+If you'd like to grow SeaweedFS even stronger, please consider joining our
+<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
 
 Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
 Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
 
 Your support will be really appreciated by me and other supporters!
 
-<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsors SeaweedFS via Patreon</a></h3>
+<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
 
 <h4 align="center">Platinum</h4>
 
@@ -61,38 +63,41 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 
 SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
 
-There is only a 40 bytes disk storage overhead for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases.
+There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
 SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
 
-SeaweedFS can work very well with just the object store. [[Filer]] is added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
+SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
 
 ## Additional Features
-* Can choose no replication or different replication level, rack and data center aware
+* Can choose no replication or different replication levels, rack and data center aware
 * Automatic master servers failover - no single point of failure (SPOF)
 * Automatic Gzip compression depending on file mime type
-* Automatic compaction to reclaimed disk spaces after deletion or update
+* Automatic compaction to reclaim disk space after deletion or update
 * Servers in the same cluster can have different disk spaces, file systems, OS etc.
 * Adding/Removing servers does **not** cause any data re-balancing
 * Optionally fix the orientation for jpeg pictures
 * Support Etag, Accept-Range, Last-Modified, etc.
 * Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
+
 ## Filer Features
 * [filer server][Filer] provide "normal" directories and files via http.
 * [mount filer][Mount] to read and write files directly as a local directory via FUSE.
 * [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
-* [Async Backup To Cloud][BackupToCloud] can enjoy extreme fast local access and backup to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+* [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
+* [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
 [Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
 [AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
 [BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
+[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
 
 ## Example Usage
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.
-Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. I just use localhost as an example.
+Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
 
-SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in JSON or JSONP format.
+SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
 
 ### Start Master Server
 
@@ -107,7 +112,6 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in
 > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
 ```
 
-
 ### Write File ###
 
 To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url:
@@ -133,17 +137,17 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 ```
 ### Save File Id ###
 
-Now you can save the `fid`, 3,01637037d6 in this case, to a database field.
+Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
 
 The number 3 at the start represents a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
 
 The volume id is an unsigned 32-bit integer. The file key is an unsigned 64-bit integer. The file cookie is an unsigned 32-bit integer, used to prevent URL guessing.
 
-The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as string.
+The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as a string.
 
-If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most usage would not need 2^32 volumes.
+If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most uses will not need 2^32 volumes.
 
-If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are more than enough.
+If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
 
 ### Read File ###
 
@@ -211,7 +215,7 @@ You can also set the default replication strategy when starting the master serve
 
 ### Allocate File Key on specific data center ###
 
-Volume servers can start with a specific data center name:
+Volume servers can be started with a specific data center name:
 
 ```
  weed volume -dir=/tmp/1 -port=8080 -dataCenter=dc1
@@ -239,9 +243,9 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
 
 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
 
-The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, might not scale well for many concurrent users.
+The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, so it might not scale well for many concurrent users.
 
-Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and is generally stable.
+Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
 
 The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
 
@@ -251,7 +255,7 @@ For comparison, consider that an xfs inode structure in Linux is 536 bytes.
 
 The architecture is fairly simple. The actual data is stored in volumes on storage nodes. One volume server can have multiple volumes, and can both support read and write access with basic authentication.
 
-All volumes are managed by a master server. The master server contains volume id to volume server mapping. This is fairly static information, and could be cached easily.
+All volumes are managed by a master server. The master server contains the volume id to volume server mapping. This is fairly static information, and can be easily cached.
 
 On each write request, the master server also generates a file key, which is a growing 64-bit unsigned integer. Since write requests are not generally as frequent as read requests, one master server should be able to handle the concurrency well.
 
@@ -259,28 +263,27 @@ On each write request, the master server also generates a file key, which is a g
 
 When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content.
 
-When a client needs to read a file based on (volume id, file key, file cookie), it can ask the master server by the volume id for the (volume node url, volume node public url), or retrieve this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
+When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node url, volume node public url), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
 
 Please see the example for details on the write-read process.
 
 ### Storage Size ###
 
-In the current implementation, each volume can be 8x2^32 bytes (32GiB). This is because of we align content to 8 bytes. We can easily increase this to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
+In the current implementation, each volume can hold 32 gibibytes (32GiB or 8x2^32 bytes). This is because we align content to 8 bytes. We can easily increase this to 64GiB, or 128GiB, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
 
-There can be 2^32 volumes. So total system size is 8 x 2^32 bytes x 2^32 = 8 x 4GiB x 4Gi = 128EiB (2^67 bytes, or 128 exbibytes).
+There can be 4 gibibytes (4GiB or 2^32 bytes) of volumes. So the total system size is 8 x 4GiB x 4GiB which is 128 exbibytes (128EiB or 2^67 bytes).
 
 Each individual file size is limited to the volume size.
 
 ### Saving memory ###
 
-All file meta information on volume server is readable from memory without disk access. Each file just takes an 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own the space cost for the map. But usually the disk runs out before the memory does.
-
+All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
 ## Compared to Other File Systems ##
 
 Most other distributed file systems seem more complicated than necessary.
 
-SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Please raise an issue with any questions or update this file with clarifications.
+SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
 
 ### Compared to HDFS ###
 
@@ -319,7 +322,7 @@ Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more com
 
 SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. 
 
-Same as SeaweedFS, Ceph is also based on a object store RADOS. Ceph is rather complicated with mixed reviews.
+Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
 
 Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS places data by assigned volumes.
 
@@ -339,31 +342,31 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Cassa
 More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
 Other key features include: Erasure Encoding, JWT security.
 
-This is a super exciting project! And I need helpers and [support](https://www.patreon.com/seaweedfs)!
+This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
 
 ## Installation guide for users who are not familiar with golang
 
-step 1: install go on your machine and setup the environment by following the instructions from the following link:
+Step 1: install go on your machine and setup the environment by following the instructions at:
 
 https://golang.org/doc/install
 
 make sure you set up your $GOPATH
 
 
-step 2: also you may need to install Mercurial by following the instructions below
+Step 2: also you may need to install Mercurial by following the instructions at:
 
 http://mercurial.selenic.com/downloads
 
-step 3: download, compile, and install the project by executing the following command
+Step 3: download, compile, and install the project by executing the following command
 
 ```bash
 go get github.com/chrislusf/seaweedfs/weed
 ```
 
-once this is done, you should see the executable "weed" under `$GOPATH/bin`
+Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-step 4: after you modify your code locally, you could start a local build by calling `go install` under 
+Step 4: after you modify your code locally, you could start a local build by calling `go install` under 
 
 ```
 $GOPATH/src/github.com/chrislusf/seaweedfs/weed
@@ -373,7 +376,7 @@ $GOPATH/src/github.com/chrislusf/seaweedfs/weed
 
 ### Hard Drive Performance ###
 
-When testing read performance on SeaweedFS, it basically becomes performance test for your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
+When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
 
 ### Solid State Disk
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM frolvlad/alpine-glibc:alpine-3.5
+FROM frolvlad/alpine-glibc
 
 # Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
 RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
@@ -17,6 +17,8 @@ EXPOSE 18888
 EXPOSE 8888
 # master server shared gprc+http port
 EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
 
 VOLUME /data
 

docker/Dockerfile.go_build

@@ -0,0 +1,25 @@
+FROM golang:latest
+RUN go get github.com/chrislusf/seaweedfs/weed
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc+http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+VOLUME /data
+
+RUN mkdir -p /etc/seaweedfs
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+RUN cp /go/bin/weed /usr/bin/
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/README.md

@@ -0,0 +1,21 @@
+# Docker
+
+
+## Try it out
+
+```bash
+
+wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-compose.yml
+
+docker-compose -f seaweedfs-compose.yml -p seaweedfs up
+
+```
+
+## Development
+
+```bash
+cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
+
+docker-compose -f dev-compose.yml -p seaweedfs up
+
+```

docker/dev-compose.yml

@@ -0,0 +1,42 @@
+version: '2'
+
+services:
+  master:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 9333:9333
+    command: "master"
+  volume:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8080:8080
+    - 18080:18080
+    command: 'volume -max=5 -mserver="master:9333" -port=8080'
+    depends_on:
+    - master
+  filer:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8888:8888
+    - 18888:18888
+    command: 'filer -master="master:9333"'
+    depends_on:
+    - master
+    - volume
+  s3:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+    - master
+    - volume
+    - filer

docker/docker-compose.yml

@@ -1,40 +0,0 @@
-version: '2'
-
-services:
-  master:
-    #image: chrislusf/seaweedfs # use a remote image
-    build: . # build our container from the local Dockerfile
-    ports:
-    - 9333:9333
-    command: "master"
-    networks:
-      default:
-        aliases:
-        - seaweed_master
-  volume:
-    #image: chrislusf/seaweedfs # use a remote image
-    build: . # build our container from the local Dockerfile
-    ports:
-    - 8080:8080
-    - 18080:18080
-    command: 'volume -max=5 -mserver="master:9333" -port=8080'
-    depends_on:
-    - master
-    networks:
-      default:
-        aliases:
-        - seaweed_volume
-  filer:
-    #image: chrislusf/seaweedfs # use a remote image
-    build: . # build our container from the local Dockerfile
-    ports:
-    - 8888:8888
-    - 18888:18888
-    command: 'filer -master="master:9333"'
-    depends_on:
-    - master
-    - volume
-    networks:
-      default:
-        aliases:
-        - seaweed_filer

docker/entrypoint.sh

@@ -29,10 +29,19 @@ case "$1" in
   	;;
 
   'filer')
-  	ARGS="-ip `hostname -i`"
+  	ARGS="-ip `hostname -i` "
   	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
 		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
 	fi
+	mkdir -p /data/filerdb
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  's3')
+  	ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
+  	if [ -n "$FILER_PORT_8888_TCP_ADDR" ] ; then
+		ARGS="$ARGS -filer=$FILER_PORT_8888_TCP_ADDR:$FILER_PORT_8888_TCP_PORT"
+	fi
   	exec /usr/bin/weed $@ $ARGS
 	;;
 

docker/filer.toml

@@ -1,3 +1,3 @@
 [leveldb]
 enabled = true
-dir = "."
+dir = "/data/filerdb"

docker/seaweedfs-compose.yml

@@ -0,0 +1,36 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 9333:9333
+    command: "master"
+  volume:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8080:8080
+    - 18080:18080
+    command: 'volume -max=15 -mserver="master:9333" -port=8080'
+    depends_on:
+    - master
+  filer:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8888:8888
+    - 18888:18888
+    command: 'filer -master="master:9333"'
+    tty: true
+    stdin_open: true
+    depends_on:
+    - master
+    - volume
+  s3:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+    - master
+    - volume
+    - filer

other/java/client/pom.xml

@@ -1,16 +1,21 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>seaweedfs</groupId>
-    <artifactId>client</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-client</artifactId>
+    <version>1.0.5</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
 
     <properties>
         <protobuf.version>3.5.1</protobuf.version>
         <grpc.version>1.16.1</grpc.version>
+        <guava.version>26.0-jre</guava.version>
     </properties>
 
     <dependencies>
@@ -20,6 +25,11 @@
             <artifactId>protobuf-java</artifactId>
             <version>${protobuf.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-netty-shaded</artifactId>
@@ -35,8 +45,30 @@
             <artifactId>grpc-stub</artifactId>
             <version>${grpc.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.6</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
     <build>
         <extensions>
             <extension>
@@ -46,6 +78,14 @@
             </extension>
         </extensions>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>7</source>
+                    <target>7</target>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.xolstice.maven.plugins</groupId>
                 <artifactId>protobuf-maven-plugin</artifactId>
@@ -64,6 +104,57 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.5</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.7</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 

other/java/client/src/main/java/seaweedfs/client/ByteBufferOutputStream.java

@@ -0,0 +1,21 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public class ByteBufferOutputStream extends OutputStream {
+    private final ByteBuffer buf;
+
+    public ByteBufferOutputStream(ByteBuffer buf) {
+        this.buf = buf;
+    }
+
+    public void write(int b) throws IOException {
+        this.buf.put((byte)b);
+    }
+
+    public void write(byte[] b, int off, int len) throws IOException {
+        this.buf.put(b, off, len);
+    }
+}

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -0,0 +1,225 @@
+package seaweedfs.client;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class FilerClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
+
+    private FilerGrpcClient filerGrpcClient;
+
+    public FilerClient(String host, int grpcPort) {
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+    }
+
+    public FilerClient(FilerGrpcClient filerGrpcClient) {
+        this.filerGrpcClient = filerGrpcClient;
+    }
+
+    public boolean mkdirs(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return mkdirs(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean mkdirs(String path, int mode, String userName, String[] groupNames) {
+        return mkdirs(path, mode, 0, 0, userName, groupNames);
+    }
+
+    public boolean mkdirs(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        if ("/".equals(path)) {
+            return true;
+        }
+
+        mkdirs(parent, mode, uid, gid, userName, groupNames);
+
+        FilerProto.Entry existingEntry = lookupEntry(parent, name);
+
+        if (existingEntry != null) {
+            return true;
+        }
+
+        return createEntry(
+            parent,
+            newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
+        );
+
+    }
+
+    public boolean rm(String path, boolean isRecursive) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        return deleteEntry(
+            parent,
+            name,
+            true,
+            isRecursive);
+    }
+
+    public boolean touch(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return touch(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        FilerProto.Entry entry = lookupEntry(parent, name);
+        if (entry == null) {
+            return createEntry(
+                parent,
+                newFileEntry(name, mode, uid, gid, userName, groupNames).build()
+            );
+        }
+        long now = System.currentTimeMillis() / 1000L;
+        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
+            .setMtime(now)
+            .setUid(uid)
+            .setGid(gid)
+            .setUserName(userName)
+            .clearGroupName()
+            .addAllGroupName(Arrays.asList(groupNames));
+        return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
+    }
+
+    public FilerProto.Entry.Builder newDirectoryEntry(String name, int mode,
+                                                      int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+            .setName(name)
+            .setIsDirectory(true)
+            .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                .setMtime(now)
+                .setCrtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setFileMode(mode | 1 << 31)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public FilerProto.Entry.Builder newFileEntry(String name, int mode,
+                                                 int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+            .setName(name)
+            .setIsDirectory(false)
+            .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                .setMtime(now)
+                .setCrtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setFileMode(mode)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public List<FilerProto.Entry> listEntries(String path) {
+        List<FilerProto.Entry> results = new ArrayList<FilerProto.Entry>();
+        String lastFileName = "";
+        for (int limit = Integer.MAX_VALUE; limit > 0; ) {
+            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024);
+            if (t == null) {
+                break;
+            }
+            int nSize = t.size();
+            if (nSize > 0) {
+                limit -= nSize;
+                lastFileName = t.get(nSize - 1).getName();
+            }
+            results.addAll(t);
+            if (t.size() < 1024) {
+                break;
+            }
+        }
+        return results;
+    }
+
+    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
+        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+            .setDirectory(path)
+            .setPrefix(entryPrefix)
+            .setStartFromFileName(lastEntryName)
+            .setLimit(limit)
+            .build()).getEntriesList();
+    }
+
+    public FilerProto.Entry lookupEntry(String directory, String entryName) {
+        try {
+            return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+                FilerProto.LookupDirectoryEntryRequest.newBuilder()
+                    .setDirectory(directory)
+                    .setName(entryName)
+                    .build()).getEntry();
+        } catch (Exception e) {
+            LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
+            return null;
+        }
+    }
+
+
+    public boolean createEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setEntry(entry)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean updateEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setEntry(entry)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive) {
+        try {
+            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setName(entryName)
+                .setIsDeleteData(isDeleteFileChunk)
+                .setIsRecursive(isRecursive)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("deleteEntry {}/{}: {}", parent, entryName, e);
+            return false;
+        }
+        return true;
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -16,8 +16,8 @@ public class FilerGrpcClient {
     private final SeaweedFilerGrpc.SeaweedFilerFutureStub futureStub;
 
 
-    public FilerGrpcClient(String host, int port) {
-        this(ManagedChannelBuilder.forAddress(host, port).usePlaintext());
+    public FilerGrpcClient(String host, int grpcPort) {
+        this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
     }
 
     public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -1,14 +1,15 @@
-package seaweed.hdfs;
+package seaweedfs.client;
 
 import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.HttpClientBuilder;
-import seaweedfs.client.FilerGrpcClient;
-import seaweedfs.client.FilerProto;
 
 import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -17,6 +18,8 @@ import java.util.Map;
 
 public class SeaweedRead {
 
+    // private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
+
     // returns bytesRead
     public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
                             final long position, final byte[] buffer, final int bufferOffset,
@@ -37,6 +40,7 @@ public class SeaweedRead {
 
         //TODO parallel this
         long readCount = 0;
+        int startOffset = bufferOffset;
         for (ChunkView chunkView : chunkViews) {
             FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId));
             if (locations.getLocationsCount() == 0) {
@@ -44,39 +48,60 @@ public class SeaweedRead {
                 return 0;
             }
 
-            HttpClient client = HttpClientBuilder.create().build();
-            HttpGet request = new HttpGet(
-                String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));
-            request.setHeader("Range",
-                String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size));
+            int len = readChunkView(position, buffer, startOffset, chunkView, locations);
 
-            try {
-                HttpResponse response = client.execute(request);
-                HttpEntity entity = response.getEntity();
+            readCount += len;
+            startOffset += len;
 
-                readCount += entity.getContent().read(buffer,
-                    (int) (chunkView.logicOffset - position),
-                    (int) (chunkView.logicOffset - position + chunkView.size));
-
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
         }
 
         return readCount;
     }
 
-    private static List<ChunkView> viewFromVisibles(List<VisibleInterval> visibleIntervals, long offset, long size) {
+    private static int readChunkView(long position, byte[] buffer, int startOffset, ChunkView chunkView, FilerProto.Locations locations) {
+        HttpClient client = HttpClientBuilder.create().build();
+        HttpGet request = new HttpGet(
+            String.format("http://%s/%s", locations.getLocations(0).getUrl(), chunkView.fileId));
+
+        if (!chunkView.isFullChunk) {
+            request.setHeader(HttpHeaders.ACCEPT_ENCODING, "");
+            request.setHeader(HttpHeaders.RANGE,
+                String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size));
+        }
+
+        try {
+            HttpResponse response = client.execute(request);
+            HttpEntity entity = response.getEntity();
+
+            int len = (int) (chunkView.logicOffset - position + chunkView.size);
+            OutputStream outputStream = new ByteBufferOutputStream(ByteBuffer.wrap(buffer, startOffset, len));
+            entity.writeTo(outputStream);
+            // LOG.debug("* read chunkView:{} startOffset:{} length:{}", chunkView, startOffset, len);
+
+            return len;
+
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return 0;
+    }
+
+    public static List<ChunkView> viewFromVisibles(List<VisibleInterval> visibleIntervals, long offset, long size) {
         List<ChunkView> views = new ArrayList<>();
 
         long stop = offset + size;
         for (VisibleInterval chunk : visibleIntervals) {
-            views.add(new ChunkView(
-                chunk.fileId,
-                offset - chunk.start,
-                Math.min(chunk.stop, stop) - offset,
-                offset
-            ));
+            if (chunk.start <= offset && offset < chunk.stop && offset < stop) {
+                boolean isFullChunk = chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop;
+                views.add(new ChunkView(
+                    chunk.fileId,
+                    offset - chunk.start,
+                    Math.min(chunk.stop, stop) - offset,
+                    offset,
+                    isFullChunk
+                ));
+                offset = Math.min(chunk.stop, stop);
+            }
         }
         return views;
     }
@@ -90,14 +115,10 @@ public class SeaweedRead {
             }
         });
 
-        List<VisibleInterval> newVisibles = new ArrayList<>();
         List<VisibleInterval> visibles = new ArrayList<>();
         for (FilerProto.FileChunk chunk : chunks) {
-            newVisibles = mergeIntoVisibles(visibles, newVisibles, chunk);
-            visibles.clear();
-            List<VisibleInterval> t = visibles;
-            visibles = newVisibles;
-            newVisibles = t;
+            List<VisibleInterval> newVisibles = new ArrayList<>();
+            visibles = mergeIntoVisibles(visibles, newVisibles, chunk);
         }
 
         return visibles;
@@ -110,7 +131,8 @@ public class SeaweedRead {
             chunk.getOffset(),
             chunk.getOffset() + chunk.getSize(),
             chunk.getFileId(),
-            chunk.getMtime()
+            chunk.getMtime(),
+            true
         );
 
         // easy cases to speed up
@@ -129,7 +151,8 @@ public class SeaweedRead {
                     v.start,
                     chunk.getOffset(),
                     v.fileId,
-                    v.modifiedTime
+                    v.modifiedTime,
+                    false
                 ));
             }
             long chunkStop = chunk.getOffset() + chunk.getSize();
@@ -138,7 +161,8 @@ public class SeaweedRead {
                     chunkStop,
                     v.stop,
                     v.fileId,
-                    v.modifiedTime
+                    v.modifiedTime,
+                    false
                 ));
             }
             if (chunkStop <= v.start || v.stop <= chunk.getOffset()) {
@@ -168,31 +192,68 @@ public class SeaweedRead {
         return fileId;
     }
 
-    public static class VisibleInterval {
-        long start;
-        long stop;
-        long modifiedTime;
-        String fileId;
+    public static long totalSize(List<FilerProto.FileChunk> chunksList) {
+        long size = 0;
+        for (FilerProto.FileChunk chunk : chunksList) {
+            long t = chunk.getOffset() + chunk.getSize();
+            if (size < t) {
+                size = t;
+            }
+        }
+        return size;
+    }
 
-        public VisibleInterval(long start, long stop, String fileId, long modifiedTime) {
+    public static class VisibleInterval {
+        public final long start;
+        public final long stop;
+        public final long modifiedTime;
+        public final String fileId;
+        public final boolean isFullChunk;
+
+        public VisibleInterval(long start, long stop, String fileId, long modifiedTime, boolean isFullChunk) {
             this.start = start;
             this.stop = stop;
             this.modifiedTime = modifiedTime;
             this.fileId = fileId;
+            this.isFullChunk = isFullChunk;
+        }
+
+        @Override
+        public String toString() {
+            return "VisibleInterval{" +
+                "start=" + start +
+                ", stop=" + stop +
+                ", modifiedTime=" + modifiedTime +
+                ", fileId='" + fileId + '\'' +
+                ", isFullChunk=" + isFullChunk +
+                '}';
         }
     }
 
     public static class ChunkView {
-        String fileId;
-        long offset;
-        long size;
-        long logicOffset;
+        public final String fileId;
+        public final long offset;
+        public final long size;
+        public final long logicOffset;
+        public final boolean isFullChunk;
 
-        public ChunkView(String fileId, long offset, long size, long logicOffset) {
+        public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk) {
             this.fileId = fileId;
             this.offset = offset;
             this.size = size;
             this.logicOffset = logicOffset;
+            this.isFullChunk = isFullChunk;
+        }
+
+        @Override
+        public String toString() {
+            return "ChunkView{" +
+                "fileId='" + fileId + '\'' +
+                ", offset=" + offset +
+                ", size=" + size +
+                ", logicOffset=" + logicOffset +
+                ", isFullChunk=" + isFullChunk +
+                '}';
         }
     }
 

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -1,14 +1,11 @@
-package seaweed.hdfs;
+package seaweedfs.client;
 
-import org.apache.hadoop.fs.Path;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.mime.HttpMultipartMode;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
-import seaweedfs.client.FilerGrpcClient;
-import seaweedfs.client.FilerProto;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -47,10 +44,10 @@ public class SeaweedWrite {
     }
 
     public static void writeMeta(final FilerGrpcClient filerGrpcClient,
-                                 final Path path, final FilerProto.Entry.Builder entry) {
+                                 final String parentDirectory, final FilerProto.Entry.Builder entry) {
         filerGrpcClient.getBlockingStub().createEntry(
             FilerProto.CreateEntryRequest.newBuilder()
-                .setDirectory(path.getParent().toUri().getPath())
+                .setDirectory(parentDirectory)
                 .setEntry(entry)
                 .build()
         );

other/java/client/src/main/proto/filer.proto

@@ -97,6 +97,7 @@ message FuseAttributes {
     int32 ttl_sec = 10;
     string user_name = 11; // for hdfs
     repeated string group_name = 12; // for hdfs
+    string symlink_target = 13;
 }
 
 message CreateEntryRequest {
@@ -117,7 +118,7 @@ message UpdateEntryResponse {
 message DeleteEntryRequest {
     string directory = 1;
     string name = 2;
-    bool is_directory = 3;
+    // bool is_directory = 3;
     bool is_delete_data = 4;
     bool is_recursive = 5;
 }

other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java

@@ -0,0 +1,63 @@
+package seaweedfs.client;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SeaweedReadTest {
+
+    @Test
+    public void testNonOverlappingVisibleIntervals() {
+        List<FilerProto.FileChunk> chunks = new ArrayList<>();
+        chunks.add(FilerProto.FileChunk.newBuilder()
+            .setFileId("aaa")
+            .setOffset(0)
+            .setSize(100)
+            .setMtime(1000)
+            .build());
+        chunks.add(FilerProto.FileChunk.newBuilder()
+            .setFileId("bbb")
+            .setOffset(100)
+            .setSize(133)
+            .setMtime(2000)
+            .build());
+
+        List<SeaweedRead.VisibleInterval> visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(chunks);
+        for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) {
+            System.out.println("visible:" + visibleInterval);
+        }
+
+        Assert.assertEquals(visibleIntervals.size(), 2);
+
+        SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0);
+        Assert.assertEquals(visibleInterval.start, 0);
+        Assert.assertEquals(visibleInterval.stop, 100);
+        Assert.assertEquals(visibleInterval.modifiedTime, 1000);
+        Assert.assertEquals(visibleInterval.fileId, "aaa");
+
+        visibleInterval = visibleIntervals.get(1);
+        Assert.assertEquals(visibleInterval.start, 100);
+        Assert.assertEquals(visibleInterval.stop, 233);
+        Assert.assertEquals(visibleInterval.modifiedTime, 2000);
+        Assert.assertEquals(visibleInterval.fileId, "bbb");
+
+        List<SeaweedRead.ChunkView> chunkViews = SeaweedRead.viewFromVisibles(visibleIntervals, 0, 233);
+
+        SeaweedRead.ChunkView chunkView = chunkViews.get(0);
+        Assert.assertEquals(chunkView.offset, 0);
+        Assert.assertEquals(chunkView.size, 100);
+        Assert.assertEquals(chunkView.logicOffset, 0);
+        Assert.assertEquals(chunkView.fileId, "aaa");
+
+        chunkView = chunkViews.get(1);
+        Assert.assertEquals(chunkView.offset, 0);
+        Assert.assertEquals(chunkView.size, 133);
+        Assert.assertEquals(chunkView.logicOffset, 100);
+        Assert.assertEquals(chunkView.fileId, "bbb");
+
+
+    }
+
+}

other/java/hdfs/pom.xml

@@ -4,9 +4,27 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <groupId>seaweedfs</groupId>
-    <artifactId>hadoop-client</artifactId>
-    <version>1.0-SNAPSHOT</version>
+    <properties>
+        <seaweedfs.client.version>1.0.5</seaweedfs.client.version>
+        <hadoop.version>3.1.1</hadoop.version>
+    </properties>
+
+    <groupId>com.github.chrislusf</groupId>
+    <artifactId>seaweedfs-hadoop-client</artifactId>
+    <version>${seaweedfs.client.version}</version>
+
+    <parent>
+        <groupId>org.sonatype.oss</groupId>
+        <artifactId>oss-parent</artifactId>
+        <version>9</version>
+    </parent>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
     <build>
         <plugins>
             <plugin>
@@ -17,13 +35,109 @@
                     <target>7</target>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                        <exclude>org/slf4j/**</exclude>
+                                        <exclude>META-INF/maven/org.slf4j/**</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.google</pattern>
+                                    <shadedPattern>shaded.com.google</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>io.grpc.internal</pattern>
+                                    <shadedPattern>shaded.io.grpc.internal</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons</pattern>
+                                    <shadedPattern>shaded.org.apache.commons</shadedPattern>
+                                    <excludes>
+                                        <exclude>org.apache.hadoop</exclude>
+                                        <exclude>org.apache.log4j</exclude>
+                                    </excludes>
+                                </relocation>
+                            </relocations>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-gpg-plugin</artifactId>
+                <version>1.5</version>
+                <executions>
+                    <execution>
+                        <id>sign-artifacts</id>
+                        <phase>verify</phase>
+                        <goals>
+                            <goal>sign</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonatype.plugins</groupId>
+                <artifactId>nexus-staging-maven-plugin</artifactId>
+                <version>1.6.7</version>
+                <extensions>true</extensions>
+                <configuration>
+                    <serverId>ossrh</serverId>
+                    <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+                    <autoReleaseAfterClose>true</autoReleaseAfterClose>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.9.1</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 
-    <properties>
-        <hadoop.version>3.1.1</hadoop.version>
-    </properties>
-
     <dependencies>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -31,20 +145,15 @@
             <version>${hadoop.version}</version>
         </dependency>
         <dependency>
-            <groupId>seaweedfs</groupId>
-            <artifactId>client</artifactId>
-            <version>1.0-SNAPSHOT</version>
+            <groupId>com.github.chrislusf</groupId>
+            <artifactId>seaweedfs-client</artifactId>
+            <version>${seaweedfs.client.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpmime</artifactId>
-            <version>4.5.2</version>
-        </dependency>
     </dependencies>
 
 </project>

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -5,21 +5,38 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
 
 public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
-    public static final int FS_SEAWEED_DEFAULT_PORT = 8333;
-    public static final String FS_SEAWEED_HOST = "fs.seaweed.host";
-    public static final String FS_SEAWEED_HOST_PORT = "fs.seaweed.host.port";
+    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
+    public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
+    public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
+    private static int BUFFER_SIZE = 16 * 1024 * 1024;
 
     private URI uri;
     private Path workingDirectory = new Path("/");
@@ -30,7 +47,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     }
 
     public String getScheme() {
-        return "seaweed";
+        return "seaweedfs";
     }
 
     @Override
@@ -39,16 +56,18 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
         // get host information from uri (overrides info in conf)
         String host = uri.getHost();
-        host = (host == null) ? conf.get(FS_SEAWEED_HOST, null) : host;
+        host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
         if (host == null) {
             throw new IOException("Invalid host specified");
         }
-        conf.set(FS_SEAWEED_HOST, host);
+        conf.set(FS_SEAWEED_FILER_HOST, host);
 
         // get port information from uri, (overrides info in conf)
         int port = uri.getPort();
         port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
-        conf.setInt(FS_SEAWEED_HOST_PORT, port);
+        conf.setInt(FS_SEAWEED_FILER_PORT, port);
+
+        conf.setInt(IO_FILE_BUFFER_SIZE_KEY, BUFFER_SIZE);
 
         setConf(conf);
         this.uri = uri;
@@ -56,41 +75,61 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
     }
 
+    @Override
     public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+
+        LOG.debug("open path: {} bufferSize:{}", path, bufferSize);
+
         path = qualify(path);
 
-        return null;
+        try {
+            InputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize);
+            return new FSDataInputStream(inputStream);
+        } catch (Exception ex) {
+            return null;
+        }
     }
 
+    @Override
     public FSDataOutputStream create(Path path, FsPermission permission, final boolean overwrite, final int bufferSize,
                                      final short replication, final long blockSize, final Progressable progress) throws IOException {
+
+        LOG.debug("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize);
+
         path = qualify(path);
 
         try {
             String replicaPlacement = String.format("%03d", replication - 1);
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, replicaPlacement);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, bufferSize, replicaPlacement);
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             return null;
         }
     }
 
+    @Override
     public FSDataOutputStream append(Path path, int bufferSize, Progressable progressable) throws IOException {
+
+        LOG.debug("append path: {} bufferSize:{}", path, bufferSize);
+
         path = qualify(path);
         try {
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, "");
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, bufferSize, "");
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             return null;
         }
     }
 
-    public boolean rename(Path src, Path dst) throws IOException {
+    @Override
+    public boolean rename(Path src, Path dst) {
 
-        Path parentFolder = src.getParent();
-        if (parentFolder == null) {
+        LOG.debug("rename path: {} => {}", src, dst);
+
+        if (src.isRoot()) {
             return false;
         }
+
         if (src.equals(dst)) {
             return true;
         }
@@ -110,14 +149,18 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         Path qualifiedDstPath = qualify(adjustedDst);
 
         seaweedFileSystemStore.rename(qualifiedSrcPath, qualifiedDstPath);
-        return false;
+        return true;
     }
 
-    public boolean delete(Path path, boolean recursive) throws IOException {
+    @Override
+    public boolean delete(Path path, boolean recursive) {
+
+        LOG.debug("delete path: {} recursive:{}", path, recursive);
 
         path = qualify(path);
 
         FileStatus fileStatus = getFileStatus(path);
+
         if (fileStatus == null) {
             return true;
         }
@@ -126,17 +169,22 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
     }
 
-    public FileStatus[] listStatus(Path path) throws FileNotFoundException, IOException {
+    @Override
+    public FileStatus[] listStatus(Path path) throws IOException {
+
+        LOG.debug("listStatus path: {}", path);
 
         path = qualify(path);
 
         return seaweedFileSystemStore.listEntries(path);
     }
 
+    @Override
     public Path getWorkingDirectory() {
         return workingDirectory;
     }
 
+    @Override
     public void setWorkingDirectory(Path path) {
         if (path.isAbsolute()) {
             workingDirectory = path;
@@ -145,35 +193,402 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         }
     }
 
+    @Override
     public boolean mkdirs(Path path, FsPermission fsPermission) throws IOException {
 
+        LOG.debug("mkdirs path: {}", path);
+
         path = qualify(path);
 
-        try {
-            FileStatus fileStatus = getFileStatus(path);
+        FileStatus fileStatus = getFileStatus(path);
+
+        if (fileStatus == null) {
 
-            if (fileStatus.isDirectory()) {
-                return true;
-            } else {
-                throw new FileAlreadyExistsException("Path is a file: " + path);
-            }
-        } catch (FileNotFoundException e) {
             UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
             return seaweedFileSystemStore.createDirectory(path, currentUser,
                 fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
                 FsPermission.getUMask(getConf()));
+
+        }
+
+        if (fileStatus.isDirectory()) {
+            return true;
+        } else {
+            throw new FileAlreadyExistsException("Path is a file: " + path);
         }
     }
 
-    public FileStatus getFileStatus(Path path) throws IOException {
+    @Override
+    public FileStatus getFileStatus(Path path) {
+
+        LOG.debug("getFileStatus path: {}", path);
 
         path = qualify(path);
 
         return seaweedFileSystemStore.getFileStatus(path);
     }
 
+    /**
+     * Set owner of a path (i.e. a file or a directory).
+     * The parameters owner and group cannot both be null.
+     *
+     * @param path  The path
+     * @param owner If it is null, the original username remains unchanged.
+     * @param group If it is null, the original groupname remains unchanged.
+     */
+    @Override
+    public void setOwner(Path path, final String owner, final String group)
+        throws IOException {
+        LOG.debug("setOwner path: {}", path);
+        path = qualify(path);
+
+        seaweedFileSystemStore.setOwner(path, owner, group);
+    }
+
+
+    /**
+     * Set permission of a path.
+     *
+     * @param path       The path
+     * @param permission Access permission
+     */
+    @Override
+    public void setPermission(Path path, final FsPermission permission) throws IOException {
+        LOG.debug("setPermission path: {}", path);
+
+        if (permission == null) {
+            throw new IllegalArgumentException("The permission can't be null");
+        }
+
+        path = qualify(path);
+
+        seaweedFileSystemStore.setPermission(path, permission);
+    }
+
     Path qualify(Path path) {
         return path.makeQualified(uri, workingDirectory);
     }
 
+    /**
+     * Concat existing files together.
+     * @param trg the path to the target destination.
+     * @param psrcs the paths to the sources to use for the concatenation.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default).
+     */
+    @Override
+    public void concat(final Path trg, final Path [] psrcs) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+            getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    /**
+     * Truncate the file in the indicated path to the indicated size.
+     * <ul>
+     *   <li>Fails if path is a directory.</li>
+     *   <li>Fails if path does not exist.</li>
+     *   <li>Fails if path is not closed.</li>
+     *   <li>Fails if new size is greater than current size.</li>
+     * </ul>
+     * @param f The path to the file to be truncated
+     * @param newLength The size the file is to be truncated to
+     *
+     * @return <code>true</code> if the file has been truncated to the desired
+     * <code>newLength</code> and is immediately available to be reused for
+     * write operations such as <code>append</code>, or
+     * <code>false</code> if a background process of adjusting the length of
+     * the last block has been started, and clients should wait for it to
+     * complete before proceeding with further file updates.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default).
+     */
+    @Override
+    public boolean truncate(Path f, long newLength) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+            getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    @Override
+    public void createSymlink(final Path target, final Path link,
+                              final boolean createParent) throws AccessControlException,
+        FileAlreadyExistsException, FileNotFoundException,
+        ParentNotDirectoryException, UnsupportedFileSystemException,
+        IOException {
+        // Supporting filesystems should override this method
+        throw new UnsupportedOperationException(
+            "Filesystem does not support symlinks!");
+    }
+
+    public boolean supportsSymlinks() {
+        return false;
+    }
+
+    /**
+     * Create a snapshot.
+     * @param path The directory where snapshots will be taken.
+     * @param snapshotName The name of the snapshot
+     * @return the snapshot path.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     */
+    @Override
+    public Path createSnapshot(Path path, String snapshotName)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support createSnapshot");
+    }
+
+    /**
+     * Rename a snapshot.
+     * @param path The directory path where the snapshot was taken
+     * @param snapshotOldName Old name of the snapshot
+     * @param snapshotNewName New name of the snapshot
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void renameSnapshot(Path path, String snapshotOldName,
+                               String snapshotNewName) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support renameSnapshot");
+    }
+
+    /**
+     * Delete a snapshot of a directory.
+     * @param path  The directory that the to-be-deleted snapshot belongs to
+     * @param snapshotName The name of the snapshot
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void deleteSnapshot(Path path, String snapshotName)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support deleteSnapshot");
+    }
+
+    /**
+     * Modifies ACL entries of files and directories.  This method can add new ACL
+     * entries or modify the permissions on existing ACL entries.  All existing
+     * ACL entries that are not specified in this call are retained without
+     * changes.  (Modifications are merged into the current ACL.)
+     *
+     * @param path Path to modify
+     * @param aclSpec List&lt;AclEntry&gt; describing modifications
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support modifyAclEntries");
+    }
+
+    /**
+     * Removes ACL entries from files and directories.  Other ACL entries are
+     * retained.
+     *
+     * @param path Path to modify
+     * @param aclSpec List describing entries to remove
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeAclEntries(Path path, List<AclEntry> aclSpec)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeAclEntries");
+    }
+
+    /**
+     * Removes all default ACL entries from files and directories.
+     *
+     * @param path Path to modify
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeDefaultAcl(Path path)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeDefaultAcl");
+    }
+
+    /**
+     * Removes all but the base ACL entries of files and directories.  The entries
+     * for user, group, and others are retained for compatibility with permission
+     * bits.
+     *
+     * @param path Path to modify
+     * @throws IOException if an ACL could not be removed
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeAcl(Path path)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeAcl");
+    }
+
+    /**
+     * Fully replaces ACL of files and directories, discarding all existing
+     * entries.
+     *
+     * @param path Path to modify
+     * @param aclSpec List describing modifications, which must include entries
+     *   for user, group, and others for compatibility with permission bits.
+     * @throws IOException if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support setAcl");
+    }
+
+    /**
+     * Gets the ACL of a file or directory.
+     *
+     * @param path Path to get
+     * @return AclStatus describing the ACL of the file or directory
+     * @throws IOException if an ACL could not be read
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public AclStatus getAclStatus(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getAclStatus");
+    }
+
+    /**
+     * Set an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to modify
+     * @param name xattr name.
+     * @param value xattr value.
+     * @param flag xattr set flag
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void setXAttr(Path path, String name, byte[] value,
+                         EnumSet<XAttrSetFlag> flag) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support setXAttr");
+    }
+
+    /**
+     * Get an xattr name and value for a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attribute
+     * @param name xattr name.
+     * @return byte[] xattr value.
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public byte[] getXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getXAttr");
+    }
+
+    /**
+     * Get all of the xattr name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattrs name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @param names XAttr names.
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path, List<String> names)
+        throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattr names for a file or directory.
+     * Only those xattr names which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return List{@literal <String>} of the XAttr names of the file or directory
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public List<String> listXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support listXAttrs");
+    }
+
+    /**
+     * Remove an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to remove extended attribute
+     * @param name xattr name
+     * @throws IOException IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *         (default outcome).
+     */
+    @Override
+    public void removeXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+            + " doesn't support removeXAttr");
+    }
+
 }

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -1,15 +1,20 @@
 package seaweed.hdfs;
 
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerClient;
 import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedRead;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -20,9 +25,24 @@ public class SeaweedFileSystemStore {
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
 
     private FilerGrpcClient filerGrpcClient;
+    private FilerClient filerClient;
 
     public SeaweedFileSystemStore(String host, int port) {
-        filerGrpcClient = new FilerGrpcClient(host, port);
+        int grpcPort = 10000 + port;
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+        filerClient = new FilerClient(filerGrpcClient);
+    }
+
+    public static String getParentDirectory(Path path) {
+        return path.isRoot() ? "/" : path.getParent().toUri().getPath();
+    }
+
+    static int permissionToMode(FsPermission permission, boolean isDirectory) {
+        int p = permission.toShort();
+        if (isDirectory) {
+            p = p | 1 << 31;
+        }
+        return p;
     }
 
     public boolean createDirectory(final Path path, UserGroupInformation currentUser,
@@ -33,23 +53,12 @@ public class SeaweedFileSystemStore {
             permission,
             umask);
 
-        long now = System.currentTimeMillis() / 1000L;
-
-        FilerProto.CreateEntryRequest.Builder request = FilerProto.CreateEntryRequest.newBuilder()
-            .setDirectory(path.getParent().toUri().getPath())
-            .setEntry(FilerProto.Entry.newBuilder()
-                .setName(path.getName())
-                .setIsDirectory(true)
-                .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                    .setMtime(now)
-                    .setCrtime(now)
-                    .setFileMode(permission.toShort())
-                    .setUserName(currentUser.getUserName())
-                    .addAllGroupName(Arrays.asList(currentUser.getGroupNames())))
-            );
-
-        FilerProto.CreateEntryResponse response = filerGrpcClient.getBlockingStub().createEntry(request.build());
-        return true;
+        return filerClient.mkdirs(
+            path.toUri().getPath(),
+            permissionToMode(permission, true),
+            currentUser.getUserName(),
+            currentUser.getGroupNames()
+        );
     }
 
     public FileStatus[] listEntries(final Path path) {
@@ -57,57 +66,56 @@ public class SeaweedFileSystemStore {
 
         List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
 
-        List<FilerProto.Entry> entries = lookupEntries(path);
+        List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
 
         for (FilerProto.Entry entry : entries) {
 
-            FileStatus fileStatus = getFileStatus(new Path(path, entry.getName()), entry);
+            FileStatus fileStatus = doGetFileStatus(new Path(path, entry.getName()), entry);
 
             fileStatuses.add(fileStatus);
         }
         return fileStatuses.toArray(new FileStatus[0]);
     }
 
-    private List<FilerProto.Entry> lookupEntries(Path path) {
-        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
-            .setDirectory(path.toUri().getPath())
-            .setLimit(100000)
-            .build()).getEntriesList();
-    }
-
     public FileStatus getFileStatus(final Path path) {
-        LOG.debug("getFileStatus path: {}", path);
 
         FilerProto.Entry entry = lookupEntry(path);
-        FileStatus fileStatus = getFileStatus(path, entry);
+        if (entry == null) {
+            return null;
+        }
+        LOG.debug("doGetFileStatus path:{} entry:{}", path, entry);
+
+        FileStatus fileStatus = doGetFileStatus(path, entry);
         return fileStatus;
     }
 
-    public boolean deleteEntries(final Path path, boolean isDirectroy, boolean recursive) {
+    public boolean deleteEntries(final Path path, boolean isDirectory, boolean recursive) {
         LOG.debug("deleteEntries path: {} isDirectory {} recursive: {}",
             path,
-            String.valueOf(isDirectroy),
+            String.valueOf(isDirectory),
             String.valueOf(recursive));
 
-        FilerProto.DeleteEntryResponse response =
-            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
-                .setDirectory(path.getParent().toUri().getPath())
-                .setName(path.getName())
-                .setIsDirectory(isDirectroy)
-                .setIsDeleteData(true)
-                .build());
+        if (path.isRoot()) {
+            return true;
+        }
 
-        return true;
+        if (recursive && isDirectory) {
+            List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
+            for (FilerProto.Entry entry : entries) {
+                deleteEntries(new Path(path, entry.getName()), entry.getIsDirectory(), true);
+            }
+        }
+
+        return filerClient.deleteEntry(getParentDirectory(path), path.getName(), true, recursive);
     }
 
-
-    private FileStatus getFileStatus(Path path, FilerProto.Entry entry) {
+    private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) {
         FilerProto.FuseAttributes attributes = entry.getAttributes();
-        long length = attributes.getFileSize();
+        long length = SeaweedRead.totalSize(entry.getChunksList());
         boolean isDir = entry.getIsDirectory();
         int block_replication = 1;
         int blocksize = 512;
-        long modification_time = attributes.getMtime();
+        long modification_time = attributes.getMtime() * 1000; // milliseconds
         long access_time = 0;
         FsPermission permission = FsPermission.createImmutable((short) attributes.getFileMode());
         String owner = attributes.getUserName();
@@ -116,26 +124,43 @@ public class SeaweedFileSystemStore {
             modification_time, access_time, permission, owner, group, null, path);
     }
 
-
     private FilerProto.Entry lookupEntry(Path path) {
-        FilerProto.LookupDirectoryEntryResponse response =
-            filerGrpcClient.getBlockingStub().lookupDirectoryEntry(FilerProto.LookupDirectoryEntryRequest.newBuilder()
-                .setDirectory(path.getParent().toUri().getPath())
-                .setName(path.getName())
-                .build());
 
-        return response.getEntry();
+        return filerClient.lookupEntry(getParentDirectory(path), path.getName());
+
     }
 
     public void rename(Path source, Path destination) {
+
+        LOG.debug("rename source: {} destination:{}", source, destination);
+
+        if (source.isRoot()) {
+            return;
+        }
+        LOG.warn("rename lookupEntry source: {}", source);
         FilerProto.Entry entry = lookupEntry(source);
+        if (entry == null) {
+            LOG.warn("rename non-existing source: {}", source);
+            return;
+        }
+        LOG.warn("rename moveEntry source: {}", source);
         moveEntry(source.getParent(), entry, destination);
     }
 
     private boolean moveEntry(Path oldParent, FilerProto.Entry entry, Path destination) {
+
+        LOG.debug("moveEntry: {}/{}  => {}", oldParent, entry.getName(), destination);
+
+        FilerProto.Entry.Builder newEntry = entry.toBuilder().setName(destination.getName());
+        boolean isDirectoryCreated = filerClient.createEntry(getParentDirectory(destination), newEntry.build());
+
+        if (!isDirectoryCreated) {
+            return false;
+        }
+
         if (entry.getIsDirectory()) {
             Path entryPath = new Path(oldParent, entry.getName());
-            List<FilerProto.Entry> entries = lookupEntries(entryPath);
+            List<FilerProto.Entry> entries = filerClient.listEntries(entryPath.toUri().getPath());
             for (FilerProto.Entry ent : entries) {
                 boolean isSucess = moveEntry(entryPath, ent, new Path(destination, ent.getName()));
                 if (!isSucess) {
@@ -144,23 +169,15 @@ public class SeaweedFileSystemStore {
             }
         }
 
-        filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
-            .setDirectory(destination.getParent().toUri().getPath())
-            .build());
+        return filerClient.deleteEntry(
+            oldParent.toUri().getPath(), entry.getName(), false, false);
 
-        filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
-            .setDirectory(oldParent.toUri().getPath())
-            .setName(entry.getName())
-            .setIsDirectory(entry.getIsDirectory())
-            .setIsDeleteData(false)
-            .build());
-
-        return true;
     }
 
     public OutputStream createFile(final Path path,
                                    final boolean overwrite,
                                    FsPermission permission,
+                                   int bufferSize,
                                    String replication) throws IOException {
 
         permission = permission == null ? FsPermission.getFileDefault() : permission;
@@ -171,29 +188,110 @@ public class SeaweedFileSystemStore {
             permission.toString());
 
         UserGroupInformation userGroupInformation = UserGroupInformation.getCurrentUser();
+        long now = System.currentTimeMillis() / 1000L;
 
-        FilerProto.Entry.Builder entry = FilerProto.Entry.newBuilder();
+        FilerProto.Entry.Builder entry = null;
         long writePosition = 0;
         if (!overwrite) {
             FilerProto.Entry existingEntry = lookupEntry(path);
+            LOG.debug("createFile merged entry path:{} existingEntry:{}", path, existingEntry);
             if (existingEntry != null) {
+                entry = FilerProto.Entry.newBuilder();
                 entry.mergeFrom(existingEntry);
+                entry.getAttributesBuilder().setMtime(now);
             }
-            writePosition = existingEntry.getAttributes().getFileSize();
+            LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
+            writePosition = SeaweedRead.totalSize(existingEntry.getChunksList());
             replication = existingEntry.getAttributes().getReplication();
         }
         if (entry == null) {
             entry = FilerProto.Entry.newBuilder()
+                .setName(path.getName())
+                .setIsDirectory(false)
                 .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                    .setFileMode(permission.toOctal())
+                    .setFileMode(permissionToMode(permission, false))
                     .setReplication(replication)
-                    .setCrtime(System.currentTimeMillis() / 1000L)
+                    .setCrtime(now)
+                    .setMtime(now)
                     .setUserName(userGroupInformation.getUserName())
+                    .clearGroupName()
                     .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames()))
                 );
         }
 
-        return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, 16 * 1024 * 1024, replication);
+        return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication);
+
+    }
+
+    public InputStream openFileForRead(final Path path, FileSystem.Statistics statistics,
+                                       int bufferSize) throws IOException {
+
+        LOG.debug("openFileForRead path:{} bufferSize:{}", path, bufferSize);
+
+        int readAheadQueueDepth = 2;
+        FilerProto.Entry entry = lookupEntry(path);
+
+        if (entry == null) {
+            throw new FileNotFoundException("read non-exist file " + path);
+        }
+
+        return new SeaweedInputStream(filerGrpcClient,
+            statistics,
+            path.toUri().getPath(),
+            entry,
+            bufferSize,
+            readAheadQueueDepth);
+    }
+
+    public void setOwner(Path path, String owner, String group) {
+
+        LOG.debug("setOwner path:{} owner:{} group:{}", path, owner, group);
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            LOG.debug("setOwner path:{} entry:{}", path, entry);
+            return;
+        }
+
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
+
+        if (owner != null) {
+            attributesBuilder.setUserName(owner);
+        }
+        if (group != null) {
+            attributesBuilder.clearGroupName();
+            attributesBuilder.addGroupName(group);
+        }
+
+        entryBuilder.setAttributes(attributesBuilder);
+
+        LOG.debug("setOwner path:{} entry:{}", path, entryBuilder);
+
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
+
+    }
+
+    public void setPermission(Path path, FsPermission permission) {
+
+        LOG.debug("setPermission path:{} permission:{}", path, permission);
+
+        FilerProto.Entry entry = lookupEntry(path);
+        if (entry == null) {
+            LOG.debug("setPermission path:{} entry:{}", path, entry);
+            return;
+        }
+
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        FilerProto.FuseAttributes.Builder attributesBuilder = entry.getAttributes().toBuilder();
+
+        attributesBuilder.setFileMode(permissionToMode(permission, entry.getIsDirectory()));
+
+        entryBuilder.setAttributes(attributesBuilder);
+
+        LOG.debug("setPermission path:{} entry:{}", path, entryBuilder);
+
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
 
     }
 }

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedInputStream.java

@@ -6,8 +6,11 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedRead;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -15,6 +18,8 @@ import java.util.List;
 
 public class SeaweedInputStream extends FSInputStream {
 
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedInputStream.class);
+
     private final FilerGrpcClient filerGrpcClient;
     private final Statistics statistics;
     private final String path;
@@ -45,12 +50,15 @@ public class SeaweedInputStream extends FSInputStream {
         this.statistics = statistics;
         this.path = path;
         this.entry = entry;
-        this.contentLength = entry.getAttributes().getFileSize();
+        this.contentLength = SeaweedRead.totalSize(entry.getChunksList());
         this.bufferSize = bufferSize;
         this.readAheadQueueDepth = (readAheadQueueDepth >= 0) ? readAheadQueueDepth : Runtime.getRuntime().availableProcessors();
         this.readAheadEnabled = true;
 
         this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(entry.getChunksList());
+
+        LOG.debug("new path:{} entry:{} visibleIntervalList:{}", path, entry, visibleIntervalList);
+
     }
 
     public String getPath() {

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedOutputStream.java

@@ -7,8 +7,11 @@ import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.Syncable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedWrite;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
@@ -22,8 +25,12 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory;
+
 public class SeaweedOutputStream extends OutputStream implements Syncable, StreamCapabilities {
 
+    private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class);
+
     private final FilerGrpcClient filerGrpcClient;
     private final Path path;
     private final int bufferSize;
@@ -71,8 +78,11 @@ public class SeaweedOutputStream extends OutputStream implements Syncable, Strea
     }
 
     private synchronized void flushWrittenBytesToServiceInternal(final long offset) throws IOException {
+
+        LOG.debug("SeaweedWrite.writeMeta path: {} entry:{}", path, entry);
+
         try {
-            SeaweedWrite.writeMeta(filerGrpcClient, path, entry);
+            SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry);
         } catch (Exception ex) {
             throw new IOException(ex);
         }
@@ -186,6 +196,7 @@ public class SeaweedOutputStream extends OutputStream implements Syncable, Strea
             return;
         }
 
+        LOG.debug("close path: {}", path);
         try {
             flushInternal();
             threadExecutor.shutdown();
@@ -293,6 +304,7 @@ public class SeaweedOutputStream extends OutputStream implements Syncable, Strea
                 throw lastError;
             }
         }
+        LOG.debug("flushWrittenBytesToService: {} position:{}", path, position);
         flushWrittenBytesToServiceInternal(position);
     }
 

unmaintained/fix_dat/fix_dat.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var (
@@ -63,7 +63,7 @@ func main() {
 
 	iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) {
 		fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
-		s, _, e := n.Append(newDatFile, superBlock.Version())
+		_, s, _, e := n.Append(newDatFile, superBlock.Version())
 		fmt.Printf("size %d error %v\n", s, e)
 	})
 
@@ -77,13 +77,13 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 	readerOffset += int64(count)
 
 	// start to read dat file
-	superblock, err := storage.ReadSuperBlock(datFile)
+	superBlock, err := storage.ReadSuperBlock(datFile)
 	if err != nil {
 		fmt.Printf("cannot read dat file super block: %v", err)
 		return
 	}
-	offset := int64(superblock.BlockSize())
-	version := superblock.Version()
+	offset := int64(superBlock.BlockSize())
+	version := superBlock.Version()
 	n, rest, err := storage.ReadNeedleHeader(datFile, version, offset)
 	if err != nil {
 		fmt.Printf("cannot read needle header: %v", err)

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -29,7 +29,7 @@ func main() {
 		rand.Read(data)
 		reader := bytes.NewReader(data)
 
-		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url,assignResult.Fid)
+		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
 
 		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, "")
 		if err != nil {

unmaintained/see_dat/see_dat.go

@@ -12,21 +12,31 @@ var (
 	volumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 )
 
+type VolumeFileScanner4SeeDat struct {
+	version storage.Version
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
+	return false
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *storage.Needle, offset int64) error {
+	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
+	return nil
+}
+
 func main() {
 	flag.Parse()
 
-	var version storage.Version
 	vid := storage.VolumeId(*volumeId)
-	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, false, func(n *storage.Needle, offset int64) error {
-			glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x",
-				*volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
-			return nil
-		})
+
+	scanner := &VolumeFileScanner4SeeDat{}
+	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
 	if err != nil {
 		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
 	}

weed/command/backup.go

@@ -38,7 +38,7 @@ var cmdBackup = &Command{
 	This will help to backup future new volumes.
 	
 	Usually backing up is just copying the .dat (and .idx) files.
-	But it's tricky to incremententally copy the differences.
+	But it's tricky to incrementally copy the differences.
 	
 	The complexity comes when there are multiple addition, deletion and compaction.
 	This tool will handle them correctly and efficiently, avoiding unnecessary data transporation.

weed/command/benchmark.go

@@ -45,7 +45,7 @@ var (
 )
 
 func init() {
-	cmdBenchmark.Run = runbenchmark // break init cycle
+	cmdBenchmark.Run = runBenchmark // break init cycle
 	cmdBenchmark.IsDebug = cmdBenchmark.Flag.Bool("debug", false, "verbose debug information")
 	b.masters = cmdBenchmark.Flag.String("master", "localhost:9333", "SeaweedFS master location")
 	b.concurrency = cmdBenchmark.Flag.Int("c", 16, "number of concurrent write or read processes")
@@ -101,7 +101,7 @@ var (
 	readStats  *stats
 )
 
-func runbenchmark(cmd *Command, args []string) bool {
+func runBenchmark(cmd *Command, args []string) bool {
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
 	if *b.maxCpu < 1 {
 		*b.maxCpu = runtime.NumCPU()
@@ -121,17 +121,17 @@ func runbenchmark(cmd *Command, args []string) bool {
 	masterClient.WaitUntilConnected()
 
 	if *b.write {
-		bench_write()
+		benchWrite()
 	}
 
 	if *b.read {
-		bench_read()
+		benchRead()
 	}
 
 	return true
 }
 
-func bench_write() {
+func benchWrite() {
 	fileIdLineChan := make(chan string)
 	finishChan := make(chan bool)
 	writeStats = newStats(*b.concurrency)
@@ -158,7 +158,7 @@ func bench_write() {
 	writeStats.printStats()
 }
 
-func bench_read() {
+func benchRead() {
 	fileIdLineChan := make(chan string)
 	finishChan := make(chan bool)
 	readStats = newStats(*b.concurrency)
@@ -213,7 +213,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 	for id := range idChan {
 		start := time.Now()
 		fileSize := int64(*b.fileSize + random.Intn(64))
-		fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
+		fp := &operation.FilePart{
+			Reader:   &FakeReader{id: uint64(id), size: fileSize},
+			FileSize: fileSize,
+			MimeType: "image/bench", // prevent gzip benchmark content
+		}
 		ar := &operation.VolumeAssignRequest{
 			Count:      1,
 			Collection: *b.collection,

weed/command/export.go

@@ -84,6 +84,62 @@ func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Versio
 	)
 }
 
+type VolumeFileScanner4Export struct {
+	version   storage.Version
+	counter   int
+	needleMap *storage.NeedleMap
+	vid       storage.VolumeId
+}
+
+func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
+	return true
+}
+
+func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset int64) error {
+	needleMap := scanner.needleMap
+	vid := scanner.vid
+
+	nv, ok := needleMap.Get(n.Id)
+	glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
+		n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
+	if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
+		if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
+			glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+				n.LastModified, newerThanUnix)
+			return nil
+		}
+		scanner.counter++
+		if *limit > 0 && scanner.counter > *limit {
+			return io.EOF
+		}
+		if tarOutputFile != nil {
+			return writeFile(vid, n)
+		} else {
+			printNeedle(vid, n, scanner.version, false)
+			return nil
+		}
+	}
+	if !ok {
+		if *showDeleted && tarOutputFile == nil {
+			if n.DataSize > 0 {
+				printNeedle(vid, n, scanner.version, true)
+			} else {
+				n.Name = []byte("*tombstone")
+				printNeedle(vid, n, scanner.version, true)
+			}
+		}
+		glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
+	} else {
+		glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
+	}
+	return nil
+}
+
 func runExport(cmd *Command, args []string) bool {
 
 	var err error
@@ -145,55 +201,16 @@ func runExport(cmd *Command, args []string) bool {
 		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
 	}
 
-	var version storage.Version
+	volumeFileScanner := &VolumeFileScanner4Export{
+		needleMap: needleMap,
+		vid:       vid,
+	}
 
 	if tarOutputFile == nil {
 		fmt.Printf("key\tname\tsize\tgzip\tmime\tmodified\tttl\tdeleted\n")
 	}
 
-	var counter = 0
-
-	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, true, func(n *storage.Needle, offset int64) error {
-			nv, ok := needleMap.Get(n.Id)
-			glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
-				n.Id, offset, n.Size, n.DiskSize(version), n.IsGzipped(), ok, nv)
-			if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
-				if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
-					glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
-						n.LastModified, newerThanUnix)
-					return nil
-				}
-				counter++
-				if *limit > 0 && counter > *limit {
-					return io.EOF
-				}
-				if tarOutputFile != nil {
-					return writeFile(vid, n)
-				} else {
-					printNeedle(vid, n, version, false)
-					return nil
-				}
-			}
-			if !ok {
-				if *showDeleted && tarOutputFile == nil {
-					if n.DataSize > 0 {
-						printNeedle(vid, n, version, true)
-					} else {
-						n.Name = []byte("*tombstone")
-						printNeedle(vid, n, version, true)
-					}
-				}
-				glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
-			} else {
-				glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
-			}
-			return nil
-		})
+	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
 	if err != nil && err != io.EOF {
 		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
 	}

weed/command/filer.go

@@ -32,6 +32,9 @@ type FilerOptions struct {
 	dirListingLimit         *int
 	dataCenter              *string
 	enableNotification      *bool
+
+	// default leveldb directory, used in "weed server" mode
+	defaultLevelDbDirectory *string
 }
 
 func init() {
@@ -88,6 +91,11 @@ func (fo *FilerOptions) startFiler() {
 		publicVolumeMux = http.NewServeMux()
 	}
 
+	defaultLevelDbDirectory := "./filerdb"
+	if fo.defaultLevelDbDirectory != nil {
+		defaultLevelDbDirectory = *fo.defaultLevelDbDirectory + "/filerdb"
+	}
+
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
 		Masters:            strings.Split(*f.masters, ","),
 		Collection:         *fo.collection,
@@ -98,6 +106,7 @@ func (fo *FilerOptions) startFiler() {
 		SecretKey:          *fo.secretKey,
 		DirListingLimit:    *fo.dirListingLimit,
 		DataCenter:         *fo.dataCenter,
+		DefaultLevelDbDir:  defaultLevelDbDirectory,
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)
@@ -117,7 +126,7 @@ func (fo *FilerOptions) startFiler() {
 		}()
 	}
 
-	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*fo.port))
+	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.VERSION, *fo.ip, *fo.port)
 	filerListener, e := util.NewListener(
 		":"+strconv.Itoa(*fo.port),
 		time.Duration(10)*time.Second,

weed/command/filer_copy.go

@@ -15,7 +15,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"io"
 	"net/http"
-	"path"
 	"strconv"
 	"time"
 )
@@ -61,10 +60,6 @@ var cmdCopy = &Command{
   All files under the folder and subfolders will be copyed.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
   If "maxMB" is set to a positive number, files larger than it would be split into chunks.
 
   `,
@@ -85,7 +80,8 @@ func runCopy(cmd *Command, args []string) bool {
 	}
 	urlPath := filerUrl.Path
 	if !strings.HasSuffix(urlPath, "/") {
-		urlPath = urlPath + "/"
+		fmt.Printf("The last argument should be a folder and end with \"/\": %v\n", err)
+		return false
 	}
 
 	if filerUrl.Port() == "" {
@@ -165,7 +161,6 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 	// upload the file content
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
-	isGzipped := isGzipped(fileName)
 
 	var chunks []*filer_pb.FileChunk
 
@@ -184,7 +179,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
 
-		uploadResult, err := operation.Upload(targetUrl, fileName, f, isGzipped, mimeType, nil, "")
+		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, "")
 		if err != nil {
 			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 			return false
@@ -318,13 +313,9 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 	return true
 }
 
-func isGzipped(filename string) bool {
-	return strings.ToLower(path.Ext(filename)) == ".gz"
-}
-
 func detectMimeType(f *os.File) string {
 	head := make([]byte, 512)
-	f.Seek(0, 0)
+	f.Seek(0, io.SeekStart)
 	n, err := f.Read(head)
 	if err == io.EOF {
 		return ""
@@ -333,7 +324,7 @@ func detectMimeType(f *os.File) string {
 		fmt.Printf("read head of %v: %v\n", f.Name(), err)
 		return "application/octet-stream"
 	}
-	f.Seek(0, 0)
+	f.Seek(0, io.SeekStart)
 	mimeType := http.DetectContentType(head[:n])
 	return mimeType
 }

weed/command/filer_export.go

@@ -14,7 +14,7 @@ func init() {
 }
 
 var cmdFilerExport = &Command{
-	UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
+	UsageLine: "filer.export -sourceStore=mysql -targetStore=cassandra",
 	Short:     "export meta data in filer store",
 	Long: `Iterate the file tree and export all metadata out
 

weed/command/filer_replication.go

@@ -41,6 +41,8 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 
 	var notificationInput sub.NotificationInput
 
+	validateOneEnabledInput(config)
+
 	for _, input := range sub.NotificationInputs {
 		if config.GetBool("notification." + input.GetName() + ".enabled") {
 			viperSub := config.Sub("notification." + input.GetName())
@@ -122,3 +124,16 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 
 	return true
 }
+
+func validateOneEnabledInput(config *viper.Viper) {
+	enabledInput := ""
+	for _, input := range sub.NotificationInputs {
+		if config.GetBool("notification." + input.GetName() + ".enabled") {
+			if enabledInput == "" {
+				enabledInput = input.GetName()
+			} else {
+				glog.Fatalf("Notification input is enabled for both %s and %s", enabledInput, input.GetName())
+			}
+		}
+	}
+}

weed/command/fix.go

@@ -28,6 +28,32 @@ var (
 	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 )
 
+type VolumeFileScanner4Fix struct {
+	version storage.Version
+	nm      *storage.NeedleMap
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
+	return false
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
+	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
+	if n.Size > 0 {
+		pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
+		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
+	} else {
+		glog.V(2).Infof("skipping deleted file ...")
+		return scanner.nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
+	}
+	return nil
+}
+
 func runFix(cmd *Command, args []string) bool {
 
 	if *fixVolumeId == -1 {
@@ -48,24 +74,12 @@ func runFix(cmd *Command, args []string) bool {
 	nm := storage.NewBtreeNeedleMap(indexFile)
 	defer nm.Close()
 
-	var version storage.Version
 	vid := storage.VolumeId(*fixVolumeId)
-	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, false, func(n *storage.Needle, offset int64) error {
-			glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(version), n.IsGzipped())
-			if n.Size > 0 {
-				pe := nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
-				glog.V(2).Infof("saved %d with error %v", n.Size, pe)
-			} else {
-				glog.V(2).Infof("skipping deleted file ...")
-				return nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
-			}
-			return nil
-		})
+	scanner := &VolumeFileScanner4Fix{
+		nm: nm,
+	}
+
+	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner)
 	if err != nil {
 		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
 		os.Remove(indexFileName)

weed/command/master.go

@@ -13,7 +13,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
 	"google.golang.org/grpc/reflection"
 )
 
@@ -32,6 +31,7 @@ var cmdMaster = &Command{
 
 var (
 	mport                   = cmdMaster.Flag.Int("port", 9333, "http listen port")
+	mGrpcPort               = cmdMaster.Flag.Int("port.grpc", 0, "grpc server listen port, default to http port + 10000")
 	masterIp                = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
 	masterBindIp            = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
 	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
@@ -79,7 +79,7 @@ func runMaster(cmd *Command, args []string) bool {
 
 	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
 
-	listener, e := util.NewListener(listeningAddress, 0)
+	masterListener, e := util.NewListener(listeningAddress, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
@@ -91,23 +91,28 @@ func runMaster(cmd *Command, args []string) bool {
 		ms.SetRaftServer(raftServer)
 	}()
 
-	// start grpc and http server
-	m := cmux.New(listener)
+	go func() {
+		// starting grpc server
+		grpcPort := *mGrpcPort
+		if grpcPort == 0 {
+			grpcPort = *mport + 10000
+		}
+		grpcL, err := util.NewListener(*masterBindIp+":"+strconv.Itoa(grpcPort), 0)
+		if err != nil {
+			glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+		}
+		// Create your protocol servers.
+		grpcS := util.NewGrpcServer()
+		master_pb.RegisterSeaweedServer(grpcS, ms)
+		reflection.Register(grpcS)
 
-	grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-	httpL := m.Match(cmux.Any())
-
-	// Create your protocol servers.
-	grpcS := util.NewGrpcServer()
-	master_pb.RegisterSeaweedServer(grpcS, ms)
-	reflection.Register(grpcS)
+		glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *masterBindIp, grpcPort)
+		grpcS.Serve(grpcL)
+	}()
 
+	// start http server
 	httpS := &http.Server{Handler: r}
-
-	go grpcS.Serve(grpcL)
-	go httpS.Serve(httpL)
-
-	if err := m.Serve(); err != nil {
+	if err := httpS.Serve(masterListener); err != nil {
 		glog.Fatalf("master server failed to serve: %v", err)
 	}
 

weed/command/mount.go

@@ -35,7 +35,7 @@ func init() {
 	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
 	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
-	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 16, "local write buffer size, also chunk large files")
+	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
@@ -51,7 +51,7 @@ var cmdMount = &Command{
   2) have a "weed filer" running
   These 2 requirements can be achieved with one command "weed server -filer=true"
 
-  This uses bazil.org/fuse, which enables writing FUSE file systems on
+  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
   Linux, and OS X.
 
   On OS X, it requires OSXFUSE (http://osxfuse.github.com/).

weed/command/mount_std.go

@@ -4,15 +4,18 @@ package command
 
 import (
 	"fmt"
+	"os"
+	"os/user"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 func runMount(cmd *Command, args []string) bool {
@@ -28,6 +31,23 @@ func runMount(cmd *Command, args []string) bool {
 
 	fuse.Unmount(*mountOptions.dir)
 
+	// detect mount folder mode
+	mountMode := os.ModeDir | 0755
+	if fileInfo, err := os.Stat(*mountOptions.dir); err == nil {
+		mountMode = os.ModeDir | fileInfo.Mode()
+	}
+
+	// detect current user
+	uid, gid := uint32(0), uint32(0)
+	if u, err := user.Current(); err == nil {
+		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+			uid = uint32(parsedId)
+		}
+		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+			gid = uint32(parsedId)
+		}
+	}
+
 	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
 
 	c, err := fuse.Mount(
@@ -37,6 +57,8 @@ func runMount(cmd *Command, args []string) bool {
 		fuse.Subtype("SeaweedFS"),
 		fuse.NoAppleDouble(),
 		fuse.NoAppleXattr(),
+		fuse.NoBrowse(),
+		fuse.AutoXattr(),
 		fuse.ExclCreate(),
 		fuse.DaemonTimeout("3600"),
 		fuse.AllowOther(),
@@ -77,6 +99,9 @@ func runMount(cmd *Command, args []string) bool {
 		DataCenter:         *mountOptions.dataCenter,
 		DirListingLimit:    *mountOptions.dirListingLimit,
 		EntryCacheTtl:      3 * time.Second,
+		MountUid:           uid,
+		MountGid:           gid,
+		MountMode:          mountMode,
 	}))
 	if err != nil {
 		fuse.Unmount(*mountOptions.dir)

weed/command/scaffold.go

@@ -191,12 +191,18 @@ topic = "seaweedfs_filer_topic"       # a topic, auto created if does not exists
 [source.filer]
 enabled = true
 grpcAddress = "localhost:18888"
-directory = "/buckets"    # all files under this directory tree are replicated
+# all files under this directory tree are replicated.
+# this is not a directory on your hard drive, but on your filer.
+# i.e., all files with this "prefix" are sent to notification message queue.
+directory = "/buckets"    
 
 [sink.filer]
 enabled = false
 grpcAddress = "localhost:18888"
-directory = "/backup"    # all replicated files are under this directory tree
+# all replicated files are under this directory tree
+# this is not a directory on your hard drive, but on your filer.     
+# i.e., all received files will be "prefixed" to this directory.
+directory = "/backup"    
 replication = ""
 collection = ""
 ttlSec = 0

weed/command/server.go

@@ -15,7 +15,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
 	"google.golang.org/grpc/reflection"
 )
 
@@ -62,6 +61,7 @@ var (
 	serverSecureKey               = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	serverGarbageThreshold        = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
+	masterGrpcPort                = cmdServer.Flag.Int("master.port.grpc", 0, "master grpc server listen port, default to http port + 10000")
 	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	masterVolumePreallocate       = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
@@ -144,6 +144,7 @@ func runServer(cmd *Command, args []string) bool {
 	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
 		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
 	}
+	filerOptions.defaultLevelDbDirectory = masterMetaFolder
 
 	if *serverWhiteListOption != "" {
 		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
@@ -172,12 +173,31 @@ func runServer(cmd *Command, args []string) bool {
 			serverWhiteList, *serverSecureKey,
 		)
 
-		glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
+		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
 		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
 		if e != nil {
 			glog.Fatalf("Master startup error: %v", e)
 		}
 
+		go func() {
+			// starting grpc server
+			grpcPort := *masterGrpcPort
+			if grpcPort == 0 {
+				grpcPort = *masterPort + 10000
+			}
+			grpcL, err := util.NewListener(*serverIp+":"+strconv.Itoa(grpcPort), 0)
+			if err != nil {
+				glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+			}
+			// Create your protocol servers.
+			grpcS := util.NewGrpcServer()
+			master_pb.RegisterSeaweedServer(grpcS, ms)
+			reflection.Register(grpcS)
+
+			glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort)
+			grpcS.Serve(grpcL)
+		}()
+
 		go func() {
 			raftWaitForMaster.Wait()
 			time.Sleep(100 * time.Millisecond)
@@ -189,23 +209,9 @@ func runServer(cmd *Command, args []string) bool {
 
 		raftWaitForMaster.Done()
 
-		// start grpc and http server
-		m := cmux.New(masterListener)
-
-		grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-		httpL := m.Match(cmux.Any())
-
-		// Create your protocol servers.
-		grpcS := util.NewGrpcServer()
-		master_pb.RegisterSeaweedServer(grpcS, ms)
-		reflection.Register(grpcS)
-
+		// start http server
 		httpS := &http.Server{Handler: r}
-
-		go grpcS.Serve(grpcL)
-		go httpS.Serve(httpL)
-
-		if err := m.Serve(); err != nil {
+		if err := httpS.Serve(masterListener); err != nil {
 			glog.Fatalf("master server failed to serve: %v", err)
 		}
 

weed/command/upload.go

@@ -53,10 +53,6 @@ var cmdUpload = &Command{
   All files under the folder and subfolders will be uploaded, each with its own file key.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
   If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
   The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
 

weed/command/volume.go

@@ -155,7 +155,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
-	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)
+	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.VERSION, listeningAddress)
 	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)

weed/filer2/cassandra/cassandra_store.go

@@ -48,8 +48,8 @@ func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
 	}
 
 	if err := store.session.Query(
-		"INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?)",
-		dir, name, meta).Exec(); err != nil {
+		"INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?) USING TTL ? ",
+		dir, name, meta, entry.TtlSec).Exec(); err != nil {
 		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
 	}
 

weed/filer2/configuration.go

@@ -13,6 +13,8 @@ var (
 
 func (f *Filer) LoadConfiguration(config *viper.Viper) {
 
+	validateOneEnabledStore(config)
+
 	for _, store := range Stores {
 		if config.GetBool(store.GetName() + ".enabled") {
 			viperSub := config.Sub(store.GetName())
@@ -34,3 +36,16 @@ func (f *Filer) LoadConfiguration(config *viper.Viper) {
 
 	os.Exit(-1)
 }
+
+func validateOneEnabledStore(config *viper.Viper) {
+	enabledStore := ""
+	for _, store := range Stores {
+		if config.GetBool(store.GetName() + ".enabled") {
+			if enabledStore == "" {
+				enabledStore = store.GetName()
+			} else {
+				glog.Fatalf("Filer store is enabled for both %s and %s", enabledStore, store.GetName())
+			}
+		}
+	}
+}

weed/filer2/entry.go

@@ -8,15 +8,18 @@ import (
 )
 
 type Attr struct {
-	Mtime       time.Time   // time of last modification
-	Crtime      time.Time   // time of creation (OS X only)
-	Mode        os.FileMode // file mode
-	Uid         uint32      // owner uid
-	Gid         uint32      // group gid
-	Mime        string      // mime type
-	Replication string      // replication
-	Collection  string      // collection name
-	TtlSec      int32       // ttl in seconds
+	Mtime         time.Time   // time of last modification
+	Crtime        time.Time   // time of creation (OS X only)
+	Mode          os.FileMode // file mode
+	Uid           uint32      // owner uid
+	Gid           uint32      // group gid
+	Mime          string      // mime type
+	Replication   string      // replication
+	Collection    string      // collection name
+	TtlSec        int32       // ttl in seconds
+	UserName      string
+	GroupNames    []string
+	SymlinkTarget string
 }
 
 func (attr Attr) IsDirectory() bool {

weed/filer2/entry_codec.go

@@ -35,15 +35,18 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
 func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
 
 	return &filer_pb.FuseAttributes{
-		Crtime:      entry.Attr.Crtime.Unix(),
-		Mtime:       entry.Attr.Mtime.Unix(),
-		FileMode:    uint32(entry.Attr.Mode),
-		Uid:         entry.Uid,
-		Gid:         entry.Gid,
-		Mime:        entry.Mime,
-		Collection:  entry.Attr.Collection,
-		Replication: entry.Attr.Replication,
-		TtlSec:      entry.Attr.TtlSec,
+		Crtime:        entry.Attr.Crtime.Unix(),
+		Mtime:         entry.Attr.Mtime.Unix(),
+		FileMode:      uint32(entry.Attr.Mode),
+		Uid:           entry.Uid,
+		Gid:           entry.Gid,
+		Mime:          entry.Mime,
+		Collection:    entry.Attr.Collection,
+		Replication:   entry.Attr.Replication,
+		TtlSec:        entry.Attr.TtlSec,
+		UserName:      entry.Attr.UserName,
+		GroupName:     entry.Attr.GroupNames,
+		SymlinkTarget: entry.Attr.SymlinkTarget,
 	}
 }
 
@@ -60,6 +63,9 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
 	t.Collection = attr.Collection
 	t.Replication = attr.Replication
 	t.TtlSec = attr.TtlSec
+	t.UserName = attr.UserName
+	t.GroupNames = attr.GroupName
+	t.SymlinkTarget = attr.SymlinkTarget
 
 	return t
 }

weed/filer2/filechunks.go

@@ -33,7 +33,7 @@ func ETag(chunks []*filer_pb.FileChunk) (etag string) {
 
 func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
 
-	visibles := nonOverlappingVisibleIntervals(chunks)
+	visibles := NonOverlappingVisibleIntervals(chunks)
 
 	fileIds := make(map[string]bool)
 	for _, interval := range visibles {
@@ -70,21 +70,30 @@ type ChunkView struct {
 	Offset      int64
 	Size        uint64
 	LogicOffset int64
+	IsFullChunk bool
 }
 
 func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) {
 
-	visibles := nonOverlappingVisibleIntervals(chunks)
+	visibles := NonOverlappingVisibleIntervals(chunks)
+
+	return ViewFromVisibleIntervals(visibles, offset, size)
+
+}
+
+func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int) (views []*ChunkView) {
 
 	stop := offset + int64(size)
 
 	for _, chunk := range visibles {
 		if chunk.start <= offset && offset < chunk.stop && offset < stop {
+			isFullChunk := chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop
 			views = append(views, &ChunkView{
 				FileId:      chunk.fileId,
 				Offset:      offset - chunk.start, // offset is the data starting location in this file id
 				Size:        uint64(min(chunk.stop, stop) - offset),
 				LogicOffset: offset,
+				IsFullChunk: isFullChunk,
 			})
 			offset = min(chunk.stop, stop)
 		}
@@ -94,7 +103,7 @@ func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views
 
 }
 
-func logPrintf(name string, visibles []*visibleInterval) {
+func logPrintf(name string, visibles []VisibleInterval) {
 	/*
 		log.Printf("%s len %d", name, len(visibles))
 		for _, v := range visibles {
@@ -105,17 +114,18 @@ func logPrintf(name string, visibles []*visibleInterval) {
 
 var bufPool = sync.Pool{
 	New: func() interface{} {
-		return new(visibleInterval)
+		return new(VisibleInterval)
 	},
 }
 
-func mergeIntoVisibles(visibles, newVisibles []*visibleInterval, chunk *filer_pb.FileChunk) []*visibleInterval {
+func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
 
 	newV := newVisibleInterval(
 		chunk.Offset,
 		chunk.Offset+int64(chunk.Size),
 		chunk.FileId,
 		chunk.Mtime,
+		true,
 	)
 
 	length := len(visibles)
@@ -135,6 +145,7 @@ func mergeIntoVisibles(visibles, newVisibles []*visibleInterval, chunk *filer_pb
 				chunk.Offset,
 				v.fileId,
 				v.modifiedTime,
+				false,
 			))
 		}
 		chunkStop := chunk.Offset + int64(chunk.Size)
@@ -144,6 +155,7 @@ func mergeIntoVisibles(visibles, newVisibles []*visibleInterval, chunk *filer_pb
 				v.stop,
 				v.fileId,
 				v.modifiedTime,
+				false,
 			))
 		}
 		if chunkStop <= v.start || v.stop <= chunk.Offset {
@@ -167,18 +179,18 @@ func mergeIntoVisibles(visibles, newVisibles []*visibleInterval, chunk *filer_pb
 	return newVisibles
 }
 
-func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) {
+func NonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
 
 	sort.Slice(chunks, func(i, j int) bool {
 		return chunks[i].Mtime < chunks[j].Mtime
 	})
 
-	var newVislbles []*visibleInterval
+	var newVisibles []VisibleInterval
 	for _, chunk := range chunks {
-		newVislbles = mergeIntoVisibles(visibles, newVislbles, chunk)
+		newVisibles = MergeIntoVisibles(visibles, newVisibles, chunk)
 		t := visibles[:0]
-		visibles = newVislbles
-		newVislbles = t
+		visibles = newVisibles
+		newVisibles = t
 
 		logPrintf("add", visibles)
 
@@ -190,19 +202,21 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v
 // find non-overlapping visible intervals
 // visible interval map to one file chunk
 
-type visibleInterval struct {
+type VisibleInterval struct {
 	start        int64
 	stop         int64
 	modifiedTime int64
 	fileId       string
+	isFullChunk  bool
 }
 
-func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64) *visibleInterval {
-	return &visibleInterval{
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval {
+	return VisibleInterval{
 		start:        start,
 		stop:         stop,
 		fileId:       fileId,
 		modifiedTime: modifiedTime,
+		isFullChunk:  isFullChunk,
 	}
 }
 

weed/filer2/filechunks_test.go

@@ -63,7 +63,7 @@ func TestIntervalMerging(t *testing.T) {
 
 	testcases := []struct {
 		Chunks   []*filer_pb.FileChunk
-		Expected []*visibleInterval
+		Expected []*VisibleInterval
 	}{
 		// case 0: normal
 		{
@@ -72,7 +72,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
 				{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 100, fileId: "abc"},
 				{start: 100, stop: 200, fileId: "asdf"},
 				{start: 200, stop: 300, fileId: "fsad"},
@@ -84,7 +84,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 200, fileId: "asdf"},
 			},
 		},
@@ -94,7 +94,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 				{Offset: 0, Size: 50, FileId: "asdf", Mtime: 134},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 50, fileId: "asdf"},
 				{start: 50, stop: 100, fileId: "abc"},
 			},
@@ -106,7 +106,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
 				{Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 50, fileId: "asdf"},
 				{start: 50, stop: 300, fileId: "xxxx"},
 			},
@@ -118,7 +118,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
 				{Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 200, fileId: "asdf"},
 				{start: 250, stop: 500, fileId: "xxxx"},
 			},
@@ -131,7 +131,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 70, Size: 150, FileId: "abc", Mtime: 143},
 				{Offset: 80, Size: 100, FileId: "xxxx", Mtime: 134},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 200, fileId: "asdf"},
 				{start: 200, stop: 220, fileId: "abc"},
 			},
@@ -143,7 +143,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 100, fileId: "abc"},
 			},
 		},
@@ -157,7 +157,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 8388608, Size: 3145728, FileId: "5,02982f80de50", Mtime: 160},
 				{Offset: 11534336, Size: 2842193, FileId: "7,0299ad723803", Mtime: 170},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 2097152, fileId: "3,029565bf3092"},
 				{start: 2097152, stop: 5242880, fileId: "6,029632f47ae2"},
 				{start: 5242880, stop: 8388608, fileId: "2,029734c5aa10"},
@@ -174,7 +174,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 208896, Size: 339968 - 208896, FileId: "2,0b4031a72689", Mtime: 150},
 				{Offset: 339968, Size: 471040 - 339968, FileId: "3,0b416a557362", Mtime: 160},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 77824, fileId: "4,0b3df938e301"},
 				{start: 77824, stop: 208896, fileId: "4,0b3f0c7202f0"},
 				{start: 208896, stop: 339968, fileId: "2,0b4031a72689"},
@@ -186,7 +186,7 @@ func TestIntervalMerging(t *testing.T) {
 
 	for i, testcase := range testcases {
 		log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i)
-		intervals := nonOverlappingVisibleIntervals(testcase.Chunks)
+		intervals := NonOverlappingVisibleIntervals(testcase.Chunks)
 		for x, interval := range intervals {
 			log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s",
 				i, x, interval.start, interval.stop, interval.fileId)

weed/filer2/filer.go

@@ -14,6 +14,11 @@ import (
 	"github.com/karlseguin/ccache"
 )
 
+var (
+	OS_UID = uint32(os.Getuid())
+	OS_GID = uint32(os.Getgid())
+)
+
 type Filer struct {
 	store              FilerStore
 	directoryCache     *ccache.Cache
@@ -51,6 +56,10 @@ func (fs *Filer) KeepConnectedToMaster() {
 
 func (f *Filer) CreateEntry(entry *Entry) error {
 
+	if string(entry.FullPath) == "/" {
+		return nil
+	}
+
 	dirParts := strings.Split(string(entry.FullPath), "/")
 
 	// fmt.Printf("directory parts: %+v\n", dirParts)
@@ -157,6 +166,21 @@ func (f *Filer) UpdateEntry(oldEntry, entry *Entry) (err error) {
 }
 
 func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
+
+	now := time.Now()
+
+	if string(p) == "/" {
+		return &Entry{
+			FullPath: p,
+			Attr: Attr{
+				Mtime:  now,
+				Crtime: now,
+				Mode:   os.ModeDir | 0755,
+				Uid:    OS_UID,
+				Gid:    OS_GID,
+			},
+		}, nil
+	}
 	return f.store.FindEntry(p)
 }
 
@@ -171,20 +195,36 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 		if isRecursive {
 			limit = math.MaxInt32
 		}
-		entries, err := f.ListDirectoryEntries(p, "", false, limit)
-		if err != nil {
-			return fmt.Errorf("list folder %s: %v", p, err)
-		}
-		if isRecursive {
-			for _, sub := range entries {
-				f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
+		lastFileName := ""
+		includeLastFile := false
+		for limit > 0 {
+			entries, err := f.ListDirectoryEntries(p, lastFileName, includeLastFile, 1024)
+			if err != nil {
+				return fmt.Errorf("list folder %s: %v", p, err)
 			}
-		} else {
-			if len(entries) > 0 {
-				return fmt.Errorf("folder %s is not empty", p)
+			if len(entries) == 0 {
+				break
+			} else {
+				if isRecursive {
+					for _, sub := range entries {
+						lastFileName = sub.Name()
+						f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
+						limit--
+						if limit <= 0 {
+							break
+						}
+					}
+				} else {
+					if len(entries) > 0 {
+						return fmt.Errorf("folder %s is not empty", p)
+					}
+				}
+				f.cacheDelDirectory(string(p))
+				if len(entries) < 1024 {
+					break
+				}
 			}
 		}
-		f.cacheDelDirectory(string(p))
 	}
 
 	if shouldDeleteChunks {

weed/filer2/leveldb/leveldb_store.go

@@ -33,11 +33,13 @@ func (store *LevelDBStore) Initialize(configuration weed_util.Configuration) (er
 }
 
 func (store *LevelDBStore) initialize(dir string) (err error) {
+	glog.Infof("filer store dir: %s", dir)
 	if err := weed_util.TestFolderWritable(dir); err != nil {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}
 
 	if store.db, err = leveldb.OpenFile(dir, nil); err != nil {
+		glog.Infof("filer store open dir %s: %v", dir, err)
 		return
 	}
 	return

weed/filer2/leveldb/leveldb_store_test.go

@@ -59,3 +59,25 @@ func TestCreateAndFind(t *testing.T) {
 	}
 
 }
+
+func TestEmptyRoot(t *testing.T) {
+	filer := filer2.NewFiler(nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	defer os.RemoveAll(dir)
+	store := &LevelDBStore{}
+	store.initialize(dir)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	// checking one upper directory
+	entries, err := filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
+	if err != nil {
+		t.Errorf("list entries: %v", err)
+		return
+	}
+	if len(entries) != 0 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}

weed/filer2/redis/universal_redis_store.go

@@ -7,6 +7,7 @@ import (
 	"github.com/go-redis/redis"
 	"sort"
 	"strings"
+	"time"
 )
 
 const (
@@ -24,7 +25,7 @@ func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	_, err = store.Client.Set(string(entry.FullPath), value, 0).Result()
+	_, err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result()
 
 	if err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)

weed/filesys/dir.go

@@ -7,11 +7,11 @@ import (
 	"path/filepath"
 	"time"
 
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 type Dir struct {
@@ -34,8 +34,10 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 	// https://github.com/bazil/fuse/issues/196
 	attr.Valid = time.Second
 
-	if dir.Path == "/" {
-		attr.Mode = os.ModeDir | 0777
+	if dir.Path == dir.wfs.option.FilerMountRootPath {
+		attr.Uid = dir.wfs.option.MountUid
+		attr.Gid = dir.wfs.option.MountGid
+		attr.Mode = dir.wfs.option.MountMode
 		return nil
 	}
 
@@ -64,6 +66,9 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
 		resp, err := client.LookupDirectoryEntry(context, request)
 		if err != nil {
+			if err == filer2.ErrNotFound {
+				return nil
+			}
 			glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
 			return err
 		}
@@ -72,7 +77,7 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 			dir.attributes = resp.Entry.Attributes
 		}
 
-		dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
+		// dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
 
 		return nil
 	})
@@ -96,10 +101,11 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
 func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
 	return &File{
-		Name:  name,
-		dir:   dir,
-		wfs:   dir.wfs,
-		entry: entry,
+		Name:           name,
+		dir:            dir,
+		wfs:            dir.wfs,
+		entry:          entry,
+		entryViewCache: nil,
 	}
 }
 
@@ -209,7 +215,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 
 			entry = resp.Entry
 
-			dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
+			// dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
 
 			return nil
 		})
@@ -239,29 +245,46 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		request := &filer_pb.ListEntriesRequest{
-			Directory: dir.Path,
-			Limit:     uint32(dir.wfs.option.DirListingLimit),
-		}
+		paginationLimit := 1024
+		remaining := dir.wfs.option.DirListingLimit
 
-		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("list %s: %v", dir.Path, err)
-			return fuse.EIO
-		}
+		lastEntryName := ""
 
-		cacheTtl := estimatedCacheTtl(len(resp.Entries))
+		for remaining >= 0 {
 
-		for _, entry := range resp.Entries {
-			if entry.IsDirectory {
-				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
-				ret = append(ret, dirent)
-			} else {
-				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
-				ret = append(ret, dirent)
+			request := &filer_pb.ListEntriesRequest{
+				Directory:         dir.Path,
+				StartFromFileName: lastEntryName,
+				Limit:             uint32(paginationLimit),
 			}
-			dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
+
+			glog.V(4).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
+			if err != nil {
+				glog.V(0).Infof("list %s: %v", dir.Path, err)
+				return fuse.EIO
+			}
+
+			cacheTtl := estimatedCacheTtl(len(resp.Entries))
+
+			for _, entry := range resp.Entries {
+				if entry.IsDirectory {
+					dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
+					ret = append(ret, dirent)
+				} else {
+					dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
+					ret = append(ret, dirent)
+				}
+				dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
+				lastEntryName = entry.Name
+			}
+
+			remaining -= len(resp.Entries)
+
+			if len(resp.Entries) < paginationLimit {
+				break
+			}
+
 		}
 
 		return nil
@@ -272,20 +295,79 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 
+	if !req.Dir {
+		return dir.removeOneFile(ctx, req)
+	}
+
+	return dir.removeFolder(ctx, req)
+
+}
+
+func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
+
+	var entry *filer_pb.Entry
+	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir.Path,
+			Name:      req.Name,
+		}
+
+		glog.V(4).Infof("lookup to-be-removed entry: %v", request)
+		resp, err := client.LookupDirectoryEntry(ctx, request)
+		if err != nil {
+			// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
+			return fuse.ENOENT
+		}
+
+		entry = resp.Entry
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	dir.wfs.deleteFileChunks(entry.Chunks)
+
+	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.DeleteEntryRequest{
+			Directory:    dir.Path,
+			Name:         req.Name,
+			IsDeleteData: false,
+		}
+
+		glog.V(3).Infof("remove file: %v", request)
+		_, err := client.DeleteEntry(ctx, request)
+		if err != nil {
+			glog.V(3).Infof("remove file %s/%s: %v", dir.Path, req.Name, err)
+			return fuse.ENOENT
+		}
+
+		dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
+
+		return nil
+	})
+
+}
+
+func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
+
 	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
 			Name:         req.Name,
-			IsDirectory:  req.Dir,
 			IsDeleteData: true,
 		}
 
-		glog.V(1).Infof("remove directory entry: %v", request)
+		glog.V(3).Infof("remove directory entry: %v", request)
 		_, err := client.DeleteEntry(ctx, request)
 		if err != nil {
-			glog.V(0).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
-			return fuse.EIO
+			glog.V(3).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
+			return fuse.ENOENT
 		}
 
 		dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
@@ -352,10 +434,7 @@ func estimatedCacheTtl(numEntries int) time.Duration {
 		// 10 ms per entry
 		return 100 * time.Second
 	}
-	if numEntries < 100000 {
-		// 10 ms per entry
-		return 1000 * time.Second
-	}
+
 	// 2 ms per entry
 	return time.Duration(numEntries*2) * time.Millisecond
 }

weed/filesys/dir_link.go

@@ -0,0 +1,66 @@
+package filesys
+
+import (
+	"context"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
+)
+
+var _ = fs.NodeSymlinker(&Dir{})
+var _ = fs.NodeReadlinker(&File{})
+
+func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {
+
+	glog.V(3).Infof("Symlink: %v/%v to %v", dir.Path, req.NewName, req.Target)
+
+	request := &filer_pb.CreateEntryRequest{
+		Directory: dir.Path,
+		Entry: &filer_pb.Entry{
+			Name:        req.NewName,
+			IsDirectory: false,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:         time.Now().Unix(),
+				Crtime:        time.Now().Unix(),
+				FileMode:      uint32(os.FileMode(0755) | os.ModeSymlink),
+				Uid:           req.Uid,
+				Gid:           req.Gid,
+				SymlinkTarget: req.Target,
+			},
+		},
+	}
+
+	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
+			glog.V(0).Infof("symlink %s/%s: %v", dir.Path, req.NewName, err)
+			return fuse.EIO
+		}
+		return nil
+	})
+
+	symlink := dir.newFile(req.NewName, request.Entry)
+
+	return symlink, err
+
+}
+
+func (file *File) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
+
+	if err := file.maybeLoadAttributes(ctx); err != nil {
+		return "", err
+	}
+
+	if os.FileMode(file.entry.Attributes.FileMode)&os.ModeSymlink == 0 {
+		return "", fuse.Errno(syscall.EINVAL)
+	}
+
+	glog.V(3).Infof("Readlink: %v/%v => %v", file.dir.Path, file.Name, file.entry.Attributes.SymlinkTarget)
+
+	return file.entry.Attributes.SymlinkTarget, nil
+
+}

weed/filesys/dir_rename.go

@@ -1,12 +1,14 @@
 package filesys
 
 import (
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"context"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
+	"math"
+	"path/filepath"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"path/filepath"
 )
 
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
@@ -24,7 +26,7 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 		glog.V(4).Infof("find existing directory entry: %v", request)
 		resp, err := client.LookupDirectoryEntry(ctx, request)
 		if err != nil {
-			glog.V(0).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
+			glog.V(3).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
 			return fuse.ENOENT
 		}
 
@@ -40,21 +42,37 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParent string, entry *filer_pb.Entry, newParent, newName string) error {
 	if entry.IsDirectory {
 		currentDirPath := filepath.Join(oldParent, entry.Name)
-		request := &filer_pb.ListEntriesRequest{
-			Directory: currentDirPath,
-		}
 
-		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("list %s: %v", oldParent, err)
-			return fuse.EIO
-		}
-
-		for _, item := range resp.Entries {
-			err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
+		lastFileName := ""
+		includeLastFile := false
+		limit := math.MaxInt32
+		for limit > 0 {
+			request := &filer_pb.ListEntriesRequest{
+				Directory:          currentDirPath,
+				StartFromFileName:  lastFileName,
+				InclusiveStartFrom: includeLastFile,
+				Limit:              1024,
+			}
+			glog.V(4).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
 			if err != nil {
-				return err
+				glog.V(0).Infof("list %s: %v", oldParent, err)
+				return fuse.EIO
+			}
+			if len(resp.Entries) == 0 {
+				break
+			}
+
+			for _, item := range resp.Entries {
+				lastFileName = item.Name
+				err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
+				if err != nil {
+					return err
+				}
+				limit--
+			}
+			if len(resp.Entries) < 1024 {
+				break
 			}
 		}
 
@@ -84,7 +102,6 @@ func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParen
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    oldParent,
 			Name:         entry.Name,
-			IsDirectory:  entry.IsDirectory,
 			IsDeleteData: false,
 		}
 

weed/filesys/dirty_page.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"sync/atomic"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -23,19 +24,22 @@ type ContinuousDirtyPages struct {
 
 func newDirtyPages(file *File) *ContinuousDirtyPages {
 	return &ContinuousDirtyPages{
-		Data: make([]byte, file.wfs.option.ChunkSizeLimit),
+		Data: nil,
 		f:    file,
 	}
 }
 
-func (pages *ContinuousDirtyPages) InitializeToFile(file *File) *ContinuousDirtyPages {
-	if len(pages.Data) != int(file.wfs.option.ChunkSizeLimit) {
-		pages.Data = make([]byte, file.wfs.option.ChunkSizeLimit)
+func (pages *ContinuousDirtyPages) releaseResource() {
+	if pages.Data != nil {
+		pages.f.wfs.bufPool.Put(pages.Data)
+		pages.Data = nil
+		atomic.AddInt32(&counter, -1)
+		glog.V(3).Infof("%s/%s releasing resource %d", pages.f.dir.Path, pages.f.Name, counter)
 	}
-	pages.f = file
-	return pages
 }
 
+var counter = int32(0)
+
 func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
 
 	pages.lock.Lock()
@@ -43,11 +47,17 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da
 
 	var chunk *filer_pb.FileChunk
 
-	if len(data) > len(pages.Data) {
+	if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) {
 		// this is more than what buffer can hold.
 		return pages.flushAndSave(ctx, offset, data)
 	}
 
+	if pages.Data == nil {
+		pages.Data = pages.f.wfs.bufPool.Get().([]byte)
+		atomic.AddInt32(&counter, 1)
+		glog.V(3).Infof("%s/%s acquire resource %d", pages.f.dir.Path, pages.f.Name, counter)
+	}
+
 	if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) ||
 		pages.Offset+int64(len(pages.Data)) < offset+int64(len(data)) {
 		// if the data is out of range,

weed/filesys/file.go

@@ -1,15 +1,17 @@
 package filesys
 
 import (
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"context"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"os"
-	"path/filepath"
-	"time"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 const blockSize = 512
@@ -20,11 +22,12 @@ var _ = fs.NodeFsyncer(&File{})
 var _ = fs.NodeSetattrer(&File{})
 
 type File struct {
-	Name   string
-	dir    *Dir
-	wfs    *WFS
-	entry  *filer_pb.Entry
-	isOpen bool
+	Name           string
+	dir            *Dir
+	wfs            *WFS
+	entry          *filer_pb.Entry
+	entryViewCache []filer2.VisibleInterval
+	isOpen         bool
 }
 
 func (file *File) fullpath() string {
@@ -82,6 +85,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		if req.Size == 0 {
 			// fmt.Printf("truncate %v \n", fullPath)
 			file.entry.Chunks = nil
+			file.entryViewCache = nil
 		}
 		file.entry.Attributes.FileSize = req.Size
 	}
@@ -97,6 +101,10 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		file.entry.Attributes.Gid = req.Gid
 	}
 
+	if req.Valid.Crtime() {
+		file.entry.Attributes.Crtime = req.Crtime.Unix()
+	}
+
 	if req.Valid.Mtime() {
 		file.entry.Attributes.Mtime = req.Mtime.Unix()
 	}
@@ -133,7 +141,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 		item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
 		if item != nil && !item.Expired() {
 			entry := item.Value().(*filer_pb.Entry)
-			file.entry = entry
+			file.setEntry(entry)
 			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
 		} else {
 			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
@@ -145,15 +153,15 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 
 				resp, err := client.LookupDirectoryEntry(ctx, request)
 				if err != nil {
-					glog.V(0).Infof("file attr read file %v: %v", request, err)
-					return err
+					glog.V(3).Infof("file attr read file %v: %v", request, err)
+					return fuse.ENOENT
 				}
 
-				file.entry = resp.Entry
+				file.setEntry(resp.Entry)
 
-				glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))
+				glog.V(3).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))
 
-				file.wfs.listDirectoryEntriesCache.Set(file.fullpath(), file.entry, file.wfs.option.EntryCacheTtl)
+				// file.wfs.listDirectoryEntriesCache.Set(file.fullpath(), file.entry, file.wfs.option.EntryCacheTtl)
 
 				return nil
 			})
@@ -165,3 +173,31 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 	}
 	return nil
 }
+
+func (file *File) addChunk(chunk *filer_pb.FileChunk) {
+	if chunk != nil {
+		file.addChunks([]*filer_pb.FileChunk{chunk})
+	}
+}
+
+func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
+
+	sort.Slice(chunks, func(i, j int) bool {
+		return chunks[i].Mtime < chunks[j].Mtime
+	})
+
+	var newVisibles []filer2.VisibleInterval
+	for _, chunk := range chunks {
+		newVisibles = filer2.MergeIntoVisibles(file.entryViewCache, newVisibles, chunk)
+		t := file.entryViewCache[:0]
+		file.entryViewCache = newVisibles
+		newVisibles = t
+	}
+
+	file.entry.Chunks = append(file.entry.Chunks, chunks...)
+}
+
+func (file *File) setEntry(entry *filer_pb.Entry) {
+	file.entry = entry
+	file.entryViewCache = filer2.NonOverlappingVisibleIntervals(file.entry.Chunks)
+}

weed/filesys/filehandle.go

@@ -1,17 +1,19 @@
 package filesys
 
 import (
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 )
 
 type FileHandle struct {
@@ -37,16 +39,6 @@ func newFileHandle(file *File, uid, gid uint32) *FileHandle {
 	}
 }
 
-func (fh *FileHandle) InitializeToFile(file *File, uid, gid uint32) *FileHandle {
-	newHandle := &FileHandle{
-		f:          file,
-		dirtyPages: fh.dirtyPages.InitializeToFile(file),
-		Uid:        uid,
-		Gid:        gid,
-	}
-	return newHandle
-}
-
 var _ = fs.Handle(&FileHandle{})
 
 // var _ = fs.HandleReadAller(&FileHandle{})
@@ -67,7 +59,11 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 
 	buff := make([]byte, req.Size)
 
-	chunkViews := filer2.ViewFromChunks(fh.f.entry.Chunks, req.Offset, req.Size)
+	if fh.f.entryViewCache == nil {
+		fh.f.entryViewCache = filer2.NonOverlappingVisibleIntervals(fh.f.entry.Chunks)
+	}
+
+	chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)
 
 	var vids []string
 	for _, chunkView := range chunkViews {
@@ -117,7 +113,8 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 				fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId),
 				chunkView.Offset,
 				int(chunkView.Size),
-				buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)])
+				buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)],
+				!chunkView.IsFullChunk)
 
 			if err != nil {
 
@@ -160,9 +157,9 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 		fh.dirtyMetadata = true
 	}
 
-	for _, chunk := range chunks {
-		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
-		glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+	fh.f.addChunks(chunks)
+
+	if len(chunks) > 0 {
 		fh.dirtyMetadata = true
 	}
 
@@ -173,6 +170,8 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 
 	glog.V(4).Infof("%v release fh %d", fh.f.fullpath(), fh.handle)
 
+	fh.dirtyPages.releaseResource()
+
 	fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
 
 	fh.f.isOpen = false
@@ -180,8 +179,6 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 	return nil
 }
 
-// Flush - experimenting with uploading at flush, this slows operations down till it has been
-// completely flushed
 func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	// fflush works at fh level
 	// send the data to the OS
@@ -192,9 +189,8 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		glog.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
 		return fmt.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
 	}
-	if chunk != nil {
-		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
-	}
+
+	fh.f.addChunk(chunk)
 
 	if !fh.dirtyMetadata {
 		return nil
@@ -206,6 +202,9 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 			fh.f.entry.Attributes.Mime = fh.contentType
 			fh.f.entry.Attributes.Uid = req.Uid
 			fh.f.entry.Attributes.Gid = req.Gid
+			fh.f.entry.Attributes.Mtime = time.Now().Unix()
+			fh.f.entry.Attributes.Crtime = time.Now().Unix()
+			fh.f.entry.Attributes.FileMode = uint32(0770)
 		}
 
 		request := &filer_pb.CreateEntryRequest{
@@ -213,10 +212,16 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 			Entry:     fh.f.entry,
 		}
 
-		glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
-		for i, chunk := range fh.f.entry.Chunks {
-			glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
-		}
+		//glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
+		//for i, chunk := range fh.f.entry.Chunks {
+		//	glog.V(4).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+		//}
+
+		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
+		fh.f.entry.Chunks = chunks
+		// fh.f.entryViewCache = nil
+		fh.f.wfs.deleteFileChunks(garbages)
+
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
@@ -225,6 +230,48 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	})
 }
 
+func deleteFileIds(ctx context.Context, client filer_pb.SeaweedFilerClient, fileIds []string) error {
+
+	var vids []string
+	for _, fileId := range fileIds {
+		vids = append(vids, volumeId(fileId))
+	}
+
+	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
+
+		m := make(map[string]operation.LookupResult)
+
+		glog.V(4).Infof("remove file lookup volume id locations: %v", vids)
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+			VolumeIds: vids,
+		})
+		if err != nil {
+			return m, err
+		}
+
+		for _, vid := range vids {
+			lr := operation.LookupResult{
+				VolumeId:  vid,
+				Locations: nil,
+			}
+			locations := resp.LocationsMap[vid]
+			for _, loc := range locations.Locations {
+				lr.Locations = append(lr.Locations, operation.Location{
+					Url:       loc.Url,
+					PublicUrl: loc.PublicUrl,
+				})
+			}
+			m[vid] = lr
+		}
+
+		return m, err
+	}
+
+	_, err := operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+
+	return err
+}
+
 func volumeId(fileId string) string {
 	lastCommaIndex := strings.LastIndex(fileId, ",")
 	if lastCommaIndex > 0 {

weed/filesys/wfs.go

@@ -4,15 +4,16 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"os"
 	"sync"
 	"time"
 
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/karlseguin/ccache"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 	"google.golang.org/grpc"
 )
 
@@ -26,6 +27,10 @@ type Option struct {
 	DataCenter         string
 	DirListingLimit    int
 	EntryCacheTtl      time.Duration
+
+	MountUid  uint32
+	MountGid  uint32
+	MountMode os.FileMode
 }
 
 var _ = fs.FS(&WFS{})
@@ -39,10 +44,9 @@ type WFS struct {
 	handles           []*FileHandle
 	pathToHandleIndex map[string]int
 	pathToHandleLock  sync.Mutex
+	bufPool           sync.Pool
 
-	// cache grpc connections
-	grpcClients     map[string]*grpc.ClientConn
-	grpcClientsLock sync.Mutex
+	fileIdsDeletionChan chan []string
 
 	stats statsCache
 }
@@ -52,12 +56,21 @@ type statsCache struct {
 }
 
 func NewSeaweedFileSystem(option *Option) *WFS {
-	return &WFS{
+	wfs := &WFS{
 		option:                    option,
-		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(int64(option.DirListingLimit) + 200).ItemsToPrune(100)),
+		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(1024 * 8).ItemsToPrune(100)),
 		pathToHandleIndex:         make(map[string]int),
-		grpcClients:               make(map[string]*grpc.ClientConn),
+		bufPool: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, option.ChunkSizeLimit)
+			},
+		},
+		fileIdsDeletionChan: make(chan []string, 32),
 	}
+
+	go wfs.loopProcessingDeletion()
+
+	return wfs
 }
 
 func (wfs *WFS) Root() (fs.Node, error) {
@@ -66,27 +79,11 @@ func (wfs *WFS) Root() (fs.Node, error) {
 
 func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	wfs.grpcClientsLock.Lock()
-
-	existingConnection, found := wfs.grpcClients[wfs.option.FilerGrpcAddress]
-	if found {
-		wfs.grpcClientsLock.Unlock()
-		client := filer_pb.NewSeaweedFilerClient(existingConnection)
+	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
-	}
+	}, wfs.option.FilerGrpcAddress)
 
-	grpcConnection, err := util.GrpcDial(wfs.option.FilerGrpcAddress)
-	if err != nil {
-		wfs.grpcClientsLock.Unlock()
-		return fmt.Errorf("fail to dial %s: %v", wfs.option.FilerGrpcAddress, err)
-	}
-
-	wfs.grpcClients[wfs.option.FilerGrpcAddress] = grpcConnection
-	wfs.grpcClientsLock.Unlock()
-
-	client := filer_pb.NewSeaweedFilerClient(grpcConnection)
-
-	return fn(client)
 }
 
 func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHandle) {
@@ -97,17 +94,10 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
 
 	index, found := wfs.pathToHandleIndex[fullpath]
 	if found && wfs.handles[index] != nil {
-		glog.V(4).Infoln(fullpath, "found fileHandle id", index)
+		glog.V(2).Infoln(fullpath, "found fileHandle id", index)
 		return wfs.handles[index]
 	}
 
-	if found && wfs.handles[index] != nil {
-		glog.V(4).Infoln(fullpath, "reuse previous fileHandle id", index)
-		wfs.handles[index].InitializeToFile(file, uid, gid)
-		fileHandle.handle = uint64(index)
-		return
-	}
-
 	fileHandle = newFileHandle(file, uid, gid)
 	for i, h := range wfs.handles {
 		if h == nil {
@@ -121,7 +111,7 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
 
 	wfs.handles = append(wfs.handles, fileHandle)
 	fileHandle.handle = uint64(len(wfs.handles) - 1)
-	glog.V(4).Infoln(fullpath, "new fileHandle id", fileHandle.handle)
+	glog.V(2).Infoln(fullpath, "new fileHandle id", fileHandle.handle)
 	wfs.pathToHandleIndex[fullpath] = int(fileHandle.handle)
 
 	return
@@ -184,11 +174,11 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
 	resp.Blocks = totalDiskSize / blockSize
 
 	// Compute the number of used blocks
-	numblocks := uint64(usedDiskSize / blockSize)
+	numBlocks := uint64(usedDiskSize / blockSize)
 
 	// Report the number of free and available blocks for the block size
-	resp.Bfree = resp.Blocks - numblocks
-	resp.Bavail = resp.Blocks - numblocks
+	resp.Bfree = resp.Blocks - numBlocks
+	resp.Bavail = resp.Blocks - numBlocks
 	resp.Bsize = uint32(blockSize)
 
 	// Report the total number of possible files in the file system (and those free)

weed/filesys/wfs_deletion.go

@@ -0,0 +1,58 @@
+package filesys
+
+import (
+	"context"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+func (wfs *WFS) loopProcessingDeletion() {
+
+	ticker := time.NewTicker(2 * time.Second)
+
+	wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		var fileIds []string
+		for {
+			select {
+			case fids := <-wfs.fileIdsDeletionChan:
+				fileIds = append(fileIds, fids...)
+				if len(fileIds) >= 1024 {
+					glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
+					deleteFileIds(context.Background(), client, fileIds)
+					fileIds = fileIds[:0]
+				}
+			case <-ticker.C:
+				if len(fileIds) > 0 {
+					glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
+					deleteFileIds(context.Background(), client, fileIds)
+					fileIds = fileIds[:0]
+				}
+			}
+		}
+	})
+
+}
+
+func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) {
+	if len(chunks) == 0 {
+		return
+	}
+
+	var fileIds []string
+	for _, chunk := range chunks {
+		fileIds = append(fileIds, chunk.FileId)
+	}
+
+	var async = false
+	if async {
+		wfs.fileIdsDeletionChan <- fileIds
+		return
+	}
+
+	wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		deleteFileIds(context.Background(), client, fileIds)
+		return nil
+	})
+}

weed/glide.lock

@@ -1,7 +1,7 @@
 hash: 2e3a065472829938d25e879451b6d1aa43e55270e1166a9c044803ef8a3b9eb1
 updated: 2018-06-28T22:01:35.910567-07:00
 imports:
-- name: bazil.org/fuse
+- name: github.com/seaweedfs/fuse
   version: 65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e
   subpackages:
   - fs

weed/glide.yaml

@@ -1,6 +1,6 @@
 package: github.com/chrislusf/seaweedfs/weed
 import:
-- package: bazil.org/fuse
+- package: github.com/seaweedfs/fuse
   subpackages:
   - fs
 - package: github.com/boltdb/bolt

weed/glog/glog.go

@@ -46,7 +46,7 @@
 //	-stderrthreshold=ERROR
 //		Log events at or above this severity are logged to standard
 //		error as well as to files.
-//	-log_dir=""
+//	-logdir=""
 //		Log files will be written to this directory instead of the
 //		default temporary directory.
 //

weed/glog/glog_file.go

@@ -38,7 +38,7 @@ var logDirs []string
 
 // If non-empty, overrides the choice of directory in which to write logs.
 // See createLogDirs for the full list of possible destinations.
-var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory")
+var logDir = flag.String("logdir", "", "If non-empty, write log files in this directory")
 
 func createLogDirs() {
 	if *logDir != "" {

weed/notification/configuration.go

@@ -27,17 +27,32 @@ func LoadConfiguration(config *viper.Viper) {
 		return
 	}
 
-	for _, store := range MessageQueues {
-		if config.GetBool(store.GetName() + ".enabled") {
-			viperSub := config.Sub(store.GetName())
-			if err := store.Initialize(viperSub); err != nil {
-				glog.Fatalf("Failed to initialize store for %s: %+v",
-					store.GetName(), err)
+	validateOneEnabledQueue(config)
+
+	for _, queue := range MessageQueues {
+		if config.GetBool(queue.GetName() + ".enabled") {
+			viperSub := config.Sub(queue.GetName())
+			if err := queue.Initialize(viperSub); err != nil {
+				glog.Fatalf("Failed to initialize notification for %s: %+v",
+					queue.GetName(), err)
 			}
-			Queue = store
-			glog.V(0).Infof("Configure notification message queue for %s", store.GetName())
+			Queue = queue
+			glog.V(0).Infof("Configure notification message queue for %s", queue.GetName())
 			return
 		}
 	}
 
 }
+
+func validateOneEnabledQueue(config *viper.Viper) {
+	enabledQueue := ""
+	for _, queue := range MessageQueues {
+		if config.GetBool(queue.GetName() + ".enabled") {
+			if enabledQueue == "" {
+				enabledQueue = queue.GetName()
+			} else {
+				glog.Fatalf("Notification message queue is enabled for both %s and %s", enabledQueue, queue.GetName())
+			}
+		}
+	}
+}

weed/operation/assign_file_id.go

@@ -3,6 +3,7 @@ package operation
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
@@ -40,6 +41,9 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 		}
 
 		lastError = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+			defer cancel()
+
 			req := &master_pb.AssignRequest{
 				Count:       primaryRequest.Count,
 				Replication: primaryRequest.Replication,
@@ -49,7 +53,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 				Rack:        primaryRequest.Rack,
 				DataNode:    primaryRequest.DataNode,
 			}
-			resp, grpcErr := masterClient.Assign(context.Background(), req)
+			resp, grpcErr := masterClient.Assign(ctx, req)
 			if grpcErr != nil {
 				return grpcErr
 			}

weed/operation/compress.go

@@ -8,21 +8,41 @@ import (
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"golang.org/x/tools/godoc/util"
 )
 
 /*
 * Default more not to gzip since gzip can be done on client side.
  */
-func IsGzippable(ext, mtype string) bool {
+func IsGzippable(ext, mtype string, data []byte) bool {
+
+	// text
 	if strings.HasPrefix(mtype, "text/") {
 		return true
 	}
+
+	// images
+	switch ext {
+	case ".svg", ".bmp":
+		return true
+	}
+	if strings.HasPrefix(mtype, "image/") {
+		return false
+	}
+
+	// by file name extention
 	switch ext {
 	case ".zip", ".rar", ".gz", ".bz2", ".xz":
 		return false
 	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
 		return true
+	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
+		return true
+	case ".png", ".jpg", ".jpeg":
+		return false
 	}
+
+	// by mime type
 	if strings.HasPrefix(mtype, "application/") {
 		if strings.HasSuffix(mtype, "xml") {
 			return true
@@ -31,7 +51,10 @@ func IsGzippable(ext, mtype string) bool {
 			return true
 		}
 	}
-	return false
+
+	isMostlyText := util.IsText(data)
+
+	return isMostlyText
 }
 
 func GzipData(input []byte) ([]byte, error) {

weed/operation/delete_content.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 )
@@ -108,12 +109,14 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
 
 	err = WithVolumeServerClient(volumeServer, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
 
 		req := &volume_server_pb.BatchDeleteRequest{
 			FileIds: fileIds,
 		}
 
-		resp, err := volumeServerClient.BatchDelete(context.Background(), req)
+		resp, err := volumeServerClient.BatchDelete(ctx, req)
 
 		// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)
 
@@ -131,7 +134,7 @@ func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []
 	}
 
 	for _, result := range ret {
-		if result.Error != "" && result.Error != "Not Found" {
+		if result.Error != "" && result.Error != "not found" {
 			return nil, fmt.Errorf("delete fileId %s: %v", result.FileId, result.Error)
 		}
 	}

weed/operation/grpc_client.go

@@ -25,27 +25,11 @@ func WithVolumeServerClient(volumeServer string, fn func(volume_server_pb.Volume
 		return err
 	}
 
-	grpcClientsLock.Lock()
-
-	existingConnection, found := grpcClients[grpcAddress]
-	if found {
-		grpcClientsLock.Unlock()
-		client := volume_server_pb.NewVolumeServerClient(existingConnection)
+	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+		client := volume_server_pb.NewVolumeServerClient(grpcConnection)
 		return fn(client)
-	}
+	}, grpcAddress)
 
-	grpcConnection, err := util.GrpcDial(grpcAddress)
-	if err != nil {
-		grpcClientsLock.Unlock()
-		return fmt.Errorf("fail to dial %s: %v", grpcAddress, err)
-	}
-
-	grpcClients[grpcAddress] = grpcConnection
-	grpcClientsLock.Unlock()
-
-	client := volume_server_pb.NewVolumeServerClient(grpcConnection)
-
-	return fn(client)
 }
 
 func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err error) {
@@ -60,25 +44,14 @@ func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err err
 
 func withMasterServerClient(masterServer string, fn func(masterClient master_pb.SeaweedClient) error) error {
 
-	grpcClientsLock.Lock()
+	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer, 0)
+	if parseErr != nil {
+		return fmt.Errorf("failed to parse master grpc %v", masterServer)
+	}
 
-	existingConnection, found := grpcClients[masterServer]
-	if found {
-		grpcClientsLock.Unlock()
-		client := master_pb.NewSeaweedClient(existingConnection)
+	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
+		client := master_pb.NewSeaweedClient(grpcConnection)
 		return fn(client)
-	}
+	}, masterGrpcAddress)
 
-	grpcConnection, err := util.GrpcDial(masterServer)
-	if err != nil {
-		grpcClientsLock.Unlock()
-		return fmt.Errorf("fail to dial %s: %v", masterServer, err)
-	}
-
-	grpcClients[masterServer] = grpcConnection
-	grpcClientsLock.Unlock()
-
-	client := master_pb.NewSeaweedClient(grpcConnection)
-
-	return fn(client)
 }

weed/operation/lookup.go

@@ -99,10 +99,13 @@ func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, err
 	//only query unknown_vids
 
 	err := withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
+
 		req := &master_pb.LookupVolumeRequest{
 			VolumeIds: unknown_vids,
 		}
-		resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
+		resp, grpcErr := masterClient.LookupVolume(ctx, req)
 		if grpcErr != nil {
 			return grpcErr
 		}

weed/operation/lookup_vid_cache.go

@@ -9,6 +9,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
+var ErrorNotFound = errors.New("not found")
+
 type VidInfo struct {
 	Locations       []Location
 	NextRefreshTime time.Time
@@ -28,14 +30,14 @@ func (vc *VidCache) Get(vid string) ([]Location, error) {
 	defer vc.RUnlock()
 	if 0 < id && id <= len(vc.cache) {
 		if vc.cache[id-1].Locations == nil {
-			return nil, errors.New("Not Set")
+			return nil, errors.New("not set")
 		}
 		if vc.cache[id-1].NextRefreshTime.Before(time.Now()) {
-			return nil, errors.New("Expired")
+			return nil, errors.New("expired")
 		}
 		return vc.cache[id-1].Locations, nil
 	}
-	return nil, errors.New("Not Found")
+	return nil, ErrorNotFound
 }
 func (vc *VidCache) Set(vid string, locations []Location, duration time.Duration) {
 	id, err := strconv.Atoi(vid)

weed/operation/stats.go

@@ -2,6 +2,7 @@ package operation
 
 import (
 	"context"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
@@ -9,7 +10,10 @@ import (
 func Statistics(server string, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
 
 	err = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
-		grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
+
+		grpcResponse, grpcErr := masterClient.Statistics(ctx, req)
 		if grpcErr != nil {
 			return grpcErr
 		}

weed/operation/submit.go

@@ -18,7 +18,6 @@ type FilePart struct {
 	Reader      io.Reader
 	FileName    string
 	FileSize    int64
-	IsGzipped   bool
 	MimeType    string
 	ModTime     int64 //in seconds
 	Replication string
@@ -103,7 +102,6 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
 	ret.ModTime = fi.ModTime().UTC().Unix()
 	ret.FileSize = fi.Size()
 	ext := strings.ToLower(path.Ext(fullPathFilename))
-	ret.IsGzipped = ext == ".gz"
 	ret.FileName = fi.Name()
 	if ext != "" {
 		ret.MimeType = mime.TypeByExtension(ext)
@@ -193,7 +191,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 			cm.DeleteChunks(master)
 		}
 	} else {
-		ret, e := Upload(fileUrl, baseName, fi.Reader, fi.IsGzipped, fi.MimeType, nil, jwt)
+		ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt)
 		if e != nil {
 			return 0, e
 		}

weed/operation/sync_volume.go

@@ -2,6 +2,9 @@ package operation
 
 import (
 	"context"
+	"fmt"
+	"io"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
@@ -11,7 +14,10 @@ import (
 func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
 
 	WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
-		resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
+		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+		defer cancel()
+
+		resp, err = client.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{
 			VolumdId: vid,
 		})
 		return nil
@@ -23,17 +29,30 @@ func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.Volu
 func GetVolumeIdxEntries(server string, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
 
 	return WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
-		resp, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
+		stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
 			VolumdId: vid,
 		})
 		if err != nil {
 			return err
 		}
 
-		dataSize := len(resp.IndexFileContent)
+		var indexFileContent []byte
+
+		for {
+			resp, err := stream.Recv()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return fmt.Errorf("read index entries: %v", err)
+			}
+			indexFileContent = append(indexFileContent, resp.IndexFileContent...)
+		}
+
+		dataSize := len(indexFileContent)
 
 		for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
-			line := resp.IndexFileContent[idx : idx+NeedleEntrySize]
+			line := indexFileContent[idx : idx+NeedleEntrySize]
 			key := BytesToNeedleId(line[:NeedleIdSize])
 			offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
 			size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])

weed/operation/upload_content.go

@@ -22,7 +22,7 @@ type UploadResult struct {
 	Name  string `json:"name,omitempty"`
 	Size  uint32 `json:"size,omitempty"`
 	Error string `json:"error,omitempty"`
-	ETag  string `json:"error,omitempty"`
+	ETag  string `json:"eTag,omitempty"`
 }
 
 var (
@@ -92,6 +92,12 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 		return nil, post_err
 	}
 	defer resp.Body.Close()
+
+	if resp.StatusCode < http.StatusOK ||
+		resp.StatusCode > http.StatusIMUsed {
+		return nil, errors.New(http.StatusText(resp.StatusCode))
+	}
+
 	etag := getEtag(resp)
 	resp_body, ra_err := ioutil.ReadAll(resp.Body)
 	if ra_err != nil {

weed/pb/filer.proto

@@ -97,6 +97,7 @@ message FuseAttributes {
     int32 ttl_sec = 10;
     string user_name = 11; // for hdfs
     repeated string group_name = 12; // for hdfs
+    string symlink_target = 13;
 }
 
 message CreateEntryRequest {
@@ -117,7 +118,7 @@ message UpdateEntryResponse {
 message DeleteEntryRequest {
     string directory = 1;
     string name = 2;
-    bool is_directory = 3;
+    // bool is_directory = 3;
     bool is_delete_data = 4;
     bool is_recursive = 5;
 }

weed/pb/filer_pb/filer.pb.go

@@ -297,18 +297,19 @@ func (m *FileChunk) GetSourceFileId() string {
 }
 
 type FuseAttributes struct {
-	FileSize    uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
-	Mtime       int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
-	FileMode    uint32   `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"`
-	Uid         uint32   `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"`
-	Gid         uint32   `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"`
-	Crtime      int64    `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"`
-	Mime        string   `protobuf:"bytes,7,opt,name=mime" json:"mime,omitempty"`
-	Replication string   `protobuf:"bytes,8,opt,name=replication" json:"replication,omitempty"`
-	Collection  string   `protobuf:"bytes,9,opt,name=collection" json:"collection,omitempty"`
-	TtlSec      int32    `protobuf:"varint,10,opt,name=ttl_sec,json=ttlSec" json:"ttl_sec,omitempty"`
-	UserName    string   `protobuf:"bytes,11,opt,name=user_name,json=userName" json:"user_name,omitempty"`
-	GroupName   []string `protobuf:"bytes,12,rep,name=group_name,json=groupName" json:"group_name,omitempty"`
+	FileSize      uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
+	Mtime         int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
+	FileMode      uint32   `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"`
+	Uid           uint32   `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"`
+	Gid           uint32   `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"`
+	Crtime        int64    `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"`
+	Mime          string   `protobuf:"bytes,7,opt,name=mime" json:"mime,omitempty"`
+	Replication   string   `protobuf:"bytes,8,opt,name=replication" json:"replication,omitempty"`
+	Collection    string   `protobuf:"bytes,9,opt,name=collection" json:"collection,omitempty"`
+	TtlSec        int32    `protobuf:"varint,10,opt,name=ttl_sec,json=ttlSec" json:"ttl_sec,omitempty"`
+	UserName      string   `protobuf:"bytes,11,opt,name=user_name,json=userName" json:"user_name,omitempty"`
+	GroupName     []string `protobuf:"bytes,12,rep,name=group_name,json=groupName" json:"group_name,omitempty"`
+	SymlinkTarget string   `protobuf:"bytes,13,opt,name=symlink_target,json=symlinkTarget" json:"symlink_target,omitempty"`
 }
 
 func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
@@ -400,6 +401,13 @@ func (m *FuseAttributes) GetGroupName() []string {
 	return nil
 }
 
+func (m *FuseAttributes) GetSymlinkTarget() string {
+	if m != nil {
+		return m.SymlinkTarget
+	}
+	return ""
+}
+
 type CreateEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
 	Entry     *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
@@ -465,11 +473,11 @@ func (*UpdateEntryResponse) ProtoMessage()               {}
 func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
 
 type DeleteEntryRequest struct {
-	Directory    string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
-	Name         string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
-	IsDirectory  bool   `protobuf:"varint,3,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"`
-	IsDeleteData bool   `protobuf:"varint,4,opt,name=is_delete_data,json=isDeleteData" json:"is_delete_data,omitempty"`
-	IsRecursive  bool   `protobuf:"varint,5,opt,name=is_recursive,json=isRecursive" json:"is_recursive,omitempty"`
+	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
+	Name      string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
+	// bool is_directory = 3;
+	IsDeleteData bool `protobuf:"varint,4,opt,name=is_delete_data,json=isDeleteData" json:"is_delete_data,omitempty"`
+	IsRecursive  bool `protobuf:"varint,5,opt,name=is_recursive,json=isRecursive" json:"is_recursive,omitempty"`
 }
 
 func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
@@ -491,13 +499,6 @@ func (m *DeleteEntryRequest) GetName() string {
 	return ""
 }
 
-func (m *DeleteEntryRequest) GetIsDirectory() bool {
-	if m != nil {
-		return m.IsDirectory
-	}
-	return false
-}
-
 func (m *DeleteEntryRequest) GetIsDeleteData() bool {
 	if m != nil {
 		return m.IsDeleteData
@@ -1158,85 +1159,86 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1272 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xcd, 0x6f, 0xdc, 0x44,
-	0x14, 0xaf, 0xf7, 0x2b, 0xeb, 0xb7, 0xbb, 0x21, 0x99, 0x0d, 0xd4, 0xda, 0x7c, 0xb0, 0x35, 0x2d,
-	0x4a, 0x45, 0xb4, 0xaa, 0x02, 0x87, 0x96, 0x0a, 0x89, 0x36, 0x1f, 0x52, 0xa4, 0xb4, 0x45, 0x4e,
-	0x83, 0x84, 0x38, 0x58, 0x8e, 0x3d, 0xbb, 0x8c, 0xe2, 0xb5, 0x8d, 0x67, 0x9c, 0xb4, 0xfc, 0x1d,
-	0x5c, 0x39, 0x70, 0xe2, 0xce, 0x1f, 0xc0, 0x85, 0x03, 0x7f, 0x13, 0x37, 0x34, 0x1f, 0xf6, 0x8e,
-	0xd7, 0x4e, 0x0a, 0x42, 0xbd, 0xcd, 0xfc, 0xde, 0x9b, 0xf7, 0x35, 0xbf, 0x79, 0xcf, 0x86, 0xde,
-	0x94, 0x84, 0x38, 0x9d, 0x24, 0x69, 0xcc, 0x62, 0xd4, 0x15, 0x1b, 0x37, 0xb9, 0xb0, 0x5f, 0xc1,
-	0xe6, 0x69, 0x1c, 0x5f, 0x66, 0xc9, 0x21, 0x49, 0xb1, 0xcf, 0xe2, 0xf4, 0xed, 0x51, 0xc4, 0xd2,
-	0xb7, 0x0e, 0xfe, 0x31, 0xc3, 0x94, 0xa1, 0x2d, 0x30, 0x83, 0x5c, 0x60, 0x19, 0x63, 0x63, 0xd7,
-	0x74, 0x16, 0x00, 0x42, 0xd0, 0x8a, 0xbc, 0x39, 0xb6, 0x1a, 0x42, 0x20, 0xd6, 0xf6, 0x11, 0x6c,
-	0xd5, 0x1b, 0xa4, 0x49, 0x1c, 0x51, 0x8c, 0x1e, 0x40, 0x1b, 0x73, 0x40, 0x58, 0xeb, 0xed, 0x7f,
-	0x30, 0xc9, 0x43, 0x99, 0x48, 0x3d, 0x29, 0xb5, 0xff, 0x30, 0x00, 0x9d, 0x12, 0xca, 0x38, 0x48,
-	0x30, 0xfd, 0x77, 0xf1, 0x7c, 0x04, 0x9d, 0x24, 0xc5, 0x53, 0xf2, 0x46, 0x45, 0xa4, 0x76, 0x68,
-	0x0f, 0xd6, 0x29, 0xf3, 0x52, 0x76, 0x9c, 0xc6, 0xf3, 0x63, 0x12, 0xe2, 0x97, 0x3c, 0xe8, 0xa6,
-	0x50, 0xa9, 0x0a, 0xd0, 0x04, 0x10, 0x89, 0xfc, 0x30, 0xa3, 0xe4, 0x0a, 0x9f, 0xe5, 0x52, 0xab,
-	0x35, 0x36, 0x76, 0xbb, 0x4e, 0x8d, 0x04, 0x6d, 0x40, 0x3b, 0x24, 0x73, 0xc2, 0xac, 0xf6, 0xd8,
-	0xd8, 0x1d, 0x38, 0x72, 0x63, 0x7f, 0x0d, 0xc3, 0x52, 0xfc, 0x2a, 0xfd, 0x87, 0xb0, 0x82, 0x25,
-	0x64, 0x19, 0xe3, 0x66, 0x5d, 0x01, 0x72, 0xb9, 0xfd, 0x4b, 0x03, 0xda, 0x02, 0x2a, 0xea, 0x6c,
-	0x2c, 0xea, 0x8c, 0xee, 0x41, 0x9f, 0x50, 0x77, 0x51, 0x8c, 0x86, 0x88, 0xaf, 0x47, 0x68, 0x51,
-	0x77, 0xf4, 0x19, 0x74, 0xfc, 0x1f, 0xb2, 0xe8, 0x92, 0x5a, 0x4d, 0xe1, 0x6a, 0xb8, 0x70, 0xc5,
-	0x93, 0x3d, 0xe0, 0x32, 0x47, 0xa9, 0xa0, 0xc7, 0x00, 0x1e, 0x63, 0x29, 0xb9, 0xc8, 0x18, 0xa6,
-	0x22, 0xdb, 0xde, 0xbe, 0xa5, 0x1d, 0xc8, 0x28, 0x7e, 0x56, 0xc8, 0x1d, 0x4d, 0x17, 0x3d, 0x81,
-	0x2e, 0x7e, 0xc3, 0x70, 0x14, 0xe0, 0xc0, 0x6a, 0x0b, 0x47, 0xdb, 0x4b, 0x39, 0x4d, 0x8e, 0x94,
-	0x5c, 0x66, 0x58, 0xa8, 0x8f, 0x9e, 0xc2, 0xa0, 0x24, 0x42, 0x6b, 0xd0, 0xbc, 0xc4, 0xf9, 0xcd,
-	0xf2, 0x25, 0xaf, 0xee, 0x95, 0x17, 0x66, 0x92, 0x64, 0x7d, 0x47, 0x6e, 0xbe, 0x6c, 0x3c, 0x36,
-	0xec, 0x9f, 0x0d, 0x58, 0x3f, 0xba, 0xc2, 0x11, 0x7b, 0x19, 0x33, 0x32, 0x25, 0xbe, 0xc7, 0x48,
-	0x1c, 0xa1, 0x3d, 0x30, 0xe3, 0x30, 0x70, 0x6f, 0xe5, 0x58, 0x37, 0x0e, 0x95, 0xbf, 0x3d, 0x30,
-	0x23, 0x7c, 0xad, 0xb4, 0x1b, 0x37, 0x68, 0x47, 0xf8, 0x5a, 0x6a, 0x7f, 0x02, 0x83, 0x00, 0x87,
-	0x98, 0x61, 0xb7, 0xa8, 0x2b, 0x2f, 0x7a, 0x5f, 0x82, 0xa2, 0x9e, 0xd4, 0xfe, 0xd5, 0x00, 0xb3,
-	0x28, 0x2f, 0xba, 0x0b, 0x2b, 0xdc, 0x9c, 0x4b, 0x02, 0x95, 0x54, 0x87, 0x6f, 0x4f, 0x02, 0xce,
-	0xd5, 0x78, 0x3a, 0xa5, 0x98, 0x09, 0xb7, 0x4d, 0x47, 0xed, 0xf8, 0x5d, 0x53, 0xf2, 0x93, 0xa4,
-	0x67, 0xcb, 0x11, 0x6b, 0x5e, 0x83, 0x39, 0x23, 0x73, 0x2c, 0xae, 0xa5, 0xe9, 0xc8, 0x0d, 0x1a,
-	0x42, 0x1b, 0xbb, 0xcc, 0x9b, 0x09, 0xde, 0x99, 0x4e, 0x0b, 0xbf, 0xf6, 0x66, 0xe8, 0x3e, 0xac,
-	0xd2, 0x38, 0x4b, 0x7d, 0xec, 0xe6, 0x6e, 0x3b, 0x42, 0xda, 0x97, 0xe8, 0xb1, 0x70, 0x6e, 0xff,
-	0xd5, 0x80, 0xd5, 0xf2, 0x8d, 0xa2, 0x4d, 0x30, 0xc5, 0x09, 0xe1, 0xdc, 0x10, 0xce, 0x45, 0x97,
-	0x38, 0x2b, 0x05, 0xd0, 0xd0, 0x03, 0xc8, 0x8f, 0xcc, 0xe3, 0x40, 0xc6, 0x3b, 0x90, 0x47, 0x5e,
-	0xc4, 0x01, 0xe6, 0x37, 0x99, 0x91, 0x40, 0x44, 0x3c, 0x70, 0xf8, 0x92, 0x23, 0x33, 0x12, 0xa8,
-	0x57, 0xc2, 0x97, 0xbc, 0x06, 0x7e, 0x2a, 0xec, 0x76, 0x64, 0x0d, 0xe4, 0x8e, 0xd7, 0x60, 0xce,
-	0xd1, 0x15, 0x99, 0x18, 0x5f, 0xa3, 0x31, 0xf4, 0x52, 0x9c, 0x84, 0xea, 0x9a, 0xad, 0xae, 0x10,
-	0xe9, 0x10, 0xda, 0x01, 0xf0, 0xe3, 0x30, 0xc4, 0xbe, 0x50, 0x30, 0x85, 0x82, 0x86, 0xf0, 0xab,
-	0x60, 0x2c, 0x74, 0x29, 0xf6, 0x2d, 0x18, 0x1b, 0xbb, 0x6d, 0xa7, 0xc3, 0x58, 0x78, 0x86, 0x7d,
-	0x9e, 0x47, 0x46, 0x71, 0xea, 0x8a, 0x37, 0xd6, 0x13, 0xe7, 0xba, 0x1c, 0x10, 0xdd, 0x60, 0x1b,
-	0x60, 0x96, 0xc6, 0x59, 0x22, 0xa5, 0xfd, 0x71, 0x93, 0xb7, 0x1c, 0x81, 0x70, 0xb1, 0xfd, 0x1d,
-	0xa0, 0x83, 0x14, 0x7b, 0x0c, 0xff, 0x87, 0xb6, 0x59, 0xb4, 0xc0, 0xc6, 0xad, 0x2d, 0xf0, 0x43,
-	0x18, 0x96, 0x4c, 0xcb, 0x0e, 0xc2, 0x3d, 0x9e, 0x27, 0xc1, 0xfb, 0xf2, 0x58, 0x32, 0xad, 0x3c,
-	0xfe, 0x6e, 0x00, 0x3a, 0x14, 0x14, 0xff, 0x7f, 0xb3, 0xa1, 0xd2, 0xb3, 0x9a, 0xd5, 0x9e, 0x75,
-	0x1f, 0x56, 0xb9, 0x8a, 0x7c, 0x65, 0x81, 0xc7, 0x3c, 0xd5, 0x78, 0xfb, 0x84, 0xca, 0x10, 0x0e,
-	0x3d, 0xe6, 0x29, 0x43, 0x29, 0xf6, 0xb3, 0x94, 0xf7, 0x62, 0xc1, 0x29, 0x61, 0xc8, 0xc9, 0x21,
-	0x9e, 0x4b, 0x29, 0x66, 0x95, 0xcb, 0x6f, 0x06, 0x0c, 0x9f, 0x51, 0x4a, 0x66, 0xd1, 0xb7, 0x71,
-	0x98, 0xcd, 0x71, 0x9e, 0xcc, 0x06, 0xb4, 0xfd, 0x38, 0x8b, 0x98, 0x48, 0xa4, 0xed, 0xc8, 0xcd,
-	0x12, 0xa5, 0x1a, 0x15, 0x4a, 0x2d, 0x91, 0xb2, 0x59, 0x25, 0xa5, 0x46, 0xba, 0x56, 0x89, 0x74,
-	0x1f, 0x43, 0x8f, 0xa7, 0xe7, 0xfa, 0x38, 0x62, 0x38, 0x55, 0x6f, 0x18, 0x38, 0x74, 0x20, 0x10,
-	0xfb, 0x0a, 0x36, 0xca, 0x81, 0xaa, 0x09, 0x72, 0x63, 0x47, 0xe1, 0x2f, 0x2e, 0x0d, 0x55, 0x94,
-	0x7c, 0xc9, 0xb9, 0x9b, 0x64, 0x17, 0x21, 0xf1, 0x5d, 0x2e, 0x90, 0xd1, 0x99, 0x12, 0x39, 0x4f,
-	0xc3, 0x45, 0xce, 0x2d, 0x2d, 0x67, 0xfb, 0x0b, 0x18, 0xca, 0x01, 0x5e, 0x2e, 0xd0, 0x36, 0xc0,
-	0x95, 0x00, 0x5c, 0x12, 0xc8, 0xd9, 0x65, 0x3a, 0xa6, 0x44, 0x4e, 0x02, 0x6a, 0x7f, 0x05, 0xe6,
-	0x69, 0x2c, 0x73, 0xa6, 0xe8, 0x11, 0x98, 0x61, 0xbe, 0x51, 0x63, 0x0e, 0x2d, 0x28, 0x97, 0xeb,
-	0x39, 0x0b, 0x25, 0xfb, 0x29, 0x74, 0x73, 0x38, 0xcf, 0xc3, 0xb8, 0x29, 0x8f, 0xc6, 0x52, 0x1e,
-	0xf6, 0x9f, 0x06, 0x6c, 0x94, 0x43, 0x56, 0xa5, 0x3a, 0x87, 0x41, 0xe1, 0xc2, 0x9d, 0x7b, 0x89,
-	0x8a, 0xe5, 0x91, 0x1e, 0x4b, 0xf5, 0x58, 0x11, 0x20, 0x7d, 0xe1, 0x25, 0x92, 0x3d, 0xfd, 0x50,
-	0x83, 0x46, 0xaf, 0x61, 0xbd, 0xa2, 0x52, 0x33, 0xb9, 0x1e, 0xea, 0x93, 0xab, 0x34, 0x7d, 0x8b,
-	0xd3, 0xfa, 0x38, 0x7b, 0x02, 0x77, 0x25, 0x61, 0x0f, 0x0a, 0x7e, 0xe5, 0xb5, 0x2f, 0xd3, 0xd0,
-	0x58, 0xa6, 0xa1, 0x3d, 0x02, 0xab, 0x7a, 0x54, 0x11, 0x7e, 0x06, 0xeb, 0x67, 0xcc, 0x63, 0x84,
-	0x32, 0xe2, 0x17, 0x9f, 0x51, 0x4b, 0xbc, 0x35, 0xde, 0xd5, 0x4c, 0xab, 0xcc, 0x5f, 0x83, 0x26,
-	0x63, 0x39, 0xa7, 0xf8, 0x92, 0xdf, 0x02, 0xd2, 0x3d, 0xa9, 0x3b, 0x78, 0x0f, 0xae, 0x38, 0x1f,
-	0x58, 0xcc, 0xbc, 0x50, 0x0e, 0xab, 0x96, 0x18, 0x56, 0xa6, 0x40, 0xc4, 0xb4, 0x92, 0xfd, 0x3c,
-	0x90, 0xd2, 0xb6, 0x1c, 0x65, 0x1c, 0x10, 0xc2, 0x6d, 0x00, 0xf1, 0x7c, 0x24, 0xf3, 0x3b, 0xf2,
-	0x2c, 0x47, 0x0e, 0x38, 0xb0, 0xff, 0x77, 0x1b, 0xfa, 0x67, 0xd8, 0xbb, 0xc6, 0x38, 0xe0, 0xb3,
-	0x32, 0x45, 0xb3, 0x9c, 0x5b, 0xe5, 0xef, 0x59, 0xf4, 0x60, 0x99, 0x44, 0xb5, 0x1f, 0xd0, 0xa3,
-	0x4f, 0xdf, 0xa5, 0xa6, 0xae, 0xe9, 0x0e, 0x3a, 0x85, 0x9e, 0xf6, 0xc1, 0x88, 0xb6, 0xb4, 0x83,
-	0x95, 0xef, 0xe0, 0xd1, 0xf6, 0x0d, 0x52, 0xdd, 0x9a, 0x36, 0x3c, 0x74, 0x6b, 0xd5, 0x71, 0xa5,
-	0x5b, 0xab, 0x9b, 0x38, 0xc2, 0x9a, 0x36, 0x18, 0x74, 0x6b, 0xd5, 0x51, 0xa4, 0x5b, 0xab, 0x9b,
-	0x26, 0xc2, 0x9a, 0xd6, 0x9a, 0x75, 0x6b, 0xd5, 0x29, 0xa3, 0x5b, 0xab, 0xeb, 0xe7, 0x77, 0xd0,
-	0x2b, 0xe8, 0xeb, 0x7d, 0x12, 0x69, 0x07, 0x6a, 0x1a, 0xfd, 0x68, 0xe7, 0x26, 0xb1, 0x6e, 0x50,
-	0x6f, 0x0b, 0xba, 0xc1, 0x9a, 0xc6, 0xa8, 0x1b, 0xac, 0xeb, 0x26, 0xf6, 0x1d, 0xf4, 0x3d, 0xac,
-	0x2d, 0x3f, 0x4f, 0x74, 0x6f, 0x39, 0xad, 0xca, 0xab, 0x1f, 0xd9, 0xb7, 0xa9, 0x14, 0xc6, 0x4f,
-	0x00, 0x16, 0xaf, 0x0e, 0x6d, 0x2e, 0xce, 0x54, 0x5e, 0xfd, 0x68, 0xab, 0x5e, 0x98, 0x9b, 0x7a,
-	0xbe, 0x03, 0x6b, 0x54, 0x52, 0x7f, 0x4a, 0x27, 0x7e, 0x48, 0x70, 0xc4, 0x9e, 0x83, 0x78, 0x05,
-	0xdf, 0xf0, 0xbf, 0xc6, 0x8b, 0x8e, 0xf8, 0x79, 0xfc, 0xfc, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff,
-	0x58, 0x55, 0x30, 0x36, 0x4b, 0x0e, 0x00, 0x00,
+	// 1291 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0x4d, 0x8f, 0xdc, 0x44,
+	0x13, 0x8e, 0xe7, 0x2b, 0xe3, 0x9a, 0x99, 0xbc, 0xbb, 0x3d, 0xfb, 0x12, 0x6b, 0xb2, 0x1b, 0x26,
+	0x86, 0xa0, 0x8d, 0x88, 0x46, 0x51, 0xe0, 0x90, 0x10, 0x21, 0x91, 0x6c, 0x36, 0x52, 0xa4, 0x4d,
+	0x82, 0xbc, 0x09, 0x12, 0xe2, 0x60, 0x79, 0xed, 0x9e, 0xa1, 0xb5, 0x1e, 0x7b, 0x70, 0xb7, 0x37,
+	0x09, 0x7f, 0x82, 0x0b, 0x57, 0x0e, 0x9c, 0xf8, 0x17, 0x5c, 0xf8, 0x3f, 0xdc, 0xb9, 0xa1, 0xae,
+	0x6e, 0x7b, 0xda, 0x63, 0xef, 0x06, 0x84, 0x72, 0xeb, 0x7e, 0xaa, 0xba, 0xbe, 0xfa, 0xe9, 0x2a,
+	0x1b, 0x06, 0x73, 0x16, 0xd3, 0x6c, 0xb6, 0xca, 0x52, 0x91, 0x92, 0x3e, 0x6e, 0xfc, 0xd5, 0x89,
+	0xfb, 0x02, 0xae, 0x1d, 0xa5, 0xe9, 0x69, 0xbe, 0x7a, 0xcc, 0x32, 0x1a, 0x8a, 0x34, 0x7b, 0x7b,
+	0x98, 0x88, 0xec, 0xad, 0x47, 0x7f, 0xc8, 0x29, 0x17, 0x64, 0x17, 0xec, 0xa8, 0x10, 0x38, 0xd6,
+	0xd4, 0xda, 0xb7, 0xbd, 0x35, 0x40, 0x08, 0x74, 0x92, 0x60, 0x49, 0x9d, 0x16, 0x0a, 0x70, 0xed,
+	0x1e, 0xc2, 0x6e, 0xb3, 0x41, 0xbe, 0x4a, 0x13, 0x4e, 0xc9, 0x4d, 0xe8, 0x52, 0x09, 0xa0, 0xb5,
+	0xc1, 0xdd, 0xff, 0xcd, 0x8a, 0x50, 0x66, 0x4a, 0x4f, 0x49, 0xdd, 0xdf, 0x2d, 0x20, 0x47, 0x8c,
+	0x0b, 0x09, 0x32, 0xca, 0xff, 0x59, 0x3c, 0x1f, 0x40, 0x6f, 0x95, 0xd1, 0x39, 0x7b, 0xa3, 0x23,
+	0xd2, 0x3b, 0x72, 0x1b, 0xb6, 0xb9, 0x08, 0x32, 0xf1, 0x24, 0x4b, 0x97, 0x4f, 0x58, 0x4c, 0x9f,
+	0xcb, 0xa0, 0xdb, 0xa8, 0x52, 0x17, 0x90, 0x19, 0x10, 0x96, 0x84, 0x71, 0xce, 0xd9, 0x19, 0x3d,
+	0x2e, 0xa4, 0x4e, 0x67, 0x6a, 0xed, 0xf7, 0xbd, 0x06, 0x09, 0xd9, 0x81, 0x6e, 0xcc, 0x96, 0x4c,
+	0x38, 0xdd, 0xa9, 0xb5, 0x3f, 0xf2, 0xd4, 0xc6, 0xfd, 0x0a, 0xc6, 0x95, 0xf8, 0x75, 0xfa, 0xb7,
+	0xe0, 0x32, 0x55, 0x90, 0x63, 0x4d, 0xdb, 0x4d, 0x05, 0x28, 0xe4, 0xee, 0x2f, 0x2d, 0xe8, 0x22,
+	0x54, 0xd6, 0xd9, 0x5a, 0xd7, 0x99, 0xdc, 0x80, 0x21, 0xe3, 0xfe, 0xba, 0x18, 0x2d, 0x8c, 0x6f,
+	0xc0, 0x78, 0x59, 0x77, 0xf2, 0x29, 0xf4, 0xc2, 0xef, 0xf3, 0xe4, 0x94, 0x3b, 0x6d, 0x74, 0x35,
+	0x5e, 0xbb, 0x92, 0xc9, 0x1e, 0x48, 0x99, 0xa7, 0x55, 0xc8, 0x3d, 0x80, 0x40, 0x88, 0x8c, 0x9d,
+	0xe4, 0x82, 0x72, 0xcc, 0x76, 0x70, 0xd7, 0x31, 0x0e, 0xe4, 0x9c, 0x3e, 0x2c, 0xe5, 0x9e, 0xa1,
+	0x4b, 0xee, 0x43, 0x9f, 0xbe, 0x11, 0x34, 0x89, 0x68, 0xe4, 0x74, 0xd1, 0xd1, 0xde, 0x46, 0x4e,
+	0xb3, 0x43, 0x2d, 0x57, 0x19, 0x96, 0xea, 0x93, 0x07, 0x30, 0xaa, 0x88, 0xc8, 0x16, 0xb4, 0x4f,
+	0x69, 0x71, 0xb3, 0x72, 0x29, 0xab, 0x7b, 0x16, 0xc4, 0xb9, 0x22, 0xd9, 0xd0, 0x53, 0x9b, 0x2f,
+	0x5a, 0xf7, 0x2c, 0xf7, 0x67, 0x0b, 0xb6, 0x0f, 0xcf, 0x68, 0x22, 0x9e, 0xa7, 0x82, 0xcd, 0x59,
+	0x18, 0x08, 0x96, 0x26, 0xe4, 0x36, 0xd8, 0x69, 0x1c, 0xf9, 0x17, 0x72, 0xac, 0x9f, 0xc6, 0xda,
+	0xdf, 0x6d, 0xb0, 0x13, 0xfa, 0x5a, 0x6b, 0xb7, 0xce, 0xd1, 0x4e, 0xe8, 0x6b, 0xa5, 0xfd, 0x11,
+	0x8c, 0x22, 0x1a, 0x53, 0x41, 0xfd, 0xb2, 0xae, 0xb2, 0xe8, 0x43, 0x05, 0x62, 0x3d, 0xb9, 0xfb,
+	0xab, 0x05, 0x76, 0x59, 0x5e, 0x72, 0x15, 0x2e, 0x4b, 0x73, 0x3e, 0x8b, 0x74, 0x52, 0x3d, 0xb9,
+	0x7d, 0x1a, 0x49, 0xae, 0xa6, 0xf3, 0x39, 0xa7, 0x02, 0xdd, 0xb6, 0x3d, 0xbd, 0x93, 0x77, 0xcd,
+	0xd9, 0x8f, 0x8a, 0x9e, 0x1d, 0x0f, 0xd7, 0xb2, 0x06, 0x4b, 0xc1, 0x96, 0x14, 0xaf, 0xa5, 0xed,
+	0xa9, 0x0d, 0x19, 0x43, 0x97, 0xfa, 0x22, 0x58, 0x20, 0xef, 0x6c, 0xaf, 0x43, 0x5f, 0x06, 0x0b,
+	0xf2, 0x31, 0x5c, 0xe1, 0x69, 0x9e, 0x85, 0xd4, 0x2f, 0xdc, 0xf6, 0x50, 0x3a, 0x54, 0xe8, 0x13,
+	0x74, 0xee, 0xfe, 0xd9, 0x82, 0x2b, 0xd5, 0x1b, 0x25, 0xd7, 0xc0, 0xc6, 0x13, 0xe8, 0xdc, 0x42,
+	0xe7, 0xd8, 0x25, 0x8e, 0x2b, 0x01, 0xb4, 0xcc, 0x00, 0x8a, 0x23, 0xcb, 0x34, 0x52, 0xf1, 0x8e,
+	0xd4, 0x91, 0x67, 0x69, 0x44, 0xe5, 0x4d, 0xe6, 0x2c, 0xc2, 0x88, 0x47, 0x9e, 0x5c, 0x4a, 0x64,
+	0xc1, 0x22, 0xfd, 0x4a, 0xe4, 0x52, 0xd6, 0x20, 0xcc, 0xd0, 0x6e, 0x4f, 0xd5, 0x40, 0xed, 0x64,
+	0x0d, 0x96, 0x12, 0xbd, 0xac, 0x12, 0x93, 0x6b, 0x32, 0x85, 0x41, 0x46, 0x57, 0xb1, 0xbe, 0x66,
+	0xa7, 0x8f, 0x22, 0x13, 0x22, 0xd7, 0x01, 0xc2, 0x34, 0x8e, 0x69, 0x88, 0x0a, 0x36, 0x2a, 0x18,
+	0x88, 0xbc, 0x0a, 0x21, 0x62, 0x9f, 0xd3, 0xd0, 0x81, 0xa9, 0xb5, 0xdf, 0xf5, 0x7a, 0x42, 0xc4,
+	0xc7, 0x34, 0x94, 0x79, 0xe4, 0x9c, 0x66, 0x3e, 0xbe, 0xb1, 0x01, 0x9e, 0xeb, 0x4b, 0x00, 0xbb,
+	0xc1, 0x1e, 0xc0, 0x22, 0x4b, 0xf3, 0x95, 0x92, 0x0e, 0xa7, 0x6d, 0xd9, 0x72, 0x10, 0x41, 0xf1,
+	0x4d, 0xb8, 0xc2, 0xdf, 0x2e, 0x63, 0x96, 0x9c, 0xfa, 0x22, 0xc8, 0x16, 0x54, 0x38, 0x23, 0x34,
+	0x30, 0xd2, 0xe8, 0x4b, 0x04, 0xdd, 0x6f, 0x81, 0x1c, 0x64, 0x34, 0x10, 0xf4, 0x5f, 0x74, 0xd7,
+	0xb2, 0x53, 0xb6, 0x2e, 0xec, 0x94, 0xff, 0x87, 0x71, 0xc5, 0xb4, 0x6a, 0x34, 0xd2, 0xe3, 0xab,
+	0x55, 0xf4, 0xbe, 0x3c, 0x56, 0x4c, 0x6b, 0x8f, 0x3f, 0x59, 0x40, 0x1e, 0xe3, 0x4b, 0xf8, 0x6f,
+	0x23, 0x44, 0x72, 0x58, 0xb6, 0x36, 0xf5, 0xd2, 0xa2, 0x40, 0x04, 0xba, 0xf9, 0x0e, 0x19, 0x57,
+	0xf6, 0x1f, 0x07, 0x22, 0xd0, 0x0d, 0x30, 0xa3, 0x61, 0x9e, 0xc9, 0x7e, 0x8c, 0xbc, 0xc2, 0x06,
+	0xe8, 0x15, 0x90, 0x0c, 0xb4, 0x12, 0x90, 0x0e, 0xf4, 0x37, 0x0b, 0xc6, 0x0f, 0x39, 0x67, 0x8b,
+	0xe4, 0x9b, 0x34, 0xce, 0x97, 0xb4, 0x88, 0x74, 0x07, 0xba, 0x61, 0x9a, 0x27, 0x02, 0xa3, 0xec,
+	0x7a, 0x6a, 0xb3, 0x41, 0xab, 0x56, 0x8d, 0x56, 0x1b, 0xc4, 0x6c, 0xd7, 0x89, 0x69, 0x10, 0xaf,
+	0x53, 0x21, 0xde, 0x87, 0x30, 0x90, 0xe9, 0xf9, 0x21, 0x4d, 0x04, 0xcd, 0xf4, 0x3b, 0x06, 0x09,
+	0x1d, 0x20, 0xe2, 0x9e, 0xc1, 0x4e, 0x35, 0x50, 0x3d, 0x45, 0xce, 0xed, 0x2a, 0xf2, 0xd5, 0x65,
+	0xb1, 0x8e, 0x52, 0x2e, 0x25, 0x7f, 0x57, 0xf9, 0x49, 0xcc, 0x42, 0x5f, 0x0a, 0x54, 0x74, 0xb6,
+	0x42, 0x5e, 0x65, 0xf1, 0x3a, 0xe7, 0x8e, 0x91, 0xb3, 0xfb, 0x39, 0x8c, 0xd5, 0x10, 0xaf, 0x16,
+	0x68, 0x0f, 0xe0, 0x0c, 0x01, 0x9f, 0x45, 0x6a, 0x7e, 0xd9, 0x9e, 0xad, 0x90, 0xa7, 0x11, 0x77,
+	0xbf, 0x04, 0xfb, 0x28, 0x55, 0x39, 0x73, 0x72, 0x07, 0xec, 0xb8, 0xd8, 0xe8, 0x51, 0x47, 0xd6,
+	0x7c, 0x2a, 0xf4, 0xbc, 0xb5, 0x92, 0xfb, 0x00, 0xfa, 0x05, 0x5c, 0xe4, 0x61, 0x9d, 0x97, 0x47,
+	0x6b, 0x23, 0x0f, 0xf7, 0x0f, 0x0b, 0x76, 0xaa, 0x21, 0xeb, 0x52, 0xbd, 0x82, 0x51, 0xe9, 0xc2,
+	0x5f, 0x06, 0x2b, 0x1d, 0xcb, 0x1d, 0x33, 0x96, 0xfa, 0xb1, 0x32, 0x40, 0xfe, 0x2c, 0x58, 0x29,
+	0xf6, 0x0c, 0x63, 0x03, 0x9a, 0xbc, 0x84, 0xed, 0x9a, 0x4a, 0xc3, 0xf4, 0xba, 0x65, 0x4e, 0xaf,
+	0xca, 0x04, 0x2e, 0x4f, 0x9b, 0x23, 0xed, 0x3e, 0x5c, 0x55, 0x84, 0x3d, 0x28, 0xf9, 0x55, 0xd4,
+	0xbe, 0x4a, 0x43, 0x6b, 0x93, 0x86, 0xee, 0x04, 0x9c, 0xfa, 0x51, 0x4d, 0xf8, 0x05, 0x6c, 0x1f,
+	0x8b, 0x40, 0x30, 0x2e, 0x58, 0x58, 0x7e, 0x4a, 0x6d, 0xf0, 0xd6, 0x7a, 0x57, 0x43, 0xad, 0x33,
+	0x7f, 0x0b, 0xda, 0x42, 0x14, 0x9c, 0x92, 0x4b, 0x79, 0x0b, 0xc4, 0xf4, 0xa4, 0xef, 0xe0, 0x3d,
+	0xb8, 0x92, 0x7c, 0x10, 0xa9, 0x08, 0x62, 0x35, 0xb0, 0x3a, 0x38, 0xb0, 0x6c, 0x44, 0x70, 0x62,
+	0xa9, 0x9e, 0x1e, 0x29, 0x69, 0x57, 0x8d, 0x33, 0x09, 0xa0, 0x70, 0x0f, 0x00, 0x9f, 0x8f, 0x62,
+	0x7e, 0x4f, 0x9d, 0x95, 0xc8, 0x81, 0x04, 0xee, 0xfe, 0xd5, 0x85, 0xe1, 0x31, 0x0d, 0x5e, 0x53,
+	0x1a, 0xc9, 0x79, 0x99, 0x91, 0x45, 0xc1, 0xad, 0xea, 0x37, 0x2d, 0xb9, 0xb9, 0x49, 0xa2, 0xc6,
+	0x8f, 0xe8, 0xc9, 0x27, 0xef, 0x52, 0xd3, 0xd7, 0x74, 0x89, 0x1c, 0xc1, 0xc0, 0xf8, 0x68, 0x24,
+	0xbb, 0xc6, 0xc1, 0xda, 0xb7, 0xf0, 0x64, 0xef, 0x1c, 0xa9, 0x69, 0xcd, 0x98, 0x0c, 0xa6, 0xb5,
+	0xfa, 0x2c, 0x32, 0xad, 0x35, 0x8d, 0x13, 0xb4, 0x66, 0x74, 0x7d, 0xd3, 0x5a, 0x7d, 0xce, 0x98,
+	0xd6, 0x9a, 0x46, 0x05, 0x5a, 0x33, 0x5a, 0xb3, 0x69, 0xad, 0x3e, 0x42, 0x4c, 0x6b, 0x4d, 0xfd,
+	0xfc, 0x12, 0x79, 0x01, 0x43, 0xb3, 0x4f, 0x12, 0xe3, 0x40, 0x43, 0xa3, 0x9f, 0x5c, 0x3f, 0x4f,
+	0x6c, 0x1a, 0x34, 0xdb, 0x82, 0x69, 0xb0, 0xa1, 0x31, 0x9a, 0x06, 0x9b, 0xba, 0x89, 0x7b, 0x89,
+	0x7c, 0x07, 0x5b, 0x9b, 0xcf, 0x93, 0xdc, 0xd8, 0x4c, 0xab, 0xf6, 0xea, 0x27, 0xee, 0x45, 0x2a,
+	0xa5, 0xf1, 0xa7, 0x00, 0xeb, 0x57, 0x47, 0xae, 0xad, 0xcf, 0xd4, 0x5e, 0xfd, 0x64, 0xb7, 0x59,
+	0x58, 0x98, 0x7a, 0x74, 0x1d, 0xb6, 0xb8, 0xa2, 0xfe, 0x9c, 0xcf, 0xc2, 0x98, 0xd1, 0x44, 0x3c,
+	0x02, 0x7c, 0x05, 0x5f, 0xcb, 0x3f, 0xc7, 0x93, 0x1e, 0xfe, 0x40, 0x7e, 0xf6, 0x77, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0x8d, 0x38, 0xa9, 0x9f, 0x4f, 0x0e, 0x00, 0x00,
 }

weed/pb/volume_server.proto

@@ -24,15 +24,17 @@ service VolumeServer {
 
     rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
     }
-    rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (VolumeSyncIndexResponse) {
+    rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) {
     }
-    rpc VolumeSyncData (VolumeSyncDataRequest) returns (VolumeSyncDataResponse) {
+    rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) {
     }
 
     rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
     }
     rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
     }
+    rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
+    }
 
     // rpc VolumeUiPage (VolumeUiPageRequest) returns (VolumeUiPageResponse) {}
 
@@ -104,6 +106,7 @@ message VolumeSyncStatusRequest {
 }
 message VolumeSyncStatusResponse {
     uint32 volumd_id = 1;
+    string collection = 2;
     string replication = 4;
     string ttl = 5;
     uint64 tail_offset = 6;
@@ -141,6 +144,12 @@ message VolumeUnmountRequest {
 message VolumeUnmountResponse {
 }
 
+message VolumeDeleteRequest {
+    uint32 volumd_id = 1;
+}
+message VolumeDeleteResponse {
+}
+
 message VolumeUiPageRequest {
 }
 message VolumeUiPageResponse {

weed/pb/volume_server_pb/volume_server.pb.go

@@ -35,6 +35,8 @@ It has these top-level messages:
 	VolumeMountResponse
 	VolumeUnmountRequest
 	VolumeUnmountResponse
+	VolumeDeleteRequest
+	VolumeDeleteResponse
 	VolumeUiPageRequest
 	VolumeUiPageResponse
 	DiskStatus
@@ -352,6 +354,7 @@ func (m *VolumeSyncStatusRequest) GetVolumdId() uint32 {
 
 type VolumeSyncStatusResponse struct {
 	VolumdId        uint32 `protobuf:"varint,1,opt,name=volumd_id,json=volumdId" json:"volumd_id,omitempty"`
+	Collection      string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
 	Replication     string `protobuf:"bytes,4,opt,name=replication" json:"replication,omitempty"`
 	Ttl             string `protobuf:"bytes,5,opt,name=ttl" json:"ttl,omitempty"`
 	TailOffset      uint64 `protobuf:"varint,6,opt,name=tail_offset,json=tailOffset" json:"tail_offset,omitempty"`
@@ -371,6 +374,13 @@ func (m *VolumeSyncStatusResponse) GetVolumdId() uint32 {
 	return 0
 }
 
+func (m *VolumeSyncStatusResponse) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
 func (m *VolumeSyncStatusResponse) GetReplication() string {
 	if m != nil {
 		return m.Replication
@@ -550,13 +560,37 @@ func (m *VolumeUnmountResponse) String() string            { return proto.Compac
 func (*VolumeUnmountResponse) ProtoMessage()               {}
 func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
+type VolumeDeleteRequest struct {
+	VolumdId uint32 `protobuf:"varint,1,opt,name=volumd_id,json=volumdId" json:"volumd_id,omitempty"`
+}
+
+func (m *VolumeDeleteRequest) Reset()                    { *m = VolumeDeleteRequest{} }
+func (m *VolumeDeleteRequest) String() string            { return proto.CompactTextString(m) }
+func (*VolumeDeleteRequest) ProtoMessage()               {}
+func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+
+func (m *VolumeDeleteRequest) GetVolumdId() uint32 {
+	if m != nil {
+		return m.VolumdId
+	}
+	return 0
+}
+
+type VolumeDeleteResponse struct {
+}
+
+func (m *VolumeDeleteResponse) Reset()                    { *m = VolumeDeleteResponse{} }
+func (m *VolumeDeleteResponse) String() string            { return proto.CompactTextString(m) }
+func (*VolumeDeleteResponse) ProtoMessage()               {}
+func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+
 type VolumeUiPageRequest struct {
 }
 
 func (m *VolumeUiPageRequest) Reset()                    { *m = VolumeUiPageRequest{} }
 func (m *VolumeUiPageRequest) String() string            { return proto.CompactTextString(m) }
 func (*VolumeUiPageRequest) ProtoMessage()               {}
-func (*VolumeUiPageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+func (*VolumeUiPageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
 
 type VolumeUiPageResponse struct {
 }
@@ -564,7 +598,7 @@ type VolumeUiPageResponse struct {
 func (m *VolumeUiPageResponse) Reset()                    { *m = VolumeUiPageResponse{} }
 func (m *VolumeUiPageResponse) String() string            { return proto.CompactTextString(m) }
 func (*VolumeUiPageResponse) ProtoMessage()               {}
-func (*VolumeUiPageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+func (*VolumeUiPageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
 
 type DiskStatus struct {
 	Dir  string `protobuf:"bytes,1,opt,name=dir" json:"dir,omitempty"`
@@ -576,7 +610,7 @@ type DiskStatus struct {
 func (m *DiskStatus) Reset()                    { *m = DiskStatus{} }
 func (m *DiskStatus) String() string            { return proto.CompactTextString(m) }
 func (*DiskStatus) ProtoMessage()               {}
-func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
 
 func (m *DiskStatus) GetDir() string {
 	if m != nil {
@@ -619,7 +653,7 @@ type MemStatus struct {
 func (m *MemStatus) Reset()                    { *m = MemStatus{} }
 func (m *MemStatus) String() string            { return proto.CompactTextString(m) }
 func (*MemStatus) ProtoMessage()               {}
-func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
 
 func (m *MemStatus) GetGoroutines() int32 {
 	if m != nil {
@@ -697,6 +731,8 @@ func init() {
 	proto.RegisterType((*VolumeMountResponse)(nil), "volume_server_pb.VolumeMountResponse")
 	proto.RegisterType((*VolumeUnmountRequest)(nil), "volume_server_pb.VolumeUnmountRequest")
 	proto.RegisterType((*VolumeUnmountResponse)(nil), "volume_server_pb.VolumeUnmountResponse")
+	proto.RegisterType((*VolumeDeleteRequest)(nil), "volume_server_pb.VolumeDeleteRequest")
+	proto.RegisterType((*VolumeDeleteResponse)(nil), "volume_server_pb.VolumeDeleteResponse")
 	proto.RegisterType((*VolumeUiPageRequest)(nil), "volume_server_pb.VolumeUiPageRequest")
 	proto.RegisterType((*VolumeUiPageResponse)(nil), "volume_server_pb.VolumeUiPageResponse")
 	proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus")
@@ -723,10 +759,11 @@ type VolumeServerClient interface {
 	DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
 	AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
 	VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error)
-	VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (*VolumeSyncIndexResponse, error)
-	VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (*VolumeSyncDataResponse, error)
+	VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error)
+	VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error)
 	VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error)
 	VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error)
+	VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error)
 }
 
 type volumeServerClient struct {
@@ -809,22 +846,68 @@ func (c *volumeServerClient) VolumeSyncStatus(ctx context.Context, in *VolumeSyn
 	return out, nil
 }
 
-func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (*VolumeSyncIndexResponse, error) {
-	out := new(VolumeSyncIndexResponse)
-	err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeSyncIndex", in, out, c.cc, opts...)
+func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncIndex", opts...)
 	if err != nil {
 		return nil, err
 	}
-	return out, nil
+	x := &volumeServerVolumeSyncIndexClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
 }
 
-func (c *volumeServerClient) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (*VolumeSyncDataResponse, error) {
-	out := new(VolumeSyncDataResponse)
-	err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeSyncData", in, out, c.cc, opts...)
+type VolumeServer_VolumeSyncIndexClient interface {
+	Recv() (*VolumeSyncIndexResponse, error)
+	grpc.ClientStream
+}
+
+type volumeServerVolumeSyncIndexClient struct {
+	grpc.ClientStream
+}
+
+func (x *volumeServerVolumeSyncIndexClient) Recv() (*VolumeSyncIndexResponse, error) {
+	m := new(VolumeSyncIndexResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *volumeServerClient) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncData", opts...)
 	if err != nil {
 		return nil, err
 	}
-	return out, nil
+	x := &volumeServerVolumeSyncDataClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type VolumeServer_VolumeSyncDataClient interface {
+	Recv() (*VolumeSyncDataResponse, error)
+	grpc.ClientStream
+}
+
+type volumeServerVolumeSyncDataClient struct {
+	grpc.ClientStream
+}
+
+func (x *volumeServerVolumeSyncDataClient) Recv() (*VolumeSyncDataResponse, error) {
+	m := new(VolumeSyncDataResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
 }
 
 func (c *volumeServerClient) VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error) {
@@ -845,6 +928,15 @@ func (c *volumeServerClient) VolumeUnmount(ctx context.Context, in *VolumeUnmoun
 	return out, nil
 }
 
+func (c *volumeServerClient) VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error) {
+	out := new(VolumeDeleteResponse)
+	err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeDelete", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for VolumeServer service
 
 type VolumeServerServer interface {
@@ -857,10 +949,11 @@ type VolumeServerServer interface {
 	DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
 	AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
 	VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error)
-	VolumeSyncIndex(context.Context, *VolumeSyncIndexRequest) (*VolumeSyncIndexResponse, error)
-	VolumeSyncData(context.Context, *VolumeSyncDataRequest) (*VolumeSyncDataResponse, error)
+	VolumeSyncIndex(*VolumeSyncIndexRequest, VolumeServer_VolumeSyncIndexServer) error
+	VolumeSyncData(*VolumeSyncDataRequest, VolumeServer_VolumeSyncDataServer) error
 	VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error)
 	VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error)
+	VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error)
 }
 
 func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) {
@@ -1011,40 +1104,46 @@ func _VolumeServer_VolumeSyncStatus_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }
 
-func _VolumeServer_VolumeSyncIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(VolumeSyncIndexRequest)
-	if err := dec(in); err != nil {
-		return nil, err
+func _VolumeServer_VolumeSyncIndex_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(VolumeSyncIndexRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
 	}
-	if interceptor == nil {
-		return srv.(VolumeServerServer).VolumeSyncIndex(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/volume_server_pb.VolumeServer/VolumeSyncIndex",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(VolumeServerServer).VolumeSyncIndex(ctx, req.(*VolumeSyncIndexRequest))
-	}
-	return interceptor(ctx, in, info, handler)
+	return srv.(VolumeServerServer).VolumeSyncIndex(m, &volumeServerVolumeSyncIndexServer{stream})
 }
 
-func _VolumeServer_VolumeSyncData_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(VolumeSyncDataRequest)
-	if err := dec(in); err != nil {
-		return nil, err
+type VolumeServer_VolumeSyncIndexServer interface {
+	Send(*VolumeSyncIndexResponse) error
+	grpc.ServerStream
+}
+
+type volumeServerVolumeSyncIndexServer struct {
+	grpc.ServerStream
+}
+
+func (x *volumeServerVolumeSyncIndexServer) Send(m *VolumeSyncIndexResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _VolumeServer_VolumeSyncData_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(VolumeSyncDataRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
 	}
-	if interceptor == nil {
-		return srv.(VolumeServerServer).VolumeSyncData(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/volume_server_pb.VolumeServer/VolumeSyncData",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(VolumeServerServer).VolumeSyncData(ctx, req.(*VolumeSyncDataRequest))
-	}
-	return interceptor(ctx, in, info, handler)
+	return srv.(VolumeServerServer).VolumeSyncData(m, &volumeServerVolumeSyncDataServer{stream})
+}
+
+type VolumeServer_VolumeSyncDataServer interface {
+	Send(*VolumeSyncDataResponse) error
+	grpc.ServerStream
+}
+
+type volumeServerVolumeSyncDataServer struct {
+	grpc.ServerStream
+}
+
+func (x *volumeServerVolumeSyncDataServer) Send(m *VolumeSyncDataResponse) error {
+	return x.ServerStream.SendMsg(m)
 }
 
 func _VolumeServer_VolumeMount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -1083,6 +1182,24 @@ func _VolumeServer_VolumeUnmount_Handler(srv interface{}, ctx context.Context, d
 	return interceptor(ctx, in, info, handler)
 }
 
+func _VolumeServer_VolumeDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(VolumeDeleteRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VolumeServerServer).VolumeDelete(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/volume_server_pb.VolumeServer/VolumeDelete",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VolumeServerServer).VolumeDelete(ctx, req.(*VolumeDeleteRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _VolumeServer_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "volume_server_pb.VolumeServer",
 	HandlerType: (*VolumeServerServer)(nil),
@@ -1119,14 +1236,6 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
 			MethodName: "VolumeSyncStatus",
 			Handler:    _VolumeServer_VolumeSyncStatus_Handler,
 		},
-		{
-			MethodName: "VolumeSyncIndex",
-			Handler:    _VolumeServer_VolumeSyncIndex_Handler,
-		},
-		{
-			MethodName: "VolumeSyncData",
-			Handler:    _VolumeServer_VolumeSyncData_Handler,
-		},
 		{
 			MethodName: "VolumeMount",
 			Handler:    _VolumeServer_VolumeMount_Handler,
@@ -1135,77 +1244,94 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
 			MethodName: "VolumeUnmount",
 			Handler:    _VolumeServer_VolumeUnmount_Handler,
 		},
+		{
+			MethodName: "VolumeDelete",
+			Handler:    _VolumeServer_VolumeDelete_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "VolumeSyncIndex",
+			Handler:       _VolumeServer_VolumeSyncIndex_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "VolumeSyncData",
+			Handler:       _VolumeServer_VolumeSyncData_Handler,
+			ServerStreams: true,
+		},
 	},
-	Streams:  []grpc.StreamDesc{},
 	Metadata: "volume_server.proto",
 }
 
 func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1013 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x57, 0xdf, 0x72, 0xdb, 0xc4,
-	0x17, 0x8e, 0x62, 0x3b, 0x76, 0x8e, 0xed, 0x5f, 0xfd, 0x5b, 0xa7, 0x89, 0xaa, 0x42, 0x30, 0x02,
-	0x5a, 0xa7, 0x0d, 0x61, 0x68, 0x07, 0x28, 0xc3, 0x0d, 0x90, 0x00, 0x93, 0x8b, 0x4e, 0x99, 0xcd,
-	0xb4, 0xc3, 0x0c, 0x9d, 0xf1, 0x28, 0xd2, 0x3a, 0xd9, 0x89, 0x2c, 0xa9, 0xda, 0x55, 0x26, 0xe5,
-	0x0d, 0x78, 0x04, 0xae, 0xb9, 0xe1, 0x9d, 0x78, 0x19, 0x66, 0xff, 0x48, 0xd1, 0x3f, 0x57, 0x82,
-	0xbb, 0xd5, 0xb7, 0xe7, 0x9c, 0x6f, 0xcf, 0xd9, 0xb3, 0xe7, 0xb3, 0x61, 0x7a, 0x1d, 0xfa, 0xc9,
-	0x8a, 0x2c, 0x18, 0x89, 0xaf, 0x49, 0x7c, 0x14, 0xc5, 0x21, 0x0f, 0xd1, 0xa4, 0x00, 0x2e, 0xa2,
-	0x73, 0xfb, 0x33, 0x40, 0xdf, 0x3b, 0xdc, 0xbd, 0x3c, 0x21, 0x3e, 0xe1, 0x04, 0x93, 0x37, 0x09,
-	0x61, 0x1c, 0xdd, 0x83, 0xc1, 0x92, 0xfa, 0x64, 0x41, 0x3d, 0x66, 0x1a, 0xb3, 0xce, 0x7c, 0x1b,
-	0xf7, 0xc5, 0xf7, 0xa9, 0xc7, 0xec, 0x17, 0x30, 0x2d, 0x38, 0xb0, 0x28, 0x0c, 0x18, 0x41, 0xcf,
-	0xa0, 0x1f, 0x13, 0x96, 0xf8, 0x5c, 0x39, 0x0c, 0x9f, 0xec, 0x1f, 0x95, 0xb9, 0x8e, 0x32, 0x97,
-	0xc4, 0xe7, 0x38, 0x35, 0xb7, 0x29, 0x8c, 0xf2, 0x1b, 0x68, 0x0f, 0xfa, 0x9a, 0xdb, 0x34, 0x66,
-	0xc6, 0x7c, 0x1b, 0x6f, 0x29, 0x6a, 0xb4, 0x0b, 0x5b, 0x8c, 0x3b, 0x3c, 0x61, 0xe6, 0xe6, 0xcc,
-	0x98, 0xf7, 0xb0, 0xfe, 0x42, 0x3b, 0xd0, 0x23, 0x71, 0x1c, 0xc6, 0x66, 0x47, 0x9a, 0xab, 0x0f,
-	0x84, 0xa0, 0xcb, 0xe8, 0x6f, 0xc4, 0xec, 0xce, 0x8c, 0xf9, 0x18, 0xcb, 0xb5, 0xdd, 0x87, 0xde,
-	0x0f, 0xab, 0x88, 0xbf, 0xb5, 0xbf, 0x02, 0xf3, 0x95, 0xe3, 0x26, 0xc9, 0xea, 0x95, 0x3c, 0xe3,
-	0xf1, 0x25, 0x71, 0xaf, 0xd2, 0xdc, 0xef, 0xc3, 0xb6, 0x3c, 0xb9, 0x97, 0x9e, 0x60, 0x8c, 0x07,
-	0x0a, 0x38, 0xf5, 0xec, 0x6f, 0xe1, 0x5e, 0x8d, 0xa3, 0xae, 0xc1, 0x47, 0x30, 0xbe, 0x70, 0xe2,
-	0x73, 0xe7, 0x82, 0x2c, 0x62, 0x87, 0xd3, 0x50, 0x7a, 0x1b, 0x78, 0xa4, 0x41, 0x2c, 0x30, 0xfb,
-	0x57, 0xb0, 0x0a, 0x11, 0xc2, 0x55, 0xe4, 0xb8, 0xbc, 0x0d, 0x39, 0x9a, 0xc1, 0x30, 0x8a, 0x89,
-	0xe3, 0xfb, 0xa1, 0xeb, 0x70, 0x22, 0xab, 0xd0, 0xc1, 0x79, 0xc8, 0x7e, 0x1f, 0xee, 0xd7, 0x06,
-	0x57, 0x07, 0xb4, 0x9f, 0x95, 0x4e, 0x1f, 0xae, 0x56, 0xb4, 0x15, 0xb5, 0xfd, 0x5e, 0xe5, 0xd4,
-	0xd2, 0x53, 0xc7, 0xfd, 0xba, 0xb4, 0xeb, 0x13, 0x27, 0x48, 0xa2, 0x56, 0x81, 0xcb, 0x27, 0x4e,
-	0x5d, 0xb3, 0xc8, 0x7b, 0xaa, 0x39, 0x8e, 0x43, 0xdf, 0x27, 0x2e, 0xa7, 0x61, 0x90, 0x86, 0xdd,
-	0x07, 0x70, 0x33, 0x50, 0xb7, 0x4a, 0x0e, 0xb1, 0x2d, 0x30, 0xab, 0xae, 0x3a, 0xec, 0x5f, 0x06,
-	0x4c, 0xbf, 0x63, 0x8c, 0x5e, 0x04, 0x8a, 0xb6, 0x55, 0xf9, 0x8b, 0x84, 0x9b, 0x65, 0xc2, 0xf2,
-	0xf5, 0x74, 0x2a, 0xd7, 0x23, 0x2c, 0x62, 0x12, 0xf9, 0xd4, 0x75, 0x64, 0x88, 0xae, 0x0c, 0x91,
-	0x87, 0xd0, 0x04, 0x3a, 0x9c, 0xfb, 0x66, 0x4f, 0xee, 0x88, 0xa5, 0xbd, 0x0b, 0x3b, 0xc5, 0x93,
-	0xea, 0x14, 0xbe, 0x84, 0x3d, 0x85, 0x9c, 0xbd, 0x0d, 0xdc, 0x33, 0xf9, 0x12, 0x5a, 0x15, 0xfc,
-	0x6f, 0x03, 0xcc, 0xaa, 0xa3, 0xee, 0xe0, 0xa6, 0xf6, 0xfb, 0xb7, 0xa7, 0x47, 0x1f, 0xc0, 0x90,
-	0x3b, 0xd4, 0x5f, 0x84, 0xcb, 0x25, 0x23, 0xdc, 0xdc, 0x9a, 0x19, 0xf3, 0x2e, 0x06, 0x01, 0xbd,
-	0x90, 0x08, 0x3a, 0x80, 0x89, 0xab, 0xba, 0x74, 0x11, 0x93, 0x6b, 0xca, 0x44, 0xe4, 0xbe, 0x24,
-	0xbe, 0xe3, 0xa6, 0xdd, 0xab, 0x60, 0x64, 0xc3, 0x98, 0x7a, 0x37, 0x0b, 0x39, 0x1c, 0xe4, 0xd3,
-	0x1e, 0xc8, 0x68, 0x43, 0xea, 0xdd, 0xfc, 0x48, 0x7d, 0x72, 0x26, 0x5e, 0xf8, 0x17, 0xb0, 0x7b,
-	0x9b, 0xdc, 0x69, 0xe0, 0x91, 0x9b, 0x56, 0x45, 0xf9, 0x29, 0x5f, 0x4c, 0xed, 0xa6, 0x4b, 0x72,
-	0x08, 0x88, 0x0a, 0x40, 0xf1, 0xba, 0x61, 0xc0, 0x49, 0xc0, 0x65, 0x80, 0x11, 0x9e, 0xc8, 0x1d,
-	0x41, 0x7e, 0xac, 0x70, 0xfb, 0x0f, 0x03, 0xee, 0xde, 0x46, 0x3a, 0x71, 0xb8, 0xd3, 0xaa, 0xb5,
-	0x2c, 0x18, 0x64, 0xd9, 0x6f, 0xaa, 0xbd, 0xf4, 0x5b, 0x8c, 0x3d, 0x5d, 0xbd, 0x8e, 0xdc, 0xd1,
-	0x5f, 0x75, 0x03, 0x4e, 0x90, 0x04, 0x84, 0x78, 0x6a, 0x7a, 0xaa, 0x6b, 0x18, 0x28, 0xe0, 0xd4,
-	0xb3, 0xbf, 0xc9, 0xd7, 0x46, 0x1d, 0x4d, 0xe7, 0xf8, 0x21, 0x8c, 0x6a, 0xb2, 0x1b, 0x2e, 0x73,
-	0x89, 0x7d, 0x0e, 0x48, 0x39, 0x3f, 0x0f, 0x93, 0xa0, 0xdd, 0xcc, 0xb8, 0x0b, 0xd3, 0x82, 0x8b,
-	0x6e, 0xdc, 0xa7, 0xb0, 0xa3, 0xe0, 0x97, 0xc1, 0xaa, 0x75, 0xac, 0xbd, 0xb4, 0xac, 0x99, 0x93,
-	0x8e, 0x96, 0x91, 0xbc, 0xa4, 0x3f, 0x8b, 0x19, 0xab, 0x82, 0x89, 0x57, 0x53, 0x84, 0xb5, 0xf9,
-	0x2f, 0x00, 0x27, 0x94, 0x5d, 0xa9, 0xb6, 0x17, 0xfd, 0xea, 0xd1, 0x58, 0xcf, 0x0e, 0xb1, 0x14,
-	0x88, 0xe3, 0xfb, 0xf2, 0x0e, 0xba, 0x58, 0x2c, 0x45, 0x99, 0x13, 0x46, 0x3c, 0x59, 0xfc, 0x2e,
-	0x96, 0x6b, 0x81, 0x2d, 0x63, 0xa2, 0x4a, 0xdf, 0xc5, 0x72, 0x6d, 0xff, 0x69, 0xc0, 0xf6, 0x73,
-	0xb2, 0xd2, 0x91, 0xf7, 0x01, 0x2e, 0xc2, 0x38, 0x4c, 0x38, 0x0d, 0x08, 0x93, 0x04, 0x3d, 0x9c,
-	0x43, 0xfe, 0x3b, 0x8f, 0xbc, 0x76, 0xe2, 0x2f, 0xe5, 0xed, 0x76, 0xb1, 0x5c, 0x0b, 0xec, 0x92,
-	0x38, 0x91, 0x7e, 0x5e, 0x72, 0x2d, 0x54, 0x91, 0x71, 0xc7, 0xbd, 0x92, 0xaf, 0xa9, 0x8b, 0xd5,
-	0xc7, 0x93, 0xdf, 0x01, 0x46, 0xba, 0x09, 0xa4, 0x2c, 0xa3, 0xd7, 0x30, 0xcc, 0xc9, 0x39, 0xfa,
-	0xb8, 0xaa, 0xda, 0xd5, 0x9f, 0x07, 0xd6, 0x27, 0x0d, 0x56, 0xba, 0xd8, 0x1b, 0x28, 0x80, 0xff,
-	0x57, 0xe4, 0x12, 0x3d, 0xaa, 0x7a, 0xaf, 0x13, 0x63, 0xeb, 0x71, 0x2b, 0xdb, 0x8c, 0x8f, 0xc3,
-	0xb4, 0x46, 0xff, 0xd0, 0x61, 0x43, 0x94, 0x82, 0x06, 0x5b, 0x9f, 0xb6, 0xb4, 0xce, 0x58, 0xdf,
-	0x00, 0xaa, 0x8a, 0x23, 0x7a, 0xdc, 0x18, 0xe6, 0x56, 0x7c, 0xad, 0xc3, 0x76, 0xc6, 0x6b, 0x13,
-	0x55, 0xb2, 0xd9, 0x98, 0x68, 0x41, 0x98, 0x1b, 0x13, 0x2d, 0x69, 0xf1, 0x06, 0xba, 0x82, 0x49,
-	0x59, 0x52, 0xd1, 0xc1, 0xba, 0xdf, 0x79, 0x15, 0xc5, 0xb6, 0x1e, 0xb5, 0x31, 0xcd, 0xc8, 0x16,
-	0x30, 0xca, 0x0b, 0x1f, 0xaa, 0x69, 0xba, 0x1a, 0x09, 0xb7, 0x1e, 0x34, 0x99, 0xe5, 0xb3, 0x29,
-	0x0b, 0x61, 0x5d, 0x36, 0x6b, 0x54, 0xb6, 0x2e, 0x9b, 0x75, 0xba, 0x6a, 0x6f, 0xa0, 0x4b, 0xb8,
-	0x53, 0x52, 0x18, 0x34, 0x7f, 0x57, 0x80, 0xbc, 0x76, 0x59, 0x07, 0x2d, 0x2c, 0x33, 0x26, 0x02,
-	0xff, 0x2b, 0x8e, 0x79, 0xf4, 0xf0, 0x5d, 0xee, 0x39, 0x8d, 0xb2, 0xe6, 0xcd, 0x86, 0x19, 0xcd,
-	0x6b, 0x18, 0xe6, 0xa6, 0x7b, 0xdd, 0xe0, 0xa8, 0xea, 0x45, 0xdd, 0xe0, 0xa8, 0x93, 0x88, 0x0d,
-	0x74, 0x0e, 0xe3, 0xc2, 0xbc, 0x47, 0x0f, 0xd6, 0x79, 0x16, 0x55, 0xc4, 0x7a, 0xd8, 0x68, 0x97,
-	0x72, 0x9c, 0x6f, 0xc9, 0xff, 0x44, 0x4f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x60, 0xe6, 0x57,
-	0xdf, 0x2a, 0x0d, 0x00, 0x00,
+	// 1044 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x57, 0xdd, 0x72, 0xdb, 0x44,
+	0x14, 0x8e, 0x6a, 0x3b, 0x76, 0x8e, 0x6d, 0x6a, 0xd6, 0x69, 0xa2, 0xaa, 0x10, 0x8c, 0x80, 0xd4,
+	0x69, 0x43, 0x80, 0x74, 0x80, 0x32, 0xdc, 0x00, 0x09, 0x30, 0xb9, 0xe8, 0x94, 0xd9, 0x4c, 0x3b,
+	0xcc, 0xd0, 0x19, 0x8f, 0x22, 0xad, 0x9d, 0x25, 0xb2, 0xe4, 0x6a, 0x57, 0x99, 0x94, 0x37, 0xe1,
+	0x9a, 0x1b, 0x9e, 0x8e, 0x17, 0xe0, 0x86, 0xd9, 0x1f, 0xd9, 0xfa, 0x73, 0x24, 0xe0, 0x6e, 0xf7,
+	0xec, 0x39, 0xdf, 0xf9, 0xd9, 0xa3, 0xf3, 0xad, 0x60, 0x78, 0x1d, 0xfa, 0xf1, 0x9c, 0x4c, 0x18,
+	0x89, 0xae, 0x49, 0x74, 0xb4, 0x88, 0x42, 0x1e, 0xa2, 0x41, 0x46, 0x38, 0x59, 0x5c, 0xd8, 0x9f,
+	0x00, 0xfa, 0xce, 0xe1, 0xee, 0xe5, 0x29, 0xf1, 0x09, 0x27, 0x98, 0xbc, 0x8e, 0x09, 0xe3, 0xe8,
+	0x3e, 0x74, 0xa6, 0xd4, 0x27, 0x13, 0xea, 0x31, 0xd3, 0x18, 0x35, 0xc6, 0x5b, 0xb8, 0x2d, 0xf6,
+	0x67, 0x1e, 0xb3, 0x9f, 0xc3, 0x30, 0x63, 0xc0, 0x16, 0x61, 0xc0, 0x08, 0x7a, 0x0a, 0xed, 0x88,
+	0xb0, 0xd8, 0xe7, 0xca, 0xa0, 0x7b, 0xbc, 0x77, 0x94, 0xf7, 0x75, 0xb4, 0x34, 0x89, 0x7d, 0x8e,
+	0x13, 0x75, 0x9b, 0x42, 0x2f, 0x7d, 0x80, 0x76, 0xa1, 0xad, 0x7d, 0x9b, 0xc6, 0xc8, 0x18, 0x6f,
+	0xe1, 0x4d, 0xe5, 0x1a, 0xed, 0xc0, 0x26, 0xe3, 0x0e, 0x8f, 0x99, 0x79, 0x67, 0x64, 0x8c, 0x5b,
+	0x58, 0xef, 0xd0, 0x36, 0xb4, 0x48, 0x14, 0x85, 0x91, 0xd9, 0x90, 0xea, 0x6a, 0x83, 0x10, 0x34,
+	0x19, 0xfd, 0x8d, 0x98, 0xcd, 0x91, 0x31, 0xee, 0x63, 0xb9, 0xb6, 0xdb, 0xd0, 0xfa, 0x7e, 0xbe,
+	0xe0, 0x6f, 0xec, 0x2f, 0xc1, 0x7c, 0xe9, 0xb8, 0x71, 0x3c, 0x7f, 0x29, 0x63, 0x3c, 0xb9, 0x24,
+	0xee, 0x55, 0x92, 0xfb, 0x03, 0xd8, 0x92, 0x91, 0x7b, 0x49, 0x04, 0x7d, 0xdc, 0x51, 0x82, 0x33,
+	0xcf, 0xfe, 0x06, 0xee, 0x97, 0x18, 0xea, 0x1a, 0x7c, 0x00, 0xfd, 0x99, 0x13, 0x5d, 0x38, 0x33,
+	0x32, 0x89, 0x1c, 0x4e, 0x43, 0x69, 0x6d, 0xe0, 0x9e, 0x16, 0x62, 0x21, 0xb3, 0x7f, 0x01, 0x2b,
+	0x83, 0x10, 0xce, 0x17, 0x8e, 0xcb, 0xeb, 0x38, 0x47, 0x23, 0xe8, 0x2e, 0x22, 0xe2, 0xf8, 0x7e,
+	0xe8, 0x3a, 0x9c, 0xc8, 0x2a, 0x34, 0x70, 0x5a, 0x64, 0xbf, 0x0b, 0x0f, 0x4a, 0xc1, 0x55, 0x80,
+	0xf6, 0xd3, 0x5c, 0xf4, 0xe1, 0x7c, 0x4e, 0x6b, 0xb9, 0xb6, 0xdf, 0x29, 0x44, 0x2d, 0x2d, 0x35,
+	0xee, 0x57, 0xb9, 0x53, 0x9f, 0x38, 0x41, 0xbc, 0xa8, 0x05, 0x9c, 0x8f, 0x38, 0x31, 0x5d, 0x22,
+	0xef, 0xaa, 0xe6, 0x38, 0x09, 0x7d, 0x9f, 0xb8, 0x9c, 0x86, 0x41, 0x02, 0xbb, 0x07, 0xe0, 0x2e,
+	0x85, 0xba, 0x55, 0x52, 0x12, 0xdb, 0x02, 0xb3, 0x68, 0xaa, 0x61, 0xff, 0x34, 0x60, 0xf8, 0x2d,
+	0x63, 0x74, 0x16, 0x28, 0xb7, 0xb5, 0xca, 0x9f, 0x75, 0x78, 0x27, 0xef, 0x30, 0x7f, 0x3d, 0x8d,
+	0xc2, 0xf5, 0x08, 0x8d, 0x88, 0x2c, 0x7c, 0xea, 0x3a, 0x12, 0xa2, 0x29, 0x21, 0xd2, 0x22, 0x34,
+	0x80, 0x06, 0xe7, 0xbe, 0xd9, 0x92, 0x27, 0x62, 0x69, 0xef, 0xc0, 0x76, 0x36, 0x52, 0x9d, 0xc2,
+	0x17, 0xb0, 0xab, 0x24, 0xe7, 0x6f, 0x02, 0xf7, 0x5c, 0x7e, 0x09, 0xb5, 0x0a, 0xfe, 0xb7, 0x01,
+	0x66, 0xd1, 0x50, 0x77, 0xf0, 0xff, 0xcd, 0xff, 0xdf, 0x66, 0x87, 0xde, 0x83, 0x2e, 0x77, 0xa8,
+	0x3f, 0x09, 0xa7, 0x53, 0x46, 0xb8, 0xb9, 0x39, 0x32, 0xc6, 0x4d, 0x0c, 0x42, 0xf4, 0x5c, 0x4a,
+	0xd0, 0x01, 0x0c, 0x5c, 0xd5, 0xc5, 0x93, 0x88, 0x5c, 0x53, 0x26, 0x90, 0xdb, 0x32, 0xb0, 0xbb,
+	0x6e, 0xd2, 0xdd, 0x4a, 0x8c, 0x6c, 0xe8, 0x53, 0xef, 0x66, 0x22, 0x87, 0x87, 0xfc, 0xf4, 0x3b,
+	0x12, 0xad, 0x4b, 0xbd, 0x9b, 0x1f, 0xa8, 0x4f, 0xce, 0xc5, 0x04, 0xf8, 0x1c, 0x76, 0x56, 0xc9,
+	0x9f, 0x05, 0x1e, 0xb9, 0xa9, 0x55, 0xb4, 0x1f, 0xd3, 0xc5, 0xd6, 0x66, 0xba, 0x64, 0x87, 0x80,
+	0xa8, 0x10, 0x28, 0xbf, 0x6e, 0x18, 0x70, 0x12, 0x70, 0x09, 0xd0, 0xc3, 0x03, 0x79, 0x22, 0x9c,
+	0x9f, 0x28, 0xb9, 0xfd, 0xbb, 0x01, 0xf7, 0x56, 0x48, 0xa7, 0x0e, 0x77, 0x6a, 0xb5, 0x9e, 0x05,
+	0x9d, 0x65, 0xf6, 0x77, 0xd4, 0x59, 0xb2, 0x17, 0x63, 0x51, 0x57, 0xaf, 0x21, 0x4f, 0xf4, 0xae,
+	0x6c, 0x00, 0x0a, 0x27, 0x01, 0x21, 0x9e, 0x9a, 0xae, 0xea, 0x1a, 0x3a, 0x4a, 0x70, 0xe6, 0xd9,
+	0x5f, 0xa7, 0x6b, 0xa3, 0x42, 0xd3, 0x39, 0xbe, 0x0f, 0xbd, 0x92, 0xec, 0xba, 0xd3, 0x54, 0x62,
+	0x9f, 0x01, 0x52, 0xc6, 0xcf, 0xc2, 0x38, 0xa8, 0x37, 0x53, 0xee, 0xc1, 0x30, 0x63, 0xa2, 0x1b,
+	0xfb, 0x09, 0x6c, 0x2b, 0xf1, 0x8b, 0x60, 0x5e, 0x1b, 0x6b, 0x37, 0x29, 0xeb, 0xd2, 0x48, 0xa3,
+	0x1d, 0x27, 0x4e, 0xb2, 0x04, 0x77, 0x2b, 0xd8, 0x4e, 0x12, 0x41, 0x96, 0xe3, 0x56, 0x01, 0xbf,
+	0xa0, 0x3f, 0x89, 0x79, 0xae, 0xb0, 0x56, 0xea, 0x89, 0x58, 0xab, 0xff, 0x0c, 0x70, 0x4a, 0xd9,
+	0x95, 0xfa, 0xc4, 0x44, 0xef, 0x7b, 0x34, 0xd2, 0x73, 0x4a, 0x2c, 0x85, 0xc4, 0xf1, 0x7d, 0x79,
+	0x9f, 0x4d, 0x2c, 0x96, 0xe2, 0xca, 0x62, 0x46, 0x3c, 0x79, 0x91, 0x4d, 0x2c, 0xd7, 0x42, 0x36,
+	0x8d, 0x88, 0xba, 0xc6, 0x26, 0x96, 0x6b, 0xfb, 0x0f, 0x03, 0xb6, 0x9e, 0x91, 0xb9, 0x46, 0xde,
+	0x03, 0x98, 0x85, 0x51, 0x18, 0x73, 0x1a, 0x10, 0x26, 0x1d, 0xb4, 0x70, 0x4a, 0xf2, 0xdf, 0xfd,
+	0xc8, 0x16, 0x22, 0xfe, 0x54, 0x76, 0x4a, 0x13, 0xcb, 0xb5, 0x90, 0x5d, 0x12, 0x67, 0xa1, 0x3f,
+	0x55, 0xb9, 0x16, 0x0c, 0xcc, 0xb8, 0xe3, 0x5e, 0xc9, 0x2f, 0xb3, 0x89, 0xd5, 0xe6, 0xf8, 0x2f,
+	0x80, 0x9e, 0x6e, 0x28, 0xf9, 0x04, 0x40, 0xaf, 0xa0, 0x9b, 0x7a, 0x3a, 0xa0, 0x0f, 0x8b, 0x2f,
+	0x84, 0xe2, 0x53, 0xc4, 0xfa, 0xa8, 0x42, 0x4b, 0x17, 0x7b, 0x03, 0x05, 0xf0, 0x76, 0x81, 0x9a,
+	0xd1, 0xa3, 0xa2, 0xf5, 0x3a, 0xe2, 0xb7, 0x1e, 0xd7, 0xd2, 0x5d, 0xfa, 0xe3, 0x30, 0x2c, 0xe1,
+	0x5a, 0x74, 0x58, 0x81, 0x92, 0xe1, 0x7b, 0xeb, 0xe3, 0x9a, 0xda, 0x4b, 0xaf, 0xaf, 0x01, 0x15,
+	0x89, 0x18, 0x3d, 0xae, 0x84, 0x59, 0x11, 0xbd, 0x75, 0x58, 0x4f, 0x79, 0x6d, 0xa2, 0x8a, 0xa2,
+	0x2b, 0x13, 0xcd, 0x3c, 0x02, 0x2a, 0x13, 0xcd, 0xf1, 0xfe, 0x06, 0xba, 0x82, 0x41, 0x9e, 0xbe,
+	0xd1, 0xc1, 0xba, 0x37, 0x65, 0xe1, 0x75, 0x60, 0x3d, 0xaa, 0xa3, 0xba, 0x74, 0x36, 0x81, 0x5e,
+	0x9a, 0x64, 0x51, 0x49, 0xd3, 0x95, 0x3c, 0x17, 0xac, 0xfd, 0x2a, 0xb5, 0x74, 0x36, 0x79, 0xd2,
+	0x2d, 0xcb, 0x66, 0x0d, 0xa3, 0x97, 0x65, 0xb3, 0x8e, 0xc3, 0xed, 0x0d, 0xf4, 0x2b, 0xdc, 0xcd,
+	0xb1, 0x15, 0x1a, 0xdf, 0x06, 0x90, 0xe6, 0x41, 0xeb, 0xa0, 0x86, 0x66, 0xe2, 0xe9, 0x53, 0x03,
+	0xcd, 0xe0, 0xad, 0x2c, 0x69, 0xa0, 0x87, 0xb7, 0x01, 0xa4, 0x18, 0xcf, 0x1a, 0x57, 0x2b, 0xa6,
+	0x1c, 0xbd, 0x82, 0x6e, 0x8a, 0x2d, 0xca, 0x86, 0x47, 0x91, 0x7f, 0xca, 0x86, 0x47, 0x19, 0xe5,
+	0x6c, 0xa0, 0x0b, 0xe8, 0x67, 0xf8, 0x03, 0xed, 0xaf, 0xb3, 0xcc, 0xb2, 0x92, 0xf5, 0xb0, 0x52,
+	0x2f, 0xdd, 0x64, 0x69, 0x5a, 0x41, 0x6b, 0x83, 0xcb, 0x0e, 0xc0, 0xfd, 0x2a, 0xb5, 0xc4, 0xc1,
+	0xc5, 0xa6, 0xfc, 0xc9, 0x7b, 0xf2, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xdb, 0x3c, 0x6d, 0xd7,
+	0xfb, 0x0d, 0x00, 0x00,
 }

weed/replication/sink/filersink/filer_sink.go

@@ -66,7 +66,6 @@ func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bo
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir,
 			Name:         name,
-			IsDirectory:  isDirectory,
 			IsDeleteData: deleteIncludeChunks,
 		}
 

weed/replication/sink/s3sink/s3_write.go

@@ -161,6 +161,6 @@ func (s3sink *S3Sink) buildReadSeeker(chunk *filer2.ChunkView) (io.ReadSeeker, e
 		return nil, err
 	}
 	buf := make([]byte, chunk.Size)
-	util.ReadUrl(fileUrl, chunk.Offset, int(chunk.Size), buf)
+	util.ReadUrl(fileUrl, chunk.Offset, int(chunk.Size), buf, true)
 	return bytes.NewReader(buf), nil
 }

weed/s3api/filer_util.go

@@ -109,7 +109,6 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath string, entryName string, isDirec
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    parentDirectoryPath,
 			Name:         entryName,
-			IsDirectory:  isDirectory,
 			IsDeleteData: isDeleteData,
 			IsRecursive:  isRecursive,
 		}

weed/s3api/s3api_bucket_handlers.go

@@ -3,10 +3,13 @@ package s3api
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
 	"os"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/gorilla/mux"
@@ -17,37 +20,38 @@ var (
 	OS_GID = uint32(os.Getgid())
 )
 
+type ListAllMyBucketsResult struct {
+	Buckets []*s3.Bucket `xml:"Buckets>Bucket"`
+	Owner   *s3.Owner
+}
+
 func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 
-	var response ListAllMyBucketsResponse
+	var response ListAllMyBucketsResult
 
-	entries, err := s3a.list(s3a.option.BucketsPath, "", "", false, 0)
+	entries, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
 
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
 
-	var buckets []ListAllMyBucketsEntry
+	var buckets []*s3.Bucket
 	for _, entry := range entries {
 		if entry.IsDirectory {
-			buckets = append(buckets, ListAllMyBucketsEntry{
-				Name:         entry.Name,
-				CreationDate: time.Unix(entry.Attributes.Crtime, 0),
+			buckets = append(buckets, &s3.Bucket{
+				Name:         aws.String(entry.Name),
+				CreationDate: aws.Time(time.Unix(entry.Attributes.Crtime, 0)),
 			})
 		}
 	}
 
-	response = ListAllMyBucketsResponse{
-		ListAllMyBucketsResponse: ListAllMyBucketsResult{
-			Owner: CanonicalUser{
-				ID:          "",
-				DisplayName: "",
-			},
-			Buckets: ListAllMyBucketsList{
-				Bucket: buckets,
-			},
+	response = ListAllMyBucketsResult{
+		Owner: &s3.Owner{
+			ID:          aws.String(""),
+			DisplayName: aws.String(""),
 		},
+		Buckets: buckets,
 	}
 
 	writeSuccessResponseXML(w, encodeResponse(response))

weed/s3api/s3api_handlers.go

@@ -77,6 +77,7 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	}
 	w.WriteHeader(statusCode)
 	if response != nil {
+		glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response))
 		w.Write(response)
 		w.(http.Flusher).Flush()
 	}

weed/s3api/s3api_object_handlers.go

@@ -73,7 +73,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	destUrl := fmt.Sprintf("http://%s%s/%s%s",
 		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
 
-	s3a.proxyToFiler(w, r, destUrl, passThroghResponse)
+	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
 
 }
 
@@ -86,7 +86,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 	destUrl := fmt.Sprintf("http://%s%s/%s%s",
 		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
 
-	s3a.proxyToFiler(w, r, destUrl, passThroghResponse)
+	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
 
 }
 
@@ -147,7 +147,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	responseFn(resp, w)
 }
-func passThroghResponse(proxyResonse *http.Response, w http.ResponseWriter) {
+func passThroughResponse(proxyResonse *http.Response, w http.ResponseWriter) {
 	for k, v := range proxyResonse.Header {
 		w.Header()[k] = v
 	}

weed/s3api/s3api_xsd_generated.go

@@ -539,11 +539,6 @@ type ListAllMyBucketsResponse struct {
 	ListAllMyBucketsResponse ListAllMyBucketsResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResponse"`
 }
 
-type ListAllMyBucketsResult struct {
-	Owner   CanonicalUser        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner"`
-	Buckets ListAllMyBucketsList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Buckets"`
-}
-
 type ListBucket struct {
 	Bucket         string    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"`
 	Prefix         string    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix,omitempty"`

weed/server/common.go

@@ -69,7 +69,7 @@ func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj inter
 // wrapper for writeJson - just logs errors
 func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
 	if err := writeJson(w, r, httpStatus, obj); err != nil {
-		glog.V(0).Infof("error writing JSON %s: %v", obj, err)
+		glog.V(0).Infof("error writing JSON %+v status %d: %v", obj, httpStatus, err)
 	}
 }
 func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
@@ -91,7 +91,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
 	}
 
 	debug("parsing upload file...")
-	fname, data, mimeType, pairMap, isGzipped, lastModified, _, _, pe := storage.ParseUpload(r)
+	fname, data, mimeType, pairMap, isGzipped, originalDataSize, lastModified, _, _, pe := storage.ParseUpload(r)
 	if pe != nil {
 		writeJsonError(w, r, http.StatusBadRequest, pe)
 		return
@@ -134,7 +134,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
 	m["fileName"] = fname
 	m["fid"] = assignResult.Fid
 	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
-	m["size"] = uploadResult.Size
+	m["size"] = originalDataSize
 	m["eTag"] = uploadResult.ETag
 	writeJsonQuiet(w, r, http.StatusCreated, m)
 	return

weed/server/filer_grpc_server.go

@@ -45,7 +45,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
 	lastFileName := req.StartFromFileName
 	includeLastFile := req.InclusiveStartFrom
 	for limit > 0 {
-		entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(req.Directory), lastFileName, includeLastFile, limit)
+		entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(req.Directory), lastFileName, includeLastFile, 1024)
 		if err != nil {
 			return nil, err
 		}
@@ -74,6 +74,10 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie
 			limit--
 		}
 
+		if len(resp.Entries) < 1024 {
+			break
+		}
+
 	}
 
 	return resp, nil
@@ -113,6 +117,10 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
 
 	fs.filer.DeleteChunks(garbages)
 
+	if req.Entry.Attributes == nil {
+		return nil, fmt.Errorf("can not create entry with empty attributes")
+	}
+
 	err = fs.filer.CreateEntry(&filer2.Entry{
 		FullPath: fullpath,
 		Attr:     filer2.PbToEntryAttribute(req.Entry.Attributes),
@@ -158,6 +166,8 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
 		newEntry.Attr.Uid = req.Entry.Attributes.Uid
 		newEntry.Attr.Gid = req.Entry.Attributes.Gid
 		newEntry.Attr.Mime = req.Entry.Attributes.Mime
+		newEntry.Attr.UserName = req.Entry.Attributes.UserName
+		newEntry.Attr.GroupNames = req.Entry.Attributes.GroupName
 
 	}
 

weed/server/filer_server.go

@@ -2,6 +2,7 @@ package weed_server
 
 import (
 	"net/http"
+	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra"
@@ -30,6 +31,7 @@ type FilerOption struct {
 	SecretKey          string
 	DirListingLimit    int
 	DataCenter         string
+	DefaultLevelDbDir  string
 }
 
 type FilerServer struct {
@@ -52,9 +54,16 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 
 	go fs.filer.KeepConnectedToMaster()
 
-	LoadConfiguration("filer", true)
-	LoadConfiguration("notification", false)
 	v := viper.GetViper()
+	if !LoadConfiguration("filer", false) {
+		v.Set("leveldb.enabled", true)
+		v.Set("leveldb.dir", option.DefaultLevelDbDir)
+		_, err := os.Stat(option.DefaultLevelDbDir)
+		if os.IsNotExist(err) {
+			os.MkdirAll(option.DefaultLevelDbDir, 0755)
+		}
+	}
+	LoadConfiguration("notification", false)
 
 	fs.filer.LoadConfiguration(v)
 
@@ -73,7 +82,7 @@ func (fs *FilerServer) jwt(fileId string) security.EncodedJwt {
 	return security.GenJwt(fs.secret, fileId)
 }
 
-func LoadConfiguration(configFileName string, required bool) {
+func LoadConfiguration(configFileName string, required bool) (loaded bool) {
 
 	// find a filer store
 	viper.SetConfigName(configFileName)     // name of config file (without extension)
@@ -93,7 +102,11 @@ func LoadConfiguration(configFileName string, required bool) {
 				"\nOr use this command to generate the default toml file\n"+
 				"    weed scaffold -config=%s -output=.\n\n\n",
 				configFileName, configFileName, configFileName)
+		} else {
+			return false
 		}
 	}
 
+	return true
+
 }

weed/server/filer_server_handlers_read.go

@@ -2,17 +2,17 @@ package weed_server
 
 import (
 	"io"
+	"mime"
+	"mime/multipart"
 	"net/http"
 	"net/url"
+	"path"
+	"strconv"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"mime"
-	"mime/multipart"
-	"path"
-	"strconv"
 )
 
 func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) {
@@ -50,6 +50,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 	w.Header().Set("Accept-Ranges", "bytes")
 	if r.Method == "HEAD" {
 		w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10))
+		w.Header().Set("Last-Modified", entry.Attr.Mtime.Format(http.TimeFormat))
 		return
 	}
 

weed/server/filer_server_handlers_write.go

@@ -31,33 +31,6 @@ type FilerPostResult struct {
 	Url   string `json:"url,omitempty"`
 }
 
-func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) {
-	var entry *filer2.Entry
-	entry, err = fs.filer.FindEntry(filer2.FullPath(path))
-	if err == filer2.ErrNotFound {
-		return "", "", nil
-	}
-
-	if err != nil {
-		glog.V(0).Infoln("failing to find path in filer store", path, err.Error())
-		writeJsonError(w, r, http.StatusInternalServerError, err)
-		return
-	}
-
-	if len(entry.Chunks) == 0 {
-		glog.V(1).Infof("empty entry: %s", path)
-		w.WriteHeader(http.StatusNoContent)
-	} else {
-		fileId = entry.Chunks[0].FileId
-		urlLocation, err = fs.filer.MasterClient.LookupFileId(fileId)
-		if err != nil {
-			glog.V(1).Infof("operation LookupFileId %s failed, err is %s", fileId, err.Error())
-			w.WriteHeader(http.StatusNotFound)
-		}
-	}
-	return
-}
-
 func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string, dataCenter string) (fileId, urlLocation string, err error) {
 	ar := &operation.VolumeAssignRequest{
 		Count:       1,
@@ -109,11 +82,10 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	fileId, urlLocation, err := fs.queryFileInfoByPath(w, r, r.URL.Path)
-	if err == nil && fileId == "" {
-		fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection, dataCenter)
-	}
+	fileId, urlLocation, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter)
+
 	if err != nil || fileId == "" || urlLocation == "" {
+		glog.V(0).Infof("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)
 		return
 	}
 
@@ -186,12 +158,21 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// update metadata in filer store
-	glog.V(4).Infoln("saving", path, "=>", fileId)
+	existingEntry, err := fs.filer.FindEntry(filer2.FullPath(path))
+	crTime := time.Now()
+	if err == nil && existingEntry != nil {
+		// glog.V(4).Infof("existing %s => %+v", path, existingEntry)
+		if existingEntry.IsDirectory() {
+			path += "/" + ret.Name
+		} else {
+			crTime = existingEntry.Crtime
+		}
+	}
 	entry := &filer2.Entry{
 		FullPath: filer2.FullPath(path),
 		Attr: filer2.Attr{
 			Mtime:       time.Now(),
-			Crtime:      time.Now(),
+			Crtime:      crTime,
 			Mode:        0660,
 			Uid:         OS_UID,
 			Gid:         OS_GID,
@@ -206,6 +187,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 			ETag:   etag,
 		}},
 	}
+	// glog.V(4).Infof("saving %s => %+v", path, entry)
 	if db_err := fs.filer.CreateEntry(entry); db_err != nil {
 		fs.filer.DeleteFileByFileId(fileId)
 		glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err)

weed/server/filer_ui/templates.go

@@ -43,7 +43,7 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
 	<div class="container">
 		<div class="page-header">
 			<h1>
-				<img src="/seaweedfsstatic/seaweed50x50.png"></img>
+				<a href="https://github.com/chrislusf/seaweedfs"><img src="/seaweedfsstatic/seaweed50x50.png"></img></a>
 				SeaweedFS Filer
 			</h1>
 		</div>

weed/server/master_grpc_server.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"time"
 
 	"github.com/chrislusf/raft"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -172,6 +173,7 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 		}
 	}()
 
+	ticker := time.NewTicker(5 * time.Second)
 	for {
 		select {
 		case message := <-messageChan:
@@ -179,6 +181,10 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 				glog.V(0).Infof("=> client %v: %+v", clientName, message)
 				return err
 			}
+		case <-ticker.C:
+			if !ms.Topo.IsLeader() {
+				return raft.NotLeaderError
+			}
 		case <-stopChan:
 			return nil
 		}

weed/server/master_grpc_server_volume.go

@@ -4,12 +4,18 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/chrislusf/raft"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/topology"
 )
 
 func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
+
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+
 	resp := &master_pb.LookupVolumeResponse{}
 	volumeLocations := ms.lookupVolumeId(req.VolumeIds, req.Collection)
 
@@ -33,6 +39,10 @@ func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupV
 
 func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest) (*master_pb.AssignResponse, error) {
 
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+
 	if req.Count == 0 {
 		req.Count = 1
 	}
@@ -87,6 +97,10 @@ func (ms *MasterServer) Assign(ctx context.Context, req *master_pb.AssignRequest
 
 func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.StatisticsRequest) (*master_pb.StatisticsResponse, error) {
 
+	if !ms.Topo.IsLeader() {
+		return nil, raft.NotLeaderError
+	}
+
 	if req.Replication == "" {
 		req.Replication = ms.defaultReplicaPlacement
 	}

weed/server/master_server_handlers_admin.go

@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
@@ -24,7 +25,10 @@ func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.R
 	}
 	for _, server := range collection.ListVolumeServers() {
 		err := operation.WithVolumeServerClient(server.Url(), func(client volume_server_pb.VolumeServerClient) error {
-			_, deleteErr := client.DeleteCollection(context.Background(), &volume_server_pb.DeleteCollectionRequest{
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*time.Second))
+			defer cancel()
+
+			_, deleteErr := client.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{
 				Collection: collection.Name,
 			})
 			return deleteErr

weed/server/master_ui/templates.go

@@ -14,7 +14,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
     <div class="container">
       <div class="page-header">
 	    <h1>
-	      <img src="/seaweedfsstatic/seaweed50x50.png"></img>
+          <a href="https://github.com/chrislusf/seaweedfs"><img src="/seaweedfsstatic/seaweed50x50.png"></img></a>
           SeaweedFS <small>{{ .Version }}</small>
 	    </h1>
       </div>

weed/server/raft_server.go

@@ -42,7 +42,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
 	raft.RegisterCommand(&topology.MaxVolumeIdCommand{})
 
 	var err error
-	transporter := raft.NewHTTPTransporter("/cluster", 0)
+	transporter := raft.NewHTTPTransporter("/cluster", time.Second)
 	transporter.Transport.MaxIdleConnsPerHost = 1024
 	glog.V(0).Infof("Starting RaftServer with %v", httpAddr)
 

weed/server/volume_grpc_admin.go

@@ -78,3 +78,19 @@ func (vs *VolumeServer) VolumeUnmount(ctx context.Context, req *volume_server_pb
 	return resp, err
 
 }
+
+func (vs *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
+
+	resp := &volume_server_pb.VolumeDeleteResponse{}
+
+	err := vs.store.DeleteVolume(storage.VolumeId(req.VolumdId))
+
+	if err != nil {
+		glog.Errorf("volume delete %v: %v", req, err)
+	} else {
+		glog.V(2).Infof("volume delete %v", req)
+	}
+
+	return resp, err
+
+}

weed/server/volume_grpc_client_to_master.go

@@ -27,7 +27,12 @@ func (vs *VolumeServer) heartbeat() {
 			if newLeader != "" {
 				master = newLeader
 			}
-			newLeader, err = vs.doHeartbeat(master, time.Duration(vs.pulseSeconds)*time.Second)
+			masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(master, 0)
+			if parseErr != nil {
+				glog.V(0).Infof("failed to parse master grpc %v", masterGrpcAddress)
+				continue
+			}
+			newLeader, err = vs.doHeartbeat(master, masterGrpcAddress, time.Duration(vs.pulseSeconds)*time.Second)
 			if err != nil {
 				glog.V(0).Infof("heartbeat error: %v", err)
 				time.Sleep(time.Duration(vs.pulseSeconds) * time.Second)
@@ -36,11 +41,11 @@ func (vs *VolumeServer) heartbeat() {
 	}
 }
 
-func (vs *VolumeServer) doHeartbeat(masterNode string, sleepInterval time.Duration) (newLeader string, err error) {
+func (vs *VolumeServer) doHeartbeat(masterNode, masterGrpcAddress string, sleepInterval time.Duration) (newLeader string, err error) {
 
-	grpcConection, err := util.GrpcDial(masterNode)
+	grpcConection, err := util.GrpcDial(masterGrpcAddress)
 	if err != nil {
-		return "", fmt.Errorf("fail to dial: %v", err)
+		return "", fmt.Errorf("fail to dial %s : %v", masterNode, err)
 	}
 	defer grpcConection.Close()
 
@@ -110,7 +115,7 @@ func (vs *VolumeServer) doHeartbeat(masterNode string, sleepInterval time.Durati
 				glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
 				return "", err
 			}
-		case <-doneChan:
+		case err = <-doneChan:
 			return
 		}
 	}

weed/server/volume_grpc_sync.go

@@ -25,13 +25,11 @@ func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server
 
 }
 
-func (vs *VolumeServer) VolumeSyncIndex(ctx context.Context, req *volume_server_pb.VolumeSyncIndexRequest) (*volume_server_pb.VolumeSyncIndexResponse, error) {
-
-	resp := &volume_server_pb.VolumeSyncIndexResponse{}
+func (vs *VolumeServer) VolumeSyncIndex(req *volume_server_pb.VolumeSyncIndexRequest, stream volume_server_pb.VolumeServer_VolumeSyncIndexServer) error {
 
 	v := vs.store.GetVolume(storage.VolumeId(req.VolumdId))
 	if v == nil {
-		return nil, fmt.Errorf("Not Found Volume Id %d", req.VolumdId)
+		return fmt.Errorf("Not Found Volume Id %d", req.VolumdId)
 	}
 
 	content, err := v.IndexFileContent()
@@ -42,46 +40,62 @@ func (vs *VolumeServer) VolumeSyncIndex(ctx context.Context, req *volume_server_
 		glog.V(2).Infof("sync volume %d index", req.VolumdId)
 	}
 
-	resp.IndexFileContent = content
+	const blockSizeLimit = 1024 * 1024 * 2
+	for i := 0; i < len(content); i += blockSizeLimit {
+		blockSize := len(content) - i
+		if blockSize > blockSizeLimit {
+			blockSize = blockSizeLimit
+		}
+		resp := &volume_server_pb.VolumeSyncIndexResponse{}
+		resp.IndexFileContent = content[i : i+blockSize]
+		stream.Send(resp)
+	}
 
-	return resp, nil
+	return nil
 
 }
 
-func (vs *VolumeServer) VolumeSyncData(ctx context.Context, req *volume_server_pb.VolumeSyncDataRequest) (*volume_server_pb.VolumeSyncDataResponse, error) {
-
-	resp := &volume_server_pb.VolumeSyncDataResponse{}
+func (vs *VolumeServer) VolumeSyncData(req *volume_server_pb.VolumeSyncDataRequest, stream volume_server_pb.VolumeServer_VolumeSyncDataServer) error {
 
 	v := vs.store.GetVolume(storage.VolumeId(req.VolumdId))
 	if v == nil {
-		return nil, fmt.Errorf("Not Found Volume Id %d", req.VolumdId)
+		return fmt.Errorf("Not Found Volume Id %d", req.VolumdId)
 	}
 
 	if uint32(v.SuperBlock.CompactRevision) != req.Revision {
-		return nil, fmt.Errorf("Requested Volume Revision is %d, but current revision is %d", req.Revision, v.SuperBlock.CompactRevision)
+		return fmt.Errorf("Requested Volume Revision is %d, but current revision is %d", req.Revision, v.SuperBlock.CompactRevision)
 	}
 
 	content, err := storage.ReadNeedleBlob(v.DataFile(), int64(req.Offset)*types.NeedlePaddingSize, req.Size, v.Version())
 	if err != nil {
-		return nil, fmt.Errorf("read offset:%d size:%d", req.Offset, req.Size)
+		return fmt.Errorf("read offset:%d size:%d", req.Offset, req.Size)
 	}
 
 	id, err := types.ParseNeedleId(req.NeedleId)
 	if err != nil {
-		return nil, fmt.Errorf("parsing needle id %s: %v", req.NeedleId, err)
+		return fmt.Errorf("parsing needle id %s: %v", req.NeedleId, err)
 	}
 	n := new(storage.Needle)
 	n.ParseNeedleHeader(content)
 	if id != n.Id {
-		return nil, fmt.Errorf("Expected file entry id %d, but found %d", id, n.Id)
+		return fmt.Errorf("Expected file entry id %d, but found %d", id, n.Id)
 	}
 
 	if err != nil {
 		glog.Errorf("sync volume %d data: %v", req.VolumdId, err)
 	}
 
-	resp.FileContent = content
+	const blockSizeLimit = 1024 * 1024 * 2
+	for i := 0; i < len(content); i += blockSizeLimit {
+		blockSize := len(content) - i
+		if blockSize > blockSizeLimit {
+			blockSize = blockSizeLimit
+		}
+		resp := &volume_server_pb.VolumeSyncDataResponse{}
+		resp.FileContent = content[i : i+blockSize]
+		stream.Send(resp)
+	}
 
-	return resp, nil
+	return nil
 
 }