Merge branch 'master' of https://github.com/chrislusf/seaweedfs

Revert "fix https://github.com/chrislusf/seaweedfs/issues/780"

.travis.yml

@@ -3,6 +3,7 @@ language: go
 go:
 - 1.10.x
 - 1.11.x
+- 1.12.x
 - tip
 
 before_install:

Dockerfile.go_build

@@ -1,6 +0,0 @@
-FROM cydev/go
-RUN go get github.com/chrislusf/seaweedfs/weed
-EXPOSE 8080
-EXPOSE 9333
-VOLUME /data
-ENTRYPOINT ["weed"]

README.md

@@ -1,24 +1,26 @@
 # SeaweedFS
 
+
 [![Build Status](https://travis-ci.org/chrislusf/seaweedfs.svg?branch=master)](https://travis-ci.org/chrislusf/seaweedfs)
 [![GoDoc](https://godoc.org/github.com/chrislusf/seaweedfs/weed?status.svg)](https://godoc.org/github.com/chrislusf/seaweedfs/weed)
 [![Wiki](https://img.shields.io/badge/docs-wiki-blue.svg)](https://github.com/chrislusf/seaweedfs/wiki)
+[![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs.svg?maxAge=604800)](https://hub.docker.com/r/chrislusf/seaweedfs/)
 
 ![SeaweedFS Logo](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/seaweedfs.png)
 
 <h2 align="center">Supporting SeaweedFS</h2>
 
-SeaweedFS is Apache-licensed open source project, independent project with its ongoing development made 
-possible entirely thanks to the support by these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
-If you'd like to grow SeaweedFS even stronger, please consider to 
-<a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a>.
+SeaweedFS is an independent Apache-licensed open source project with its ongoing development made 
+possible entirely thanks to the support of these awesome [backers](https://github.com/chrislusf/seaweedfs/blob/master/backers.md). 
+If you'd like to grow SeaweedFS even stronger, please consider joining our
+<a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
 
 Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
 Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
 
 Your support will be really appreciated by me and other supporters!
 
-<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsors SeaweedFS via Patreon</a></h3>
+<h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>
 
 <h4 align="center">Platinum</h4>
 
@@ -61,38 +63,41 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
 
 SeaweedFS started as an Object Store to handle small files efficiently. Instead of managing all file metadata in a central master, the central master only manages file volumes, and it lets these volume servers manage files and their metadata. This relieves concurrency pressure from the central master and spreads file metadata into volume servers, allowing faster file access (just one disk read operation).
 
-There is only a 40 bytes disk storage overhead for each file's metadata. It is so simple with O(1) disk read that you are welcome to challenge the performance with your actual use cases.
+There is only 40 bytes of disk storage overhead for each file's metadata. It is so simple with O(1) disk reads that you are welcome to challenge the performance with your actual use cases.
 
 SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
 
-SeaweedFS can work very well with just the object store. [[Filer]] is added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
+SeaweedFS can work very well with just the object store. [[Filer]] can then be added later to support directories and POSIX attributes. Filer is a separate linearly-scalable stateless server with customizable metadata stores, e.g., MySql/Postgres/Redis/Cassandra/LevelDB.
 
 ## Additional Features
-* Can choose no replication or different replication level, rack and data center aware
+* Can choose no replication or different replication levels, rack and data center aware
 * Automatic master servers failover - no single point of failure (SPOF)
 * Automatic Gzip compression depending on file mime type
-* Automatic compaction to reclaimed disk spaces after deletion or update
+* Automatic compaction to reclaim disk space after deletion or update
 * Servers in the same cluster can have different disk spaces, file systems, OS etc.
 * Adding/Removing servers does **not** cause any data re-balancing
 * Optionally fix the orientation for jpeg pictures
 * Support Etag, Accept-Range, Last-Modified, etc.
 * Support in-memory/leveldb/boltdb/btree mode tuning for memory/performance balance.
+
 ## Filer Features
 * [filer server][Filer] provide "normal" directories and files via http.
 * [mount filer][Mount] to read and write files directly as a local directory via FUSE.
 * [Amazon S3 compatible API][AmazonS3API] to access files with S3 tooling.
-* [Async Backup To Cloud][BackupToCloud] can enjoy extreme fast local access and backup to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
+* [Hadoop Compatible File System][Hadoop] to access files from Hadoop/Spark/Flink/etc jobs.
+* [Async Backup To Cloud][BackupToCloud] has extremely fast local access and backups to Amazon S3, Google Cloud Storage, Azure, BackBlaze.
 
 [Filer]: https://github.com/chrislusf/seaweedfs/wiki/Directories-and-Files
 [Mount]: https://github.com/chrislusf/seaweedfs/wiki/Mount
 [AmazonS3API]: https://github.com/chrislusf/seaweedfs/wiki/Amazon-S3-API
 [BackupToCloud]: https://github.com/chrislusf/seaweedfs/wiki/Backup-to-Cloud
+[Hadoop]: https://github.com/chrislusf/seaweedfs/wiki/Hadoop-Compatible-File-System
 
 ## Example Usage
 By default, the master node runs on port 9333, and the volume nodes run on port 8080.
-Here I will start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. I just use localhost as an example.
+Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
 
-SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in JSON or JSONP format.
+SeaweedFS uses HTTP REST operations to read, write, and delete. The responses are in JSON or JSONP format.
 
 ### Start Master Server
 
@@ -107,7 +112,6 @@ SeaweedFS uses HTTP REST operations to write, read, delete. The responses are in
 > weed volume -dir="/tmp/data2" -max=10 -mserver="localhost:9333" -port=8081 &
 ```
 
-
 ### Write File ###
 
 To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server url:
@@ -133,17 +137,17 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
 ```
 ### Save File Id ###
 
-Now you can save the `fid`, 3,01637037d6 in this case, to a database field.
+Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
 
 The number 3 at the start represents a volume id. After the comma, it's one file key, 01, and a file cookie, 637037d6.
 
 The volume id is an unsigned 32-bit integer. The file key is an unsigned 64-bit integer. The file cookie is an unsigned 32-bit integer, used to prevent URL guessing.
 
-The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as string.
+The file key and file cookie are both coded in hex. You can store the <volume id, file key, file cookie> tuple in your own format, or simply store the `fid` as a string.
 
-If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most usage would not need 2^32 volumes.
+If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most uses will not need 2^32 volumes.
 
-If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, 4-byte integer for file cookie. So 16 bytes are more than enough.
+If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
 
 ### Read File ###
 
@@ -211,7 +215,7 @@ You can also set the default replication strategy when starting the master serve
 
 ### Allocate File Key on specific data center ###
 
-Volume servers can start with a specific data center name:
+Volume servers can be started with a specific data center name:
 
 ```
  weed volume -dir=/tmp/1 -port=8080 -dataCenter=dc1
@@ -239,9 +243,9 @@ When requesting a file key, an optional "dataCenter" parameter can limit the ass
 
 Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
 
-The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, might not scale well for many concurrent users.
+The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, so it might not scale well for many concurrent users.
 
-Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is size 32GB, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is fairly small amount of data and is generally stable.
+Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
 
 The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
 
@@ -251,7 +255,7 @@ For comparison, consider that an xfs inode structure in Linux is 536 bytes.
 
 The architecture is fairly simple. The actual data is stored in volumes on storage nodes. One volume server can have multiple volumes, and can both support read and write access with basic authentication.
 
-All volumes are managed by a master server. The master server contains volume id to volume server mapping. This is fairly static information, and could be cached easily.
+All volumes are managed by a master server. The master server contains the volume id to volume server mapping. This is fairly static information, and can be easily cached.
 
 On each write request, the master server also generates a file key, which is a growing 64-bit unsigned integer. Since write requests are not generally as frequent as read requests, one master server should be able to handle the concurrency well.
 
@@ -259,28 +263,27 @@ On each write request, the master server also generates a file key, which is a g
 
 When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node url) for the file. The client then contacts the volume node and POSTs the file content.
 
-When a client needs to read a file based on (volume id, file key, file cookie), it can ask the master server by the volume id for the (volume node url, volume node public url), or retrieve this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
+When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node url, volume node public url), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
 
 Please see the example for details on the write-read process.
 
 ### Storage Size ###
 
-In the current implementation, each volume can be 8x2^32 bytes (32GiB). This is because of we align content to 8 bytes. We can easily increase this to 64G, or 128G, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
+In the current implementation, each volume can hold 32 gibibytes (32GiB or 8x2^32 bytes). This is because we align content to 8 bytes. We can easily increase this to 64GiB, or 128GiB, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
 
-There can be 2^32 volumes. So total system size is 8 x 2^32 bytes x 2^32 = 8 x 4GiB x 4Gi = 128EiB (2^67 bytes, or 128 exbibytes).
+There can be 4 gibibytes (4GiB or 2^32 bytes) of volumes. So the total system size is 8 x 4GiB x 4GiB which is 128 exbibytes (128EiB or 2^67 bytes).
 
 Each individual file size is limited to the volume size.
 
 ### Saving memory ###
 
-All file meta information on volume server is readable from memory without disk access. Each file just takes an 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own the space cost for the map. But usually the disk runs out before the memory does.
-
+All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
 ## Compared to Other File Systems ##
 
 Most other distributed file systems seem more complicated than necessary.
 
-SeaweedFS is meant to be fast and simple, both during usage and during setup. If you do not understand how it works when you reach here, we failed! Please raise an issue with any questions or update this file with clarifications.
+SeaweedFS is meant to be fast and simple, in both setup and operation. If you do not understand how it works when you reach here, we've failed! Please raise an issue with any questions or update this file with clarifications.
 
 ### Compared to HDFS ###
 
@@ -319,7 +322,7 @@ Ceph can be setup similar to SeaweedFS as a key->blob store. It is much more com
 
 SeaweedFS has a centralized master group to look up free volumes, while Ceph uses hashing and metadata servers to locate its objects. Having a centralized master makes it easy to code and manage. 
 
-Same as SeaweedFS, Ceph is also based on a object store RADOS. Ceph is rather complicated with mixed reviews.
+Same as SeaweedFS, Ceph is also based on the object store RADOS. Ceph is rather complicated with mixed reviews.
 
 Ceph uses CRUSH hashing to automatically manage the data placement. SeaweedFS places data by assigned volumes.
 
@@ -339,31 +342,31 @@ SeaweedFS Filer uses off-the-shelf stores, such as MySql, Postgres, Redis, Cassa
 More tools and documentation, on how to maintain and scale the system. For example, how to move volumes, automatically balancing data, how to grow volumes, how to check system status, etc.
 Other key features include: Erasure Encoding, JWT security.
 
-This is a super exciting project! And I need helpers and [support](https://www.patreon.com/seaweedfs)!
+This is a super exciting project! And we need helpers and [support](https://www.patreon.com/seaweedfs)!
 
 
 ## Installation guide for users who are not familiar with golang
 
-step 1: install go on your machine and setup the environment by following the instructions from the following link:
+Step 1: install go on your machine and setup the environment by following the instructions at:
 
 https://golang.org/doc/install
 
 make sure you set up your $GOPATH
 
 
-step 2: also you may need to install Mercurial by following the instructions below
+Step 2: also you may need to install Mercurial by following the instructions at:
 
 http://mercurial.selenic.com/downloads
 
-step 3: download, compile, and install the project by executing the following command
+Step 3: download, compile, and install the project by executing the following command
 
 ```bash
 go get github.com/chrislusf/seaweedfs/weed
 ```
 
-once this is done, you should see the executable "weed" under `$GOPATH/bin`
+Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
-step 4: after you modify your code locally, you could start a local build by calling `go install` under 
+Step 4: after you modify your code locally, you could start a local build by calling `go install` under 
 
 ```
 $GOPATH/src/github.com/chrislusf/seaweedfs/weed
@@ -373,7 +376,7 @@ $GOPATH/src/github.com/chrislusf/seaweedfs/weed
 
 ### Hard Drive Performance ###
 
-When testing read performance on SeaweedFS, it basically becomes performance test for your hard drive's random read speed. Hard Drive usually get 100MB/s~200MB/s.
+When testing read performance on SeaweedFS, it basically becomes a performance test of your hard drive's random read speed. Hard drives usually get 100MB/s~200MB/s.
 
 ### Solid State Disk
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM frolvlad/alpine-glibc:alpine-3.5
+FROM frolvlad/alpine-glibc
 
 # Tried to use curl only (curl -o /tmp/linux_amd64.tar.gz ...), however it turned out that the following tar command failed with "gzip: stdin: not in gzip format"
 RUN apk add --no-cache --virtual build-dependencies --update wget curl ca-certificates && \
@@ -15,8 +15,12 @@ EXPOSE 8080
 EXPOSE 18888
 # filer server http port
 EXPOSE 8888
-# master server shared gprc+http port
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
 EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
 
 VOLUME /data
 

docker/Dockerfile.go_build

@@ -0,0 +1,27 @@
+FROM golang:latest
+RUN go get github.com/chrislusf/seaweedfs/weed
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+
+VOLUME /data
+
+RUN mkdir -p /etc/seaweedfs
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+RUN cp /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+RUN cp /go/bin/weed /usr/bin/
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/README.md

@@ -0,0 +1,21 @@
+# Docker
+
+
+## Try it out
+
+```bash
+
+wget https://raw.githubusercontent.com/chrislusf/seaweedfs/master/docker/seaweedfs-compose.yml
+
+docker-compose -f seaweedfs-compose.yml -p seaweedfs up
+
+```
+
+## Development
+
+```bash
+cd $GOPATH/src/github.com/chrislusf/seaweedfs/docker
+
+docker-compose -f dev-compose.yml -p seaweedfs up
+
+```

docker/dev-compose.yml

@@ -0,0 +1,43 @@
+version: '2'
+
+services:
+  master:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 9333:9333
+    - 19333:19333
+    command: "master"
+  volume:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8080:8080
+    - 18080:18080
+    command: 'volume -max=5 -mserver="master:9333" -port=8080'
+    depends_on:
+    - master
+  filer:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8888:8888
+    - 18888:18888
+    command: 'filer -master="master:9333"'
+    depends_on:
+    - master
+    - volume
+  s3:
+    build:
+      context: .
+      dockerfile: Dockerfile.go_build
+    ports:
+    - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+    - master
+    - volume
+    - filer

docker/docker-compose.yml

@@ -1,40 +0,0 @@
-version: '2'
-
-services:
-  master:
-    #image: chrislusf/seaweedfs # use a remote image
-    build: . # build our container from the local Dockerfile
-    ports:
-    - 9333:9333
-    command: "master"
-    networks:
-      default:
-        aliases:
-        - seaweed_master
-  volume:
-    #image: chrislusf/seaweedfs # use a remote image
-    build: . # build our container from the local Dockerfile
-    ports:
-    - 8080:8080
-    - 18080:18080
-    command: 'volume -max=5 -mserver="master:9333" -port=8080'
-    depends_on:
-    - master
-    networks:
-      default:
-        aliases:
-        - seaweed_volume
-  filer:
-    #image: chrislusf/seaweedfs # use a remote image
-    build: . # build our container from the local Dockerfile
-    ports:
-    - 8888:8888
-    - 18888:18888
-    command: 'filer -master="master:9333"'
-    depends_on:
-    - master
-    - volume
-    networks:
-      default:
-        aliases:
-        - seaweed_filer

docker/entrypoint.sh

@@ -29,10 +29,19 @@ case "$1" in
   	;;
 
   'filer')
-  	ARGS="-ip `hostname -i`"
+  	ARGS="-ip `hostname -i` "
   	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
 		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
 	fi
+	mkdir -p /data/filerdb
+  	exec /usr/bin/weed $@ $ARGS
+	;;
+
+  's3')
+  	ARGS="-domainName=$S3_DOMAIN_NAME -key.file=$S3_KEY_FILE -cert.file=$S3_CERT_FILE"
+  	if [ -n "$FILER_PORT_8888_TCP_ADDR" ] ; then
+		ARGS="$ARGS -filer=$FILER_PORT_8888_TCP_ADDR:$FILER_PORT_8888_TCP_PORT"
+	fi
   	exec /usr/bin/weed $@ $ARGS
 	;;
 

docker/filer.toml

@@ -1,3 +1,3 @@
 [leveldb]
 enabled = true
-dir = "."
+dir = "/data/filerdb"

docker/seaweedfs-compose.yml

@@ -0,0 +1,37 @@
+version: '2'
+
+services:
+  master:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 9333:9333
+    - 19333:19333
+    command: "master"
+  volume:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8080:8080
+    - 18080:18080
+    command: 'volume -max=15 -mserver="master:9333" -port=8080'
+    depends_on:
+    - master
+  filer:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8888:8888
+    - 18888:18888
+    command: 'filer -master="master:9333"'
+    tty: true
+    stdin_open: true
+    depends_on:
+    - master
+    - volume
+  s3:
+    image: chrislusf/seaweedfs # use a remote image
+    ports:
+    - 8333:8333
+    command: 's3 -filer="filer:8888"'
+    depends_on:
+    - master
+    - volume
+    - filer

other/java/client/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.0</version>
+    <version>1.0.8</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -45,6 +45,22 @@
             <artifactId>grpc-stub</artifactId>
             <version>${grpc.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.25</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpmime</artifactId>
+            <version>4.5.6</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.12</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <distributionManagement>
@@ -62,6 +78,14 @@
             </extension>
         </extensions>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>7</source>
+                    <target>7</target>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.xolstice.maven.plugins</groupId>
                 <artifactId>protobuf-maven-plugin</artifactId>

other/java/client/src/main/java/seaweedfs/client/ByteBufferOutputStream.java

@@ -0,0 +1,21 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+public class ByteBufferOutputStream extends OutputStream {
+    private final ByteBuffer buf;
+
+    public ByteBufferOutputStream(ByteBuffer buf) {
+        this.buf = buf;
+    }
+
+    public void write(int b) throws IOException {
+        this.buf.put((byte)b);
+    }
+
+    public void write(byte[] b, int off, int len) throws IOException {
+        this.buf.put(b, off, len);
+    }
+}

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -0,0 +1,225 @@
+package seaweedfs.client;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class FilerClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
+
+    private FilerGrpcClient filerGrpcClient;
+
+    public FilerClient(String host, int grpcPort) {
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+    }
+
+    public FilerClient(FilerGrpcClient filerGrpcClient) {
+        this.filerGrpcClient = filerGrpcClient;
+    }
+
+    public boolean mkdirs(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return mkdirs(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean mkdirs(String path, int mode, String userName, String[] groupNames) {
+        return mkdirs(path, mode, 0, 0, userName, groupNames);
+    }
+
+    public boolean mkdirs(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        if ("/".equals(path)) {
+            return true;
+        }
+
+        mkdirs(parent, mode, uid, gid, userName, groupNames);
+
+        FilerProto.Entry existingEntry = lookupEntry(parent, name);
+
+        if (existingEntry != null) {
+            return true;
+        }
+
+        return createEntry(
+            parent,
+            newDirectoryEntry(name, mode, uid, gid, userName, groupNames).build()
+        );
+
+    }
+
+    public boolean rm(String path, boolean isRecursive) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        return deleteEntry(
+            parent,
+            name,
+            true,
+            isRecursive);
+    }
+
+    public boolean touch(String path, int mode) {
+        String currentUser = System.getProperty("user.name");
+        return touch(path, mode, 0, 0, currentUser, new String[]{});
+    }
+
+    public boolean touch(String path, int mode, int uid, int gid, String userName, String[] groupNames) {
+
+        Path pathObject = Paths.get(path);
+        String parent = pathObject.getParent().toString();
+        String name = pathObject.getFileName().toString();
+
+        FilerProto.Entry entry = lookupEntry(parent, name);
+        if (entry == null) {
+            return createEntry(
+                parent,
+                newFileEntry(name, mode, uid, gid, userName, groupNames).build()
+            );
+        }
+        long now = System.currentTimeMillis() / 1000L;
+        FilerProto.FuseAttributes.Builder attr = entry.getAttributes().toBuilder()
+            .setMtime(now)
+            .setUid(uid)
+            .setGid(gid)
+            .setUserName(userName)
+            .clearGroupName()
+            .addAllGroupName(Arrays.asList(groupNames));
+        return updateEntry(parent, entry.toBuilder().setAttributes(attr).build());
+    }
+
+    public FilerProto.Entry.Builder newDirectoryEntry(String name, int mode,
+                                                      int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+            .setName(name)
+            .setIsDirectory(true)
+            .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                .setMtime(now)
+                .setCrtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setFileMode(mode | 1 << 31)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public FilerProto.Entry.Builder newFileEntry(String name, int mode,
+                                                 int uid, int gid, String userName, String[] groupNames) {
+
+        long now = System.currentTimeMillis() / 1000L;
+
+        return FilerProto.Entry.newBuilder()
+            .setName(name)
+            .setIsDirectory(false)
+            .setAttributes(FilerProto.FuseAttributes.newBuilder()
+                .setMtime(now)
+                .setCrtime(now)
+                .setUid(uid)
+                .setGid(gid)
+                .setFileMode(mode)
+                .setUserName(userName)
+                .clearGroupName()
+                .addAllGroupName(Arrays.asList(groupNames)));
+    }
+
+    public List<FilerProto.Entry> listEntries(String path) {
+        List<FilerProto.Entry> results = new ArrayList<FilerProto.Entry>();
+        String lastFileName = "";
+        for (int limit = Integer.MAX_VALUE; limit > 0; ) {
+            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024);
+            if (t == null) {
+                break;
+            }
+            int nSize = t.size();
+            if (nSize > 0) {
+                limit -= nSize;
+                lastFileName = t.get(nSize - 1).getName();
+            }
+            results.addAll(t);
+            if (t.size() < 1024) {
+                break;
+            }
+        }
+        return results;
+    }
+
+    public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
+        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+            .setDirectory(path)
+            .setPrefix(entryPrefix)
+            .setStartFromFileName(lastEntryName)
+            .setLimit(limit)
+            .build()).getEntriesList();
+    }
+
+    public FilerProto.Entry lookupEntry(String directory, String entryName) {
+        try {
+            return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+                FilerProto.LookupDirectoryEntryRequest.newBuilder()
+                    .setDirectory(directory)
+                    .setName(entryName)
+                    .build()).getEntry();
+        } catch (Exception e) {
+            LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
+            return null;
+        }
+    }
+
+
+    public boolean createEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setEntry(entry)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean updateEntry(String parent, FilerProto.Entry entry) {
+        try {
+            filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setEntry(entry)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("createEntry {}/{}: {}", parent, entry.getName(), e);
+            return false;
+        }
+        return true;
+    }
+
+    public boolean deleteEntry(String parent, String entryName, boolean isDeleteFileChunk, boolean isRecursive) {
+        try {
+            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
+                .setDirectory(parent)
+                .setName(entryName)
+                .setIsDeleteData(isDeleteFileChunk)
+                .setIsRecursive(isRecursive)
+                .build());
+        } catch (Exception e) {
+            LOG.warn("deleteEntry {}/{}: {}", parent, entryName, e);
+            return false;
+        }
+        return true;
+    }
+
+}

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -2,7 +2,14 @@ package seaweedfs.client;
 
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
+import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
 
+import javax.net.ssl.SSLException;
+import java.io.File;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 
@@ -20,6 +27,16 @@ public class FilerGrpcClient {
         this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
     }
 
+    public FilerGrpcClient(String host, int grpcPort,
+                           String caFilePath,
+                           String clientCertFilePath,
+                           String clientPrivateKeyFilePath) throws SSLException {
+
+        this(NettyChannelBuilder.forAddress(host, grpcPort)
+                .negotiationType(NegotiationType.TLS)
+                .sslContext(buildSslContext(caFilePath,clientCertFilePath,clientPrivateKeyFilePath)));
+    }
+
     public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
         channel = channelBuilder.build();
         blockingStub = SeaweedFilerGrpc.newBlockingStub(channel);
@@ -42,4 +59,18 @@ public class FilerGrpcClient {
     public SeaweedFilerGrpc.SeaweedFilerFutureStub getFutureStub() {
         return futureStub;
     }
+
+    private static SslContext buildSslContext(String trustCertCollectionFilePath,
+                                              String clientCertChainFilePath,
+                                              String clientPrivateKeyFilePath) throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (trustCertCollectionFilePath != null) {
+            builder.trustManager(new File(trustCertCollectionFilePath));
+        }
+        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
+        }
+        return builder.build();
+    }
+
 }

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -1,16 +1,11 @@
-package seaweed.hdfs;
+package seaweedfs.client;
 
-import org.apache.hadoop.hdfs.util.ByteBufferOutputStream;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.HttpClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import seaweedfs.client.FilerGrpcClient;
-import seaweedfs.client.FilerProto;
 
 import java.io.IOException;
 import java.io.OutputStream;
@@ -23,7 +18,7 @@ import java.util.Map;
 
 public class SeaweedRead {
 
-    private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
+    // private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class);
 
     // returns bytesRead
     public static long read(FilerGrpcClient filerGrpcClient, List<VisibleInterval> visibleIntervals,
@@ -81,7 +76,7 @@ public class SeaweedRead {
             int len = (int) (chunkView.logicOffset - position + chunkView.size);
             OutputStream outputStream = new ByteBufferOutputStream(ByteBuffer.wrap(buffer, startOffset, len));
             entity.writeTo(outputStream);
-            LOG.debug("* read chunkView:{} startOffset:{} length:{}", chunkView, startOffset, len);
+            // LOG.debug("* read chunkView:{} startOffset:{} length:{}", chunkView, startOffset, len);
 
             return len;
 

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -1,21 +1,16 @@
-package seaweed.hdfs;
+package seaweedfs.client;
 
-import org.apache.hadoop.fs.Path;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.mime.HttpMultipartMode;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
-import seaweedfs.client.FilerGrpcClient;
-import seaweedfs.client.FilerProto;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 
-import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory;
-
 public class SeaweedWrite {
 
     public static void writeData(FilerProto.Entry.Builder entry,
@@ -25,40 +20,42 @@ public class SeaweedWrite {
                                  final byte[] bytes,
                                  final long bytesOffset, final long bytesLength) throws IOException {
         FilerProto.AssignVolumeResponse response = filerGrpcClient.getBlockingStub().assignVolume(
-            FilerProto.AssignVolumeRequest.newBuilder()
-                .setCollection("")
-                .setReplication(replication)
-                .setDataCenter("")
-                .setReplication("")
-                .setTtlSec(0)
-                .build());
+                FilerProto.AssignVolumeRequest.newBuilder()
+                        .setCollection("")
+                        .setReplication(replication)
+                        .setDataCenter("")
+                        .setReplication("")
+                        .setTtlSec(0)
+                        .build());
         String fileId = response.getFileId();
         String url = response.getUrl();
+        String auth = response.getAuth();
         String targetUrl = String.format("http://%s/%s", url, fileId);
 
-        String etag = multipartUpload(targetUrl, bytes, bytesOffset, bytesLength);
+        String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength);
 
         entry.addChunks(FilerProto.FileChunk.newBuilder()
-            .setFileId(fileId)
-            .setOffset(offset)
-            .setSize(bytesLength)
-            .setMtime(System.currentTimeMillis() / 10000L)
-            .setETag(etag)
+                .setFileId(fileId)
+                .setOffset(offset)
+                .setSize(bytesLength)
+                .setMtime(System.currentTimeMillis() / 10000L)
+                .setETag(etag)
         );
 
     }
 
     public static void writeMeta(final FilerGrpcClient filerGrpcClient,
-                                 final Path path, final FilerProto.Entry.Builder entry) {
+                                 final String parentDirectory, final FilerProto.Entry.Builder entry) {
         filerGrpcClient.getBlockingStub().createEntry(
-            FilerProto.CreateEntryRequest.newBuilder()
-                .setDirectory(getParentDirectory(path))
-                .setEntry(entry)
-                .build()
+                FilerProto.CreateEntryRequest.newBuilder()
+                        .setDirectory(parentDirectory)
+                        .setEntry(entry)
+                        .build()
         );
     }
 
     private static String multipartUpload(String targetUrl,
+                                          String auth,
                                           final byte[] bytes,
                                           final long bytesOffset, final long bytesLength) throws IOException {
 
@@ -67,11 +64,14 @@ public class SeaweedWrite {
         InputStream inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
 
         HttpPost post = new HttpPost(targetUrl);
+        if (auth != null && auth.length() != 0) {
+            post.addHeader("Authorization", "BEARER " + auth);
+        }
 
         post.setEntity(MultipartEntityBuilder.create()
-            .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
-            .addBinaryBody("upload", inputStream)
-            .build());
+                .setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
+                .addBinaryBody("upload", inputStream)
+                .build());
 
         try {
             HttpResponse response = client.execute(post);

other/java/client/src/main/proto/filer.proto

@@ -97,6 +97,7 @@ message FuseAttributes {
     int32 ttl_sec = 10;
     string user_name = 11; // for hdfs
     repeated string group_name = 12; // for hdfs
+    string symlink_target = 13;
 }
 
 message CreateEntryRequest {
@@ -117,7 +118,7 @@ message UpdateEntryResponse {
 message DeleteEntryRequest {
     string directory = 1;
     string name = 2;
-    bool is_directory = 3;
+    // bool is_directory = 3;
     bool is_delete_data = 4;
     bool is_recursive = 5;
 }
@@ -138,6 +139,7 @@ message AssignVolumeResponse {
     string url = 2;
     string public_url = 3;
     int32 count = 4;
+    string auth = 5;
 }
 
 message LookupVolumeRequest {

other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java

@@ -1,14 +1,11 @@
-package seaweedfs.hdfs;
+package seaweedfs.client;
 
+import org.junit.Assert;
 import org.junit.Test;
-import seaweed.hdfs.SeaweedRead;
-import seaweedfs.client.FilerProto;
 
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.junit.Assert.assertEquals;
-
 public class SeaweedReadTest {
 
     @Test
@@ -32,33 +29,33 @@ public class SeaweedReadTest {
             System.out.println("visible:" + visibleInterval);
         }
 
-        assertEquals(visibleIntervals.size(), 2);
+        Assert.assertEquals(visibleIntervals.size(), 2);
 
         SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0);
-        assertEquals(visibleInterval.start, 0);
-        assertEquals(visibleInterval.stop, 100);
-        assertEquals(visibleInterval.modifiedTime, 1000);
-        assertEquals(visibleInterval.fileId, "aaa");
+        Assert.assertEquals(visibleInterval.start, 0);
+        Assert.assertEquals(visibleInterval.stop, 100);
+        Assert.assertEquals(visibleInterval.modifiedTime, 1000);
+        Assert.assertEquals(visibleInterval.fileId, "aaa");
 
         visibleInterval = visibleIntervals.get(1);
-        assertEquals(visibleInterval.start, 100);
-        assertEquals(visibleInterval.stop, 233);
-        assertEquals(visibleInterval.modifiedTime, 2000);
-        assertEquals(visibleInterval.fileId, "bbb");
+        Assert.assertEquals(visibleInterval.start, 100);
+        Assert.assertEquals(visibleInterval.stop, 233);
+        Assert.assertEquals(visibleInterval.modifiedTime, 2000);
+        Assert.assertEquals(visibleInterval.fileId, "bbb");
 
         List<SeaweedRead.ChunkView> chunkViews = SeaweedRead.viewFromVisibles(visibleIntervals, 0, 233);
 
         SeaweedRead.ChunkView chunkView = chunkViews.get(0);
-        assertEquals(chunkView.offset, 0);
-        assertEquals(chunkView.size, 100);
-        assertEquals(chunkView.logicOffset, 0);
-        assertEquals(chunkView.fileId, "aaa");
+        Assert.assertEquals(chunkView.offset, 0);
+        Assert.assertEquals(chunkView.size, 100);
+        Assert.assertEquals(chunkView.logicOffset, 0);
+        Assert.assertEquals(chunkView.fileId, "aaa");
 
         chunkView = chunkViews.get(1);
-        assertEquals(chunkView.offset, 0);
-        assertEquals(chunkView.size, 133);
-        assertEquals(chunkView.logicOffset, 100);
-        assertEquals(chunkView.fileId, "bbb");
+        Assert.assertEquals(chunkView.offset, 0);
+        Assert.assertEquals(chunkView.size, 133);
+        Assert.assertEquals(chunkView.logicOffset, 100);
+        Assert.assertEquals(chunkView.fileId, "bbb");
 
 
     }

other/java/hdfs/pom.xml

@@ -4,9 +4,14 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
+    <properties>
+        <seaweedfs.client.version>1.0.8</seaweedfs.client.version>
+        <hadoop.version>3.1.1</hadoop.version>
+    </properties>
+
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-hadoop-client</artifactId>
-    <version>1.0</version>
+    <version>${seaweedfs.client.version}</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -41,6 +46,18 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                        <exclude>org/slf4j/**</exclude>
+                                        <exclude>META-INF/maven/org.slf4j/**</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
                             <transformers>
                                 <transformer
                                         implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
@@ -54,6 +71,14 @@
                                     <pattern>io.grpc.internal</pattern>
                                     <shadedPattern>shaded.io.grpc.internal</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons</pattern>
+                                    <shadedPattern>shaded.org.apache.commons</shadedPattern>
+                                    <excludes>
+                                        <exclude>org.apache.hadoop</exclude>
+                                        <exclude>org.apache.log4j</exclude>
+                                    </excludes>
+                                </relocation>
                             </relocations>
                         </configuration>
                     </execution>
@@ -113,10 +138,6 @@
         </plugins>
     </build>
 
-    <properties>
-        <hadoop.version>3.1.1</hadoop.version>
-    </properties>
-
     <dependencies>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -126,24 +147,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.0</version>
+            <version>${seaweedfs.client.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.httpcomponents</groupId>
-            <artifactId>httpmime</artifactId>
-            <version>4.5.6</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 
 </project>

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -5,17 +5,27 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
 
@@ -24,6 +34,9 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
+    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
+    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
     private static int BUFFER_SIZE = 16 * 1024 * 1024;
@@ -62,7 +75,17 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
+        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
+                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
+            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
+                    conf.get(FS_SEAWEED_GRPC_CA),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
+                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
+        } else {
+            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
+        }
+
     }
 
     @Override
@@ -196,8 +219,8 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
 
             UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
             return seaweedFileSystemStore.createDirectory(path, currentUser,
-                fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
-                FsPermission.getUMask(getConf()));
+                    fsPermission == null ? FsPermission.getDirDefault() : fsPermission,
+                    FsPermission.getUMask(getConf()));
 
         }
 
@@ -228,7 +251,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
      */
     @Override
     public void setOwner(Path path, final String owner, final String group)
-        throws IOException {
+            throws IOException {
         LOG.debug("setOwner path: {}", path);
         path = qualify(path);
 
@@ -259,4 +282,330 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         return path.makeQualified(uri, workingDirectory);
     }
 
+    /**
+     * Concat existing files together.
+     *
+     * @param trg   the path to the target destination.
+     * @param psrcs the paths to the sources to use for the concatenation.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default).
+     */
+    @Override
+    public void concat(final Path trg, final Path[] psrcs) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+                getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    /**
+     * Truncate the file in the indicated path to the indicated size.
+     * <ul>
+     * <li>Fails if path is a directory.</li>
+     * <li>Fails if path does not exist.</li>
+     * <li>Fails if path is not closed.</li>
+     * <li>Fails if new size is greater than current size.</li>
+     * </ul>
+     *
+     * @param f         The path to the file to be truncated
+     * @param newLength The size the file is to be truncated to
+     * @return <code>true</code> if the file has been truncated to the desired
+     * <code>newLength</code> and is immediately available to be reused for
+     * write operations such as <code>append</code>, or
+     * <code>false</code> if a background process of adjusting the length of
+     * the last block has been started, and clients should wait for it to
+     * complete before proceeding with further file updates.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default).
+     */
+    @Override
+    public boolean truncate(Path f, long newLength) throws IOException {
+        throw new UnsupportedOperationException("Not implemented by the " +
+                getClass().getSimpleName() + " FileSystem implementation");
+    }
+
+    @Override
+    public void createSymlink(final Path target, final Path link,
+                              final boolean createParent) throws AccessControlException,
+            FileAlreadyExistsException, FileNotFoundException,
+            ParentNotDirectoryException, UnsupportedFileSystemException,
+            IOException {
+        // Supporting filesystems should override this method
+        throw new UnsupportedOperationException(
+                "Filesystem does not support symlinks!");
+    }
+
+    public boolean supportsSymlinks() {
+        return false;
+    }
+
+    /**
+     * Create a snapshot.
+     *
+     * @param path         The directory where snapshots will be taken.
+     * @param snapshotName The name of the snapshot
+     * @return the snapshot path.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     */
+    @Override
+    public Path createSnapshot(Path path, String snapshotName)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support createSnapshot");
+    }
+
+    /**
+     * Rename a snapshot.
+     *
+     * @param path            The directory path where the snapshot was taken
+     * @param snapshotOldName Old name of the snapshot
+     * @param snapshotNewName New name of the snapshot
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void renameSnapshot(Path path, String snapshotOldName,
+                               String snapshotNewName) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support renameSnapshot");
+    }
+
+    /**
+     * Delete a snapshot of a directory.
+     *
+     * @param path         The directory that the to-be-deleted snapshot belongs to
+     * @param snapshotName The name of the snapshot
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void deleteSnapshot(Path path, String snapshotName)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support deleteSnapshot");
+    }
+
+    /**
+     * Modifies ACL entries of files and directories.  This method can add new ACL
+     * entries or modify the permissions on existing ACL entries.  All existing
+     * ACL entries that are not specified in this call are retained without
+     * changes.  (Modifications are merged into the current ACL.)
+     *
+     * @param path    Path to modify
+     * @param aclSpec List&lt;AclEntry&gt; describing modifications
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void modifyAclEntries(Path path, List<AclEntry> aclSpec)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support modifyAclEntries");
+    }
+
+    /**
+     * Removes ACL entries from files and directories.  Other ACL entries are
+     * retained.
+     *
+     * @param path    Path to modify
+     * @param aclSpec List describing entries to remove
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeAclEntries(Path path, List<AclEntry> aclSpec)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeAclEntries");
+    }
+
+    /**
+     * Removes all default ACL entries from files and directories.
+     *
+     * @param path Path to modify
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeDefaultAcl(Path path)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeDefaultAcl");
+    }
+
+    /**
+     * Removes all but the base ACL entries of files and directories.  The entries
+     * for user, group, and others are retained for compatibility with permission
+     * bits.
+     *
+     * @param path Path to modify
+     * @throws IOException                   if an ACL could not be removed
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeAcl(Path path)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeAcl");
+    }
+
+    /**
+     * Fully replaces ACL of files and directories, discarding all existing
+     * entries.
+     *
+     * @param path    Path to modify
+     * @param aclSpec List describing modifications, which must include entries
+     *                for user, group, and others for compatibility with permission bits.
+     * @throws IOException                   if an ACL could not be modified
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void setAcl(Path path, List<AclEntry> aclSpec) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support setAcl");
+    }
+
+    /**
+     * Gets the ACL of a file or directory.
+     *
+     * @param path Path to get
+     * @return AclStatus describing the ACL of the file or directory
+     * @throws IOException                   if an ACL could not be read
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public AclStatus getAclStatus(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getAclStatus");
+    }
+
+    /**
+     * Set an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path  Path to modify
+     * @param name  xattr name.
+     * @param value xattr value.
+     * @param flag  xattr set flag
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void setXAttr(Path path, String name, byte[] value,
+                         EnumSet<XAttrSetFlag> flag) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support setXAttr");
+    }
+
+    /**
+     * Get an xattr name and value for a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attribute
+     * @param name xattr name.
+     * @return byte[] xattr value.
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public byte[] getXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getXAttr");
+    }
+
+    /**
+     * Get all of the xattr name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattrs name/value pairs for a file or directory.
+     * Only those xattrs which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path  Path to get extended attributes
+     * @param names XAttr names.
+     * @return Map describing the XAttrs of the file or directory
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public Map<String, byte[]> getXAttrs(Path path, List<String> names)
+            throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support getXAttrs");
+    }
+
+    /**
+     * Get all of the xattr names for a file or directory.
+     * Only those xattr names which the logged-in user has permissions to view
+     * are returned.
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to get extended attributes
+     * @return List{@literal <String>} of the XAttr names of the file or directory
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public List<String> listXAttrs(Path path) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support listXAttrs");
+    }
+
+    /**
+     * Remove an xattr of a file or directory.
+     * The name must be prefixed with the namespace followed by ".". For example,
+     * "user.attr".
+     * <p>
+     * Refer to the HDFS extended attributes user documentation for details.
+     *
+     * @param path Path to remove extended attribute
+     * @param name xattr name
+     * @throws IOException                   IO failure
+     * @throws UnsupportedOperationException if the operation is unsupported
+     *                                       (default outcome).
+     */
+    @Override
+    public void removeXAttr(Path path, String name) throws IOException {
+        throw new UnsupportedOperationException(getClass().getSimpleName()
+                + " doesn't support removeXAttr");
+    }
+
 }

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -7,9 +7,12 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import seaweedfs.client.FilerClient;
 import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedRead;
 
+import javax.net.ssl.SSLException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -23,10 +26,19 @@ public class SeaweedFileSystemStore {
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystemStore.class);
 
     private FilerGrpcClient filerGrpcClient;
+    private FilerClient filerClient;
 
     public SeaweedFileSystemStore(String host, int port) {
         int grpcPort = 10000 + port;
         filerGrpcClient = new FilerGrpcClient(host, grpcPort);
+        filerClient = new FilerClient(filerGrpcClient);
+    }
+
+    public SeaweedFileSystemStore(String host, int port,
+                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
+        int grpcPort = 10000 + port;
+        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
+        filerClient = new FilerClient(filerGrpcClient);
     }
 
     public static String getParentDirectory(Path path) {
@@ -49,23 +61,12 @@ public class SeaweedFileSystemStore {
             permission,
             umask);
 
-        long now = System.currentTimeMillis() / 1000L;
-
-        FilerProto.CreateEntryRequest.Builder request = FilerProto.CreateEntryRequest.newBuilder()
-            .setDirectory(getParentDirectory(path))
-            .setEntry(FilerProto.Entry.newBuilder()
-                .setName(path.getName())
-                .setIsDirectory(true)
-                .setAttributes(FilerProto.FuseAttributes.newBuilder()
-                    .setMtime(now)
-                    .setCrtime(now)
-                    .setFileMode(permissionToMode(permission, true))
-                    .setUserName(currentUser.getUserName())
-                    .addAllGroupName(Arrays.asList(currentUser.getGroupNames())))
-            );
-
-        FilerProto.CreateEntryResponse response = filerGrpcClient.getBlockingStub().createEntry(request.build());
-        return true;
+        return filerClient.mkdirs(
+            path.toUri().getPath(),
+            permissionToMode(permission, true),
+            currentUser.getUserName(),
+            currentUser.getGroupNames()
+        );
     }
 
     public FileStatus[] listEntries(final Path path) {
@@ -73,7 +74,7 @@ public class SeaweedFileSystemStore {
 
         List<FileStatus> fileStatuses = new ArrayList<FileStatus>();
 
-        List<FilerProto.Entry> entries = lookupEntries(path);
+        List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
 
         for (FilerProto.Entry entry : entries) {
 
@@ -84,16 +85,6 @@ public class SeaweedFileSystemStore {
         return fileStatuses.toArray(new FileStatus[0]);
     }
 
-    private List<FilerProto.Entry> lookupEntries(Path path) {
-
-        LOG.debug("listEntries path: {}", path);
-
-        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
-            .setDirectory(path.toUri().getPath())
-            .setLimit(100000)
-            .build()).getEntriesList();
-    }
-
     public FileStatus getFileStatus(final Path path) {
 
         FilerProto.Entry entry = lookupEntry(path);
@@ -106,32 +97,24 @@ public class SeaweedFileSystemStore {
         return fileStatus;
     }
 
-    public boolean deleteEntries(final Path path, boolean isDirectroy, boolean recursive) {
+    public boolean deleteEntries(final Path path, boolean isDirectory, boolean recursive) {
         LOG.debug("deleteEntries path: {} isDirectory {} recursive: {}",
             path,
-            String.valueOf(isDirectroy),
+            String.valueOf(isDirectory),
             String.valueOf(recursive));
 
         if (path.isRoot()) {
             return true;
         }
 
-        if (recursive && isDirectroy) {
-            List<FilerProto.Entry> entries = lookupEntries(path);
+        if (recursive && isDirectory) {
+            List<FilerProto.Entry> entries = filerClient.listEntries(path.toUri().getPath());
             for (FilerProto.Entry entry : entries) {
-                deleteEntries(new Path(path, entry.getName()), entry.getIsDirectory(), recursive);
+                deleteEntries(new Path(path, entry.getName()), entry.getIsDirectory(), true);
             }
         }
 
-        FilerProto.DeleteEntryResponse response =
-            filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
-                .setDirectory(getParentDirectory(path))
-                .setName(path.getName())
-                .setIsDirectory(isDirectroy)
-                .setIsDeleteData(true)
-                .build());
-
-        return true;
+        return filerClient.deleteEntry(getParentDirectory(path), path.getName(), true, recursive);
     }
 
     private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) {
@@ -151,18 +134,8 @@ public class SeaweedFileSystemStore {
 
     private FilerProto.Entry lookupEntry(Path path) {
 
-        String directory = getParentDirectory(path);
+        return filerClient.lookupEntry(getParentDirectory(path), path.getName());
 
-        try {
-            FilerProto.LookupDirectoryEntryResponse response =
-                filerGrpcClient.getBlockingStub().lookupDirectoryEntry(FilerProto.LookupDirectoryEntryRequest.newBuilder()
-                    .setDirectory(directory)
-                    .setName(path.getName())
-                    .build());
-            return response.getEntry();
-        } catch (io.grpc.StatusRuntimeException e) {
-            return null;
-        }
     }
 
     public void rename(Path source, Path destination) {
@@ -186,9 +159,16 @@ public class SeaweedFileSystemStore {
 
         LOG.debug("moveEntry: {}/{}  => {}", oldParent, entry.getName(), destination);
 
+        FilerProto.Entry.Builder newEntry = entry.toBuilder().setName(destination.getName());
+        boolean isDirectoryCreated = filerClient.createEntry(getParentDirectory(destination), newEntry.build());
+
+        if (!isDirectoryCreated) {
+            return false;
+        }
+
         if (entry.getIsDirectory()) {
             Path entryPath = new Path(oldParent, entry.getName());
-            List<FilerProto.Entry> entries = lookupEntries(entryPath);
+            List<FilerProto.Entry> entries = filerClient.listEntries(entryPath.toUri().getPath());
             for (FilerProto.Entry ent : entries) {
                 boolean isSucess = moveEntry(entryPath, ent, new Path(destination, ent.getName()));
                 if (!isSucess) {
@@ -197,20 +177,9 @@ public class SeaweedFileSystemStore {
             }
         }
 
-        FilerProto.Entry.Builder newEntry = entry.toBuilder().setName(destination.getName());
-        filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder()
-            .setDirectory(getParentDirectory(destination))
-            .setEntry(newEntry)
-            .build());
+        return filerClient.deleteEntry(
+            oldParent.toUri().getPath(), entry.getName(), false, false);
 
-        filerGrpcClient.getBlockingStub().deleteEntry(FilerProto.DeleteEntryRequest.newBuilder()
-            .setDirectory(oldParent.toUri().getPath())
-            .setName(entry.getName())
-            .setIsDirectory(entry.getIsDirectory())
-            .setIsDeleteData(false)
-            .build());
-
-        return true;
     }
 
     public OutputStream createFile(final Path path,
@@ -253,6 +222,7 @@ public class SeaweedFileSystemStore {
                     .setCrtime(now)
                     .setMtime(now)
                     .setUserName(userGroupInformation.getUserName())
+                    .clearGroupName()
                     .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames()))
                 );
         }
@@ -306,10 +276,7 @@ public class SeaweedFileSystemStore {
 
         LOG.debug("setOwner path:{} entry:{}", path, entryBuilder);
 
-        filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
-            .setDirectory(getParentDirectory(path))
-            .setEntry(entryBuilder)
-            .build());
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
 
     }
 
@@ -332,10 +299,7 @@ public class SeaweedFileSystemStore {
 
         LOG.debug("setPermission path:{} entry:{}", path, entryBuilder);
 
-        filerGrpcClient.getBlockingStub().updateEntry(FilerProto.UpdateEntryRequest.newBuilder()
-            .setDirectory(getParentDirectory(path))
-            .setEntry(entryBuilder)
-            .build());
+        filerClient.updateEntry(getParentDirectory(path), entryBuilder.build());
 
     }
 }

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedInputStream.java

@@ -10,6 +10,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedRead;
 
 import java.io.EOFException;
 import java.io.IOException;

other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedOutputStream.java

@@ -11,6 +11,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
+import seaweedfs.client.SeaweedWrite;
 
 import java.io.IOException;
 import java.io.InterruptedIOException;
@@ -24,6 +25,8 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import static seaweed.hdfs.SeaweedFileSystemStore.getParentDirectory;
+
 public class SeaweedOutputStream extends OutputStream implements Syncable, StreamCapabilities {
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedOutputStream.class);
@@ -79,7 +82,7 @@ public class SeaweedOutputStream extends OutputStream implements Syncable, Strea
         LOG.debug("SeaweedWrite.writeMeta path: {} entry:{}", path, entry);
 
         try {
-            SeaweedWrite.writeMeta(filerGrpcClient, path, entry);
+            SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry);
         } catch (Exception ex) {
             throw new IOException(ex);
         }

unmaintained/fix_dat/fix_dat.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var (
@@ -63,7 +63,7 @@ func main() {
 
 	iterateEntries(datFile, indexFile, func(n *storage.Needle, offset int64) {
 		fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
-		s, _, e := n.Append(newDatFile, superBlock.Version())
+		_, s, _, e := n.Append(newDatFile, superBlock.Version())
 		fmt.Printf("size %d error %v\n", s, e)
 	})
 
@@ -77,13 +77,13 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *storage.Needl
 	readerOffset += int64(count)
 
 	// start to read dat file
-	superblock, err := storage.ReadSuperBlock(datFile)
+	superBlock, err := storage.ReadSuperBlock(datFile)
 	if err != nil {
 		fmt.Printf("cannot read dat file super block: %v", err)
 		return
 	}
-	offset := int64(superblock.BlockSize())
-	version := superblock.Version()
+	offset := int64(superBlock.BlockSize())
+	version := superBlock.Version()
 	n, rest, err := storage.ReadNeedleHeader(datFile, version, offset)
 	if err != nil {
 		fmt.Printf("cannot read needle header: %v", err)

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -4,6 +4,9 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
 	"log"
 	"math/rand"
 
@@ -19,8 +22,11 @@ var (
 func main() {
 	flag.Parse()
 
+	weed_server.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	for i := 0; i < *repeat; i++ {
-		assignResult, err := operation.Assign(*master, &operation.VolumeAssignRequest{Count: 1})
+		assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{Count: 1})
 		if err != nil {
 			log.Fatalf("assign: %v", err)
 		}
@@ -29,14 +35,14 @@ func main() {
 		rand.Read(data)
 		reader := bytes.NewReader(data)
 
-		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url,assignResult.Fid)
+		targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
 
-		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, "")
+		_, err = operation.Upload(targetUrl, fmt.Sprintf("test%d", i), reader, false, "", nil, assignResult.Auth)
 		if err != nil {
 			log.Fatalf("upload: %v", err)
 		}
 
-		util.Delete(targetUrl, "")
+		util.Delete(targetUrl, string(assignResult.Auth))
 
 		util.Get(fmt.Sprintf("http://%s/vol/vacuum", *master))
 

unmaintained/see_dat/see_dat.go

@@ -12,21 +12,31 @@ var (
 	volumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 )
 
+type VolumeFileScanner4SeeDat struct {
+	version storage.Version
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
+	return false
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *storage.Needle, offset int64) error {
+	glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x", *volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
+	return nil
+}
+
 func main() {
 	flag.Parse()
 
-	var version storage.Version
 	vid := storage.VolumeId(*volumeId)
-	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, false, func(n *storage.Needle, offset int64) error {
-			glog.V(0).Infof("%d,%s%x offset %d size %d cookie %x",
-				*volumeId, n.Id, n.Cookie, offset, n.Size, n.Cookie)
-			return nil
-		})
+
+	scanner := &VolumeFileScanner4SeeDat{}
+	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
 	if err != nil {
 		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
 	}

weed/command/backup.go

@@ -2,6 +2,9 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
 
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/storage"
@@ -38,14 +41,18 @@ var cmdBackup = &Command{
 	This will help to backup future new volumes.
 	
 	Usually backing up is just copying the .dat (and .idx) files.
-	But it's tricky to incremententally copy the differences.
+	But it's tricky to incrementally copy the differences.
 	
 	The complexity comes when there are multiple addition, deletion and compaction.
-	This tool will handle them correctly and efficiently, avoiding unnecessary data transporation.
+	This tool will handle them correctly and efficiently, avoiding unnecessary data transportation.
   `,
 }
 
 func runBackup(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	if *s.volumeId == -1 {
 		return false
 	}
@@ -59,7 +66,7 @@ func runBackup(cmd *Command, args []string) bool {
 	}
 	volumeServer := lookup.Locations[0].Url
 
-	stats, err := operation.GetVolumeSyncStatus(volumeServer, uint32(vid))
+	stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
 	if err != nil {
 		fmt.Printf("Error get volume %d status: %v\n", vid, err)
 		return true
@@ -81,7 +88,7 @@ func runBackup(cmd *Command, args []string) bool {
 		return true
 	}
 
-	if err := v.Synchronize(volumeServer); err != nil {
+	if err := v.Synchronize(volumeServer, grpcDialOption); err != nil {
 		fmt.Printf("Error synchronizing volume %d: %v\n", vid, err)
 		return true
 	}

weed/command/benchmark.go

@@ -4,6 +4,9 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 	"io"
 	"math"
 	"math/rand"
@@ -35,17 +38,18 @@ type BenchmarkOptions struct {
 	collection       *string
 	cpuprofile       *string
 	maxCpu           *int
-	secretKey        *string
+	grpcDialOption   grpc.DialOption
 }
 
 var (
 	b            BenchmarkOptions
 	sharedBytes  []byte
 	masterClient *wdclient.MasterClient
+	isSecure     bool
 )
 
 func init() {
-	cmdBenchmark.Run = runbenchmark // break init cycle
+	cmdBenchmark.Run = runBenchmark // break init cycle
 	cmdBenchmark.IsDebug = cmdBenchmark.Flag.Bool("debug", false, "verbose debug information")
 	b.masters = cmdBenchmark.Flag.String("master", "localhost:9333", "SeaweedFS master location")
 	b.concurrency = cmdBenchmark.Flag.Int("c", 16, "number of concurrent write or read processes")
@@ -59,7 +63,6 @@ func init() {
 	b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
 	b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
 	b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
-	b.secretKey = cmdBenchmark.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	sharedBytes = make([]byte, 1024)
 }
 
@@ -101,7 +104,11 @@ var (
 	readStats  *stats
 )
 
-func runbenchmark(cmd *Command, args []string) bool {
+func runBenchmark(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+	b.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
 	if *b.maxCpu < 1 {
 		*b.maxCpu = runtime.NumCPU()
@@ -116,22 +123,22 @@ func runbenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	masterClient = wdclient.NewMasterClient(context.Background(), "benchmark", strings.Split(*b.masters, ","))
+	masterClient = wdclient.NewMasterClient(context.Background(), b.grpcDialOption, "client", strings.Split(*b.masters, ","))
 	go masterClient.KeepConnectedToMaster()
 	masterClient.WaitUntilConnected()
 
 	if *b.write {
-		bench_write()
+		benchWrite()
 	}
 
 	if *b.read {
-		bench_read()
+		benchRead()
 	}
 
 	return true
 }
 
-func bench_write() {
+func benchWrite() {
 	fileIdLineChan := make(chan string)
 	finishChan := make(chan bool)
 	writeStats = newStats(*b.concurrency)
@@ -158,7 +165,7 @@ func bench_write() {
 	writeStats.printStats()
 }
 
-func bench_read() {
+func benchRead() {
 	fileIdLineChan := make(chan string)
 	finishChan := make(chan bool)
 	readStats = newStats(*b.concurrency)
@@ -188,7 +195,6 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 	defer wait.Done()
 	delayedDeleteChan := make(chan *delayedFile, 100)
 	var waitForDeletions sync.WaitGroup
-	secret := security.Secret(*b.secretKey)
 
 	for i := 0; i < 7; i++ {
 		waitForDeletions.Add(1)
@@ -198,8 +204,11 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 				if df.enterTime.After(time.Now()) {
 					time.Sleep(df.enterTime.Sub(time.Now()))
 				}
-				if e := util.Delete("http://"+df.fp.Server+"/"+df.fp.Fid,
-					security.GenJwt(secret, df.fp.Fid)); e == nil {
+				var jwtAuthorization security.EncodedJwt
+				if isSecure {
+					jwtAuthorization = operation.LookupJwt(masterClient.GetMaster(), df.fp.Fid)
+				}
+				if e := util.Delete(fmt.Sprintf("http://%s/%s", df.fp.Server, df.fp.Fid), string(jwtAuthorization)); e == nil {
 					s.completed++
 				} else {
 					s.failed++
@@ -213,14 +222,21 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
 	for id := range idChan {
 		start := time.Now()
 		fileSize := int64(*b.fileSize + random.Intn(64))
-		fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
+		fp := &operation.FilePart{
+			Reader:   &FakeReader{id: uint64(id), size: fileSize},
+			FileSize: fileSize,
+			MimeType: "image/bench", // prevent gzip benchmark content
+		}
 		ar := &operation.VolumeAssignRequest{
 			Count:      1,
 			Collection: *b.collection,
 		}
-		if assignResult, err := operation.Assign(masterClient.GetMaster(), ar); err == nil {
+		if assignResult, err := operation.Assign(masterClient.GetMaster(), b.grpcDialOption, ar); err == nil {
 			fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection
-			if _, err := fp.Upload(0, masterClient.GetMaster(), secret); err == nil {
+			if !isSecure && assignResult.Auth != "" {
+				isSecure = true
+			}
+			if _, err := fp.Upload(0, masterClient.GetMaster(), assignResult.Auth, b.grpcDialOption); err == nil {
 				if random.Intn(100) < *b.deletePercentage {
 					s.total++
 					delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}

weed/command/export.go

@@ -84,6 +84,62 @@ func printNeedle(vid storage.VolumeId, n *storage.Needle, version storage.Versio
 	)
 }
 
+type VolumeFileScanner4Export struct {
+	version   storage.Version
+	counter   int
+	needleMap *storage.NeedleMap
+	vid       storage.VolumeId
+}
+
+func (scanner *VolumeFileScanner4Export) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4Export) ReadNeedleBody() bool {
+	return true
+}
+
+func (scanner *VolumeFileScanner4Export) VisitNeedle(n *storage.Needle, offset int64) error {
+	needleMap := scanner.needleMap
+	vid := scanner.vid
+
+	nv, ok := needleMap.Get(n.Id)
+	glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
+		n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped(), ok, nv)
+	if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
+		if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
+			glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
+				n.LastModified, newerThanUnix)
+			return nil
+		}
+		scanner.counter++
+		if *limit > 0 && scanner.counter > *limit {
+			return io.EOF
+		}
+		if tarOutputFile != nil {
+			return writeFile(vid, n)
+		} else {
+			printNeedle(vid, n, scanner.version, false)
+			return nil
+		}
+	}
+	if !ok {
+		if *showDeleted && tarOutputFile == nil {
+			if n.DataSize > 0 {
+				printNeedle(vid, n, scanner.version, true)
+			} else {
+				n.Name = []byte("*tombstone")
+				printNeedle(vid, n, scanner.version, true)
+			}
+		}
+		glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
+	} else {
+		glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
+	}
+	return nil
+}
+
 func runExport(cmd *Command, args []string) bool {
 
 	var err error
@@ -145,55 +201,16 @@ func runExport(cmd *Command, args []string) bool {
 		glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err)
 	}
 
-	var version storage.Version
+	volumeFileScanner := &VolumeFileScanner4Export{
+		needleMap: needleMap,
+		vid:       vid,
+	}
 
 	if tarOutputFile == nil {
 		fmt.Printf("key\tname\tsize\tgzip\tmime\tmodified\tttl\tdeleted\n")
 	}
 
-	var counter = 0
-
-	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, true, func(n *storage.Needle, offset int64) error {
-			nv, ok := needleMap.Get(n.Id)
-			glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v",
-				n.Id, offset, n.Size, n.DiskSize(version), n.IsGzipped(), ok, nv)
-			if ok && nv.Size > 0 && int64(nv.Offset)*types.NeedlePaddingSize == offset {
-				if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) {
-					glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d",
-						n.LastModified, newerThanUnix)
-					return nil
-				}
-				counter++
-				if *limit > 0 && counter > *limit {
-					return io.EOF
-				}
-				if tarOutputFile != nil {
-					return writeFile(vid, n)
-				} else {
-					printNeedle(vid, n, version, false)
-					return nil
-				}
-			}
-			if !ok {
-				if *showDeleted && tarOutputFile == nil {
-					if n.DataSize > 0 {
-						printNeedle(vid, n, version, true)
-					} else {
-						n.Name = []byte("*tombstone")
-						printNeedle(vid, n, version, true)
-					}
-				}
-				glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size)
-			} else {
-				glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size)
-			}
-			return nil
-		})
+	err = storage.ScanVolumeFile(*export.dir, *export.collection, vid, storage.NeedleMapInMemory, volumeFileScanner)
 	if err != nil && err != io.EOF {
 		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
 	}

weed/command/filer.go

@@ -1,6 +1,8 @@
 package command
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
 	"net/http"
 	"strconv"
 	"strings"
@@ -28,7 +30,6 @@ type FilerOptions struct {
 	redirectOnRead          *bool
 	disableDirListing       *bool
 	maxMB                   *int
-	secretKey               *string
 	dirListingLimit         *int
 	dataCenter              *string
 	enableNotification      *bool
@@ -49,7 +50,6 @@ func init() {
 	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
 	f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit")
-	f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
 	f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to write to volumes in this data center")
 }
@@ -70,13 +70,15 @@ var cmdFiler = &Command{
 
 	The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
 
-	The example filer.toml configuration file can be generated by "weed scaffold filer"
+	The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
 
 `,
 }
 
 func runFiler(cmd *Command, args []string) bool {
 
+	weed_server.LoadConfiguration("security", false)
+
 	f.startFiler()
 
 	return true
@@ -103,7 +105,6 @@ func (fo *FilerOptions) startFiler() {
 		RedirectOnRead:     *fo.redirectOnRead,
 		DisableDirListing:  *fo.disableDirListing,
 		MaxMB:              *fo.maxMB,
-		SecretKey:          *fo.secretKey,
 		DirListingLimit:    *fo.dirListingLimit,
 		DataCenter:         *fo.dataCenter,
 		DefaultLevelDbDir:  defaultLevelDbDirectory,
@@ -126,7 +127,7 @@ func (fo *FilerOptions) startFiler() {
 		}()
 	}
 
-	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*fo.port))
+	glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", util.VERSION, *fo.ip, *fo.port)
 	filerListener, e := util.NewListener(
 		":"+strconv.Itoa(*fo.port),
 		time.Duration(10)*time.Second,
@@ -144,7 +145,7 @@ func (fo *FilerOptions) startFiler() {
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	grpcS := util.NewGrpcServer()
+	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "filer"))
 	filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
 	reflection.Register(grpcS)
 	go grpcS.Serve(grpcL)

weed/command/filer_copy.go

@@ -2,6 +2,10 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 	"io/ioutil"
 	"net/url"
 	"os"
@@ -11,11 +15,9 @@ import (
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"io"
 	"net/http"
-	"path"
 	"strconv"
 	"time"
 )
@@ -25,16 +27,14 @@ var (
 )
 
 type CopyOptions struct {
-	filerGrpcPort *int
-	master        *string
-	include       *string
-	replication   *string
-	collection    *string
-	ttl           *string
-	maxMB         *int
-	secretKey     *string
-
-	secret security.Secret
+	filerGrpcPort  *int
+	master         *string
+	include        *string
+	replication    *string
+	collection     *string
+	ttl            *string
+	maxMB          *int
+	grpcDialOption grpc.DialOption
 }
 
 func init() {
@@ -47,7 +47,6 @@ func init() {
 	copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 	copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit")
 	copy.filerGrpcPort = cmdCopy.Flag.Int("filer.port.grpc", 0, "filer grpc server listen port, default to filer port + 10000")
-	copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 }
 
 var cmdCopy = &Command{
@@ -61,17 +60,15 @@ var cmdCopy = &Command{
   All files under the folder and subfolders will be copyed.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
   If "maxMB" is set to a positive number, files larger than it would be split into chunks.
 
   `,
 }
 
 func runCopy(cmd *Command, args []string) bool {
-	copy.secret = security.Secret(*copy.secretKey)
+
+	weed_server.LoadConfiguration("security", false)
+
 	if len(args) <= 1 {
 		return false
 	}
@@ -85,7 +82,8 @@ func runCopy(cmd *Command, args []string) bool {
 	}
 	urlPath := filerUrl.Path
 	if !strings.HasSuffix(urlPath, "/") {
-		urlPath = urlPath + "/"
+		fmt.Printf("The last argument should be a folder and end with \"/\": %v\n", err)
+		return false
 	}
 
 	if filerUrl.Port() == "" {
@@ -105,16 +103,17 @@ func runCopy(cmd *Command, args []string) bool {
 	}
 
 	filerGrpcAddress := fmt.Sprintf("%s:%d", filerUrl.Hostname(), filerGrpcPort)
+	copy.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 
 	for _, fileOrDir := range fileOrDirs {
-		if !doEachCopy(fileOrDir, filerUrl.Host, filerGrpcAddress, urlPath) {
+		if !doEachCopy(fileOrDir, filerUrl.Host, filerGrpcAddress, copy.grpcDialOption, urlPath) {
 			return false
 		}
 	}
 	return true
 }
 
-func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path string) bool {
+func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, path string) bool {
 	f, err := os.Open(fileOrDir)
 	if err != nil {
 		fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err)
@@ -132,7 +131,7 @@ func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path st
 	if mode.IsDir() {
 		files, _ := ioutil.ReadDir(fileOrDir)
 		for _, subFileOrDir := range files {
-			if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, path+fi.Name()+"/") {
+			if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), filerAddress, filerGrpcAddress, grpcDialOption, path+fi.Name()+"/") {
 				return false
 			}
 		}
@@ -154,25 +153,24 @@ func doEachCopy(fileOrDir string, filerAddress, filerGrpcAddress string, path st
 	}
 
 	if chunkCount == 1 {
-		return uploadFileAsOne(filerAddress, filerGrpcAddress, path, f, fi)
+		return uploadFileAsOne(filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi)
 	}
 
-	return uploadFileInChunks(filerAddress, filerGrpcAddress, path, f, fi, chunkCount, chunkSize)
+	return uploadFileInChunks(filerAddress, filerGrpcAddress, grpcDialOption, path, f, fi, chunkCount, chunkSize)
 }
 
-func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo) bool {
+func uploadFileAsOne(filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo) bool {
 
 	// upload the file content
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
-	isGzipped := isGzipped(fileName)
 
 	var chunks []*filer_pb.FileChunk
 
 	if fi.Size() > 0 {
 
 		// assign a volume
-		assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(*copy.master, grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
 			Replication: *copy.replication,
 			Collection:  *copy.collection,
@@ -184,7 +182,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 
 		targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
 
-		uploadResult, err := operation.Upload(targetUrl, fileName, f, isGzipped, mimeType, nil, "")
+		uploadResult, err := operation.Upload(targetUrl, fileName, f, false, mimeType, nil, assignResult.Auth)
 		if err != nil {
 			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 			return false
@@ -206,7 +204,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 		fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerAddress, urlFolder, fileName)
 	}
 
-	if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: urlFolder,
 			Entry: &filer_pb.Entry{
@@ -239,7 +237,7 @@ func uploadFileAsOne(filerAddress, filerGrpcAddress string, urlFolder string, f
 	return true
 }
 
-func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
+func uploadFileInChunks(filerAddress, filerGrpcAddress string, grpcDialOption grpc.DialOption, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool {
 
 	fileName := filepath.Base(f.Name())
 	mimeType := detectMimeType(f)
@@ -249,7 +247,7 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 	for i := int64(0); i < int64(chunkCount); i++ {
 
 		// assign a volume
-		assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{
+		assignResult, err := operation.Assign(*copy.master, grpcDialOption, &operation.VolumeAssignRequest{
 			Count:       1,
 			Replication: *copy.replication,
 			Collection:  *copy.collection,
@@ -264,7 +262,7 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 		uploadResult, err := operation.Upload(targetUrl,
 			fileName+"-"+strconv.FormatInt(i+1, 10),
 			io.LimitReader(f, chunkSize),
-			false, "application/octet-stream", nil, "")
+			false, "application/octet-stream", nil, assignResult.Auth)
 		if err != nil {
 			fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err)
 			return false
@@ -283,7 +281,7 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 		fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size))
 	}
 
-	if err := withFilerClient(filerGrpcAddress, func(client filer_pb.SeaweedFilerClient) error {
+	if err := withFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: urlFolder,
 			Entry: &filer_pb.Entry{
@@ -318,13 +316,9 @@ func uploadFileInChunks(filerAddress, filerGrpcAddress string, urlFolder string,
 	return true
 }
 
-func isGzipped(filename string) bool {
-	return strings.ToLower(path.Ext(filename)) == ".gz"
-}
-
 func detectMimeType(f *os.File) string {
 	head := make([]byte, 512)
-	f.Seek(0, 0)
+	f.Seek(0, io.SeekStart)
 	n, err := f.Read(head)
 	if err == io.EOF {
 		return ""
@@ -333,14 +327,14 @@ func detectMimeType(f *os.File) string {
 		fmt.Printf("read head of %v: %v\n", f.Name(), err)
 		return "application/octet-stream"
 	}
-	f.Seek(0, 0)
+	f.Seek(0, io.SeekStart)
 	mimeType := http.DetectContentType(head[:n])
 	return mimeType
 }
 
-func withFilerClient(filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error {
+func withFilerClient(filerAddress string, grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(filerAddress)
+	grpcConnection, err := util.GrpcDial(filerAddress, grpcDialOption)
 	if err != nil {
 		return fmt.Errorf("fail to dial %s: %v", filerAddress, err)
 	}

weed/command/filer_export.go

@@ -14,7 +14,7 @@ func init() {
 }
 
 var cmdFilerExport = &Command{
-	UsageLine: "filer.export -sourceStore=mysql -targetStroe=cassandra",
+	UsageLine: "filer.export -sourceStore=mysql -targetStore=cassandra",
 	Short:     "export meta data in filer store",
 	Long: `Iterate the file tree and export all metadata out
 

weed/command/filer_replication.go

@@ -28,13 +28,14 @@ var cmdFilerReplicate = &Command{
 	filer.replicate listens on filer notifications. If any file is updated, it will fetch the updated content,
 	and write to the other destination.
 
-	Run "weed scaffold -config replication" to generate a replication.toml file and customize the parameters.
+	Run "weed scaffold -config=replication" to generate a replication.toml file and customize the parameters.
 
   `,
 }
 
 func runFilerReplicate(cmd *Command, args []string) bool {
 
+	weed_server.LoadConfiguration("security", false)
 	weed_server.LoadConfiguration("replication", true)
 	weed_server.LoadConfiguration("notification", true)
 	config := viper.GetViper()

weed/command/fix.go

@@ -28,6 +28,32 @@ var (
 	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 )
 
+type VolumeFileScanner4Fix struct {
+	version storage.Version
+	nm      *storage.NeedleMap
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock storage.SuperBlock) error {
+	scanner.version = superBlock.Version()
+	return nil
+
+}
+func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool {
+	return false
+}
+
+func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *storage.Needle, offset int64) error {
+	glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsGzipped())
+	if n.Size > 0 {
+		pe := scanner.nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
+		glog.V(2).Infof("saved %d with error %v", n.Size, pe)
+	} else {
+		glog.V(2).Infof("skipping deleted file ...")
+		return scanner.nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
+	}
+	return nil
+}
+
 func runFix(cmd *Command, args []string) bool {
 
 	if *fixVolumeId == -1 {
@@ -48,24 +74,12 @@ func runFix(cmd *Command, args []string) bool {
 	nm := storage.NewBtreeNeedleMap(indexFile)
 	defer nm.Close()
 
-	var version storage.Version
 	vid := storage.VolumeId(*fixVolumeId)
-	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid,
-		storage.NeedleMapInMemory,
-		func(superBlock storage.SuperBlock) error {
-			version = superBlock.Version()
-			return nil
-		}, false, func(n *storage.Needle, offset int64) error {
-			glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(version), n.IsGzipped())
-			if n.Size > 0 {
-				pe := nm.Put(n.Id, types.Offset(offset/types.NeedlePaddingSize), n.Size)
-				glog.V(2).Infof("saved %d with error %v", n.Size, pe)
-			} else {
-				glog.V(2).Infof("skipping deleted file ...")
-				return nm.Delete(n.Id, types.Offset(offset/types.NeedlePaddingSize))
-			}
-			return nil
-		})
+	scanner := &VolumeFileScanner4Fix{
+		nm: nm,
+	}
+
+	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, storage.NeedleMapInMemory, scanner)
 	if err != nil {
 		glog.Fatalf("Export Volume File [ERROR] %s\n", err)
 		os.Remove(indexFileName)

weed/command/master.go

@@ -1,20 +1,20 @@
 package command
 
 import (
+	"github.com/chrislusf/raft/protobuf"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/gorilla/mux"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc/reflection"
 	"net/http"
 	"os"
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
-	"github.com/chrislusf/seaweedfs/weed/server"
-	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
-	"google.golang.org/grpc/reflection"
 )
 
 func init() {
@@ -24,8 +24,11 @@ func init() {
 var cmdMaster = &Command{
 	UsageLine: "master -port=9333",
 	Short:     "start a master server",
-	Long: `start a master server to provide volume=>location mapping service
-  and sequence number of file ids
+	Long: `start a master server to provide volume=>location mapping service and sequence number of file ids
+
+	The configuration file "security.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order.
+
+	The example security.toml configuration file can be generated by "weed scaffold -config=security"
 
   `,
 }
@@ -44,7 +47,7 @@ var (
 	mMaxCpu               = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
 	garbageThreshold      = cmdMaster.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
-	masterSecureKey       = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
+	httpReadOnly          = cmdMaster.Flag.Bool("httpReadOnly", false, "disable http operations, only gRPC operations are allowed.")
 	masterCpuProfile      = cmdMaster.Flag.String("cpuprofile", "", "cpu profile output file")
 	masterMemProfile      = cmdMaster.Flag.String("memprofile", "", "memory profile output file")
 
@@ -52,6 +55,9 @@ var (
 )
 
 func runMaster(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
 	if *mMaxCpu < 1 {
 		*mMaxCpu = runtime.NumCPU()
 	}
@@ -72,42 +78,46 @@ func runMaster(cmd *Command, args []string) bool {
 	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
 		*volumeSizeLimitMB, *volumePreallocate,
 		*mpulse, *defaultReplicaPlacement, *garbageThreshold,
-		masterWhiteList, *masterSecureKey,
+		masterWhiteList,
+		*httpReadOnly,
 	)
 
 	listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
 
 	glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)
 
-	listener, e := util.NewListener(listeningAddress, 0)
+	masterListener, e := util.NewListener(listeningAddress, 0)
 	if e != nil {
 		glog.Fatalf("Master startup error: %v", e)
 	}
 
 	go func() {
-		time.Sleep(100 * time.Millisecond)
+		// start raftServer
 		myMasterAddress, peers := checkPeers(*masterIp, *mport, *masterPeers)
-		raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
+		raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
+			peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse)
 		ms.SetRaftServer(raftServer)
+		r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
+
+		// starting grpc server
+		grpcPort := *mport + 10000
+		grpcL, err := util.NewListener(*masterBindIp+":"+strconv.Itoa(grpcPort), 0)
+		if err != nil {
+			glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+		}
+		// Create your protocol servers.
+		grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
+		master_pb.RegisterSeaweedServer(grpcS, ms)
+		protobuf.RegisterRaftServer(grpcS, raftServer)
+		reflection.Register(grpcS)
+
+		glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *masterBindIp, grpcPort)
+		grpcS.Serve(grpcL)
 	}()
 
-	// start grpc and http server
-	m := cmux.New(listener)
-
-	grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-	httpL := m.Match(cmux.Any())
-
-	// Create your protocol servers.
-	grpcS := util.NewGrpcServer()
-	master_pb.RegisterSeaweedServer(grpcS, ms)
-	reflection.Register(grpcS)
-
+	// start http server
 	httpS := &http.Server{Handler: r}
-
-	go grpcS.Serve(grpcL)
-	go httpS.Serve(httpL)
-
-	if err := m.Serve(); err != nil {
+	if err := httpS.Serve(masterListener); err != nil {
 		glog.Fatalf("master server failed to serve: %v", err)
 	}
 

weed/command/mount.go

@@ -17,6 +17,7 @@ type MountOptions struct {
 	ttlSec             *int
 	chunkSizeLimitMB   *int
 	dataCenter         *string
+	allowOthers        *bool
 }
 
 var (
@@ -35,8 +36,9 @@ func init() {
 	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
 	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
-	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 16, "local write buffer size, also chunk large files")
+	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
+	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
 	mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
 }
@@ -51,7 +53,7 @@ var cmdMount = &Command{
   2) have a "weed filer" running
   These 2 requirements can be achieved with one command "weed server -filer=true"
 
-  This uses bazil.org/fuse, which enables writing FUSE file systems on
+  This uses github.com/seaweedfs/fuse, which enables writing FUSE file systems on
   Linux, and OS X.
 
   On OS X, it requires OSXFUSE (http://osxfuse.github.com/).

weed/command/mount_std.go

@@ -4,18 +4,27 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
+	"os"
+	"os/user"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/filesys"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 func runMount(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
 	fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
 	if *mountOptions.dir == "" {
 		fmt.Printf("Please specify the mount directory via \"-dir\"")
@@ -28,24 +37,46 @@ func runMount(cmd *Command, args []string) bool {
 
 	fuse.Unmount(*mountOptions.dir)
 
+	// detect mount folder mode
+	mountMode := os.ModeDir | 0755
+	if fileInfo, err := os.Stat(*mountOptions.dir); err == nil {
+		mountMode = os.ModeDir | fileInfo.Mode()
+	}
+
+	// detect current user
+	uid, gid := uint32(0), uint32(0)
+	if u, err := user.Current(); err == nil {
+		if parsedId, pe := strconv.ParseUint(u.Uid, 10, 32); pe == nil {
+			uid = uint32(parsedId)
+		}
+		if parsedId, pe := strconv.ParseUint(u.Gid, 10, 32); pe == nil {
+			gid = uint32(parsedId)
+		}
+	}
+
 	util.SetupProfiling(*mountCpuProfile, *mountMemProfile)
 
-	c, err := fuse.Mount(
-		*mountOptions.dir,
+	options := []fuse.MountOption{
 		fuse.VolumeName("SeaweedFS"),
 		fuse.FSName("SeaweedFS"),
 		fuse.Subtype("SeaweedFS"),
 		fuse.NoAppleDouble(),
 		fuse.NoAppleXattr(),
+		fuse.NoBrowse(),
+		fuse.AutoXattr(),
 		fuse.ExclCreate(),
 		fuse.DaemonTimeout("3600"),
-		fuse.AllowOther(),
 		fuse.AllowSUID(),
 		fuse.DefaultPermissions(),
-		fuse.MaxReadahead(1024*128),
+		fuse.MaxReadahead(1024 * 128),
 		fuse.AsyncRead(),
 		fuse.WritebackCache(),
-	)
+	}
+	if *mountOptions.allowOthers {
+		options = append(options, fuse.AllowOther())
+	}
+
+	c, err := fuse.Mount(*mountOptions.dir, options...)
 	if err != nil {
 		glog.Fatal(err)
 		return false
@@ -69,6 +100,7 @@ func runMount(cmd *Command, args []string) bool {
 
 	err = fs.Serve(c, filesys.NewSeaweedFileSystem(&filesys.Option{
 		FilerGrpcAddress:   filerGrpcAddress,
+		GrpcDialOption:     security.LoadClientTLS(viper.Sub("grpc"), "client"),
 		FilerMountRootPath: mountRoot,
 		Collection:         *mountOptions.collection,
 		Replication:        *mountOptions.replication,
@@ -77,6 +109,9 @@ func runMount(cmd *Command, args []string) bool {
 		DataCenter:         *mountOptions.dataCenter,
 		DirListingLimit:    *mountOptions.dirListingLimit,
 		EntryCacheTtl:      3 * time.Second,
+		MountUid:           uid,
+		MountGid:           gid,
+		MountMode:          mountMode,
 	}))
 	if err != nil {
 		fuse.Unmount(*mountOptions.dir)

weed/command/s3.go

@@ -1,6 +1,9 @@
 package command
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
 	"net/http"
 	"time"
 
@@ -46,6 +49,8 @@ var cmdS3 = &Command{
 
 func runS3(cmd *Command, args []string) bool {
 
+	weed_server.LoadConfiguration("security", false)
+
 	filerGrpcAddress, err := parseFilerGrpcAddress(*s3options.filer, *s3options.filerGrpcPort)
 	if err != nil {
 		glog.Fatal(err)
@@ -59,6 +64,7 @@ func runS3(cmd *Command, args []string) bool {
 		FilerGrpcAddress: filerGrpcAddress,
 		DomainName:       *s3options.domainName,
 		BucketsPath:      *s3options.filerBucketsPath,
+		GrpcDialOption:   security.LoadClientTLS(viper.Sub("grpc"), "client"),
 	})
 	if s3ApiServer_err != nil {
 		glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)

weed/command/scaffold.go

@@ -10,7 +10,7 @@ func init() {
 }
 
 var cmdScaffold = &Command{
-	UsageLine: "scaffold [filer]",
+	UsageLine: "scaffold -config=[filer|notification|replication|security]",
 	Short:     "generate basic configuration files",
 	Long: `Generate filer.toml with all possible configurations for you to customize.
 
@@ -19,7 +19,7 @@ var cmdScaffold = &Command{
 
 var (
 	outputPath = cmdScaffold.Flag.String("output", "", "if not empty, save the configuration file to this directory")
-	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication] the configuration file to generate")
+	config     = cmdScaffold.Flag.String("config", "filer", "[filer|notification|replication|security] the configuration file to generate")
 )
 
 func runScaffold(cmd *Command, args []string) bool {
@@ -32,6 +32,8 @@ func runScaffold(cmd *Command, args []string) bool {
 		content = NOTIFICATION_TOML_EXAMPLE
 	case "replication":
 		content = REPLICATION_TOML_EXAMPLE
+	case "security":
+		content = SECURITY_TOML_EXAMPLE
 	}
 	if content == "" {
 		println("need a valid -config option")
@@ -239,5 +241,53 @@ b2_master_application_key  = ""
 bucket = "mybucket"            # an existing bucket
 directory = "/"                # destination directory
 
+`
+
+	SECURITY_TOML_EXAMPLE = `
+# Put this file to one of the location, with descending priority
+#    ./security.toml
+#    $HOME/.seaweedfs/security.toml
+#    /etc/seaweedfs/security.toml
+# this file is read by master, volume server, and filer
+
+# the jwt signing key is read by master and volume server
+# a jwt expires in 10 seconds
+[jwt.signing]
+key = ""
+
+# all grpc tls authentications are mutual
+# the values for the following ca, cert, and key are paths to the PERM files.
+[grpc]
+ca = ""
+
+[grpc.volume]
+cert = ""
+key  = ""
+
+[grpc.master]
+cert = ""
+key  = ""
+
+[grpc.filer]
+cert = ""
+key  = ""
+
+# use this for any place needs a grpc client
+# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
+[grpc.client]
+cert = ""
+key  = ""
+
+
+# volume server https options
+# Note: work in progress!
+#     this does not work with other clients, e.g., "weed filer|mount" etc, yet.
+[https.client]
+enabled = true
+[https.volume]
+cert = ""
+key  = ""
+
+
 `
 )

weed/command/server.go

@@ -1,6 +1,9 @@
 package command
 
 import (
+	"github.com/chrislusf/raft/protobuf"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
 	"net/http"
 	"os"
 	"runtime"
@@ -15,7 +18,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/server"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/gorilla/mux"
-	"github.com/soheilhy/cmux"
 	"google.golang.org/grpc/reflection"
 )
 
@@ -58,8 +60,8 @@ var (
 	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
 	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
 	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
+	masterHttpReadOnly            = cmdServer.Flag.Bool("master.httpReadOnly", false, "disable http operations, only gRPC operations are allowed.")
 	serverPeers                   = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
-	serverSecureKey               = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 	serverGarbageThreshold        = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
 	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
@@ -96,7 +98,9 @@ func init() {
 }
 
 func runServer(cmd *Command, args []string) bool {
-	filerOptions.secretKey = serverSecureKey
+
+	weed_server.LoadConfiguration("security", false)
+
 	if *serverOptions.cpuprofile != "" {
 		f, err := os.Create(*serverOptions.cpuprofile)
 		if err != nil {
@@ -159,10 +163,8 @@ func runServer(cmd *Command, args []string) bool {
 		}()
 	}
 
-	var raftWaitForMaster sync.WaitGroup
 	var volumeWait sync.WaitGroup
 
-	raftWaitForMaster.Add(1)
 	volumeWait.Add(1)
 
 	go func() {
@@ -170,43 +172,45 @@ func runServer(cmd *Command, args []string) bool {
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
 			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
 			*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
-			serverWhiteList, *serverSecureKey,
+			serverWhiteList, *masterHttpReadOnly,
 		)
 
-		glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
+		glog.V(0).Infof("Start Seaweed Master %s at %s:%d", util.VERSION, *serverIp, *masterPort)
 		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), 0)
 		if e != nil {
 			glog.Fatalf("Master startup error: %v", e)
 		}
 
 		go func() {
-			raftWaitForMaster.Wait()
-			time.Sleep(100 * time.Millisecond)
-			myAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
-			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
+			// start raftServer
+			myMasterAddress, peers := checkPeers(*serverIp, *masterPort, *serverPeers)
+			raftServer := weed_server.NewRaftServer(security.LoadClientTLS(viper.Sub("grpc"), "master"),
+				peers, myMasterAddress, *masterMetaFolder, ms.Topo, *pulseSeconds)
 			ms.SetRaftServer(raftServer)
-			volumeWait.Done()
+			r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
+
+			// starting grpc server
+			grpcPort := *masterPort + 10000
+			grpcL, err := util.NewListener(*serverIp+":"+strconv.Itoa(grpcPort), 0)
+			if err != nil {
+				glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
+			}
+			// Create your protocol servers.
+			glog.V(0).Infof("grpc config %+v", viper.Sub("grpc"))
+			grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
+			master_pb.RegisterSeaweedServer(grpcS, ms)
+			protobuf.RegisterRaftServer(grpcS, raftServer)
+			reflection.Register(grpcS)
+
+			glog.V(0).Infof("Start Seaweed Master %s grpc server at %s:%d", util.VERSION, *serverIp, grpcPort)
+			grpcS.Serve(grpcL)
 		}()
 
-		raftWaitForMaster.Done()
-
-		// start grpc and http server
-		m := cmux.New(masterListener)
-
-		grpcL := m.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))
-		httpL := m.Match(cmux.Any())
-
-		// Create your protocol servers.
-		grpcS := util.NewGrpcServer()
-		master_pb.RegisterSeaweedServer(grpcS, ms)
-		reflection.Register(grpcS)
+		volumeWait.Done()
 
+		// start http server
 		httpS := &http.Server{Handler: r}
-
-		go grpcS.Serve(grpcL)
-		go httpS.Serve(httpL)
-
-		if err := m.Serve(); err != nil {
+		if err := httpS.Serve(masterListener); err != nil {
 			glog.Fatalf("master server failed to serve: %v", err)
 		}
 

weed/command/upload.go

@@ -3,11 +3,13 @@ package command
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/server"
+	"github.com/spf13/viper"
 	"os"
 	"path/filepath"
 
 	"github.com/chrislusf/seaweedfs/weed/operation"
-	"github.com/chrislusf/seaweedfs/weed/security"
 )
 
 var (
@@ -23,7 +25,6 @@ type UploadOptions struct {
 	dataCenter  *string
 	ttl         *string
 	maxMB       *int
-	secretKey   *string
 }
 
 func init() {
@@ -37,7 +38,6 @@ func init() {
 	upload.dataCenter = cmdUpload.Flag.String("dataCenter", "", "optional data center name")
 	upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 	upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit")
-	upload.secretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)")
 }
 
 var cmdUpload = &Command{
@@ -53,18 +53,17 @@ var cmdUpload = &Command{
   All files under the folder and subfolders will be uploaded, each with its own file key.
   Optional parameter "-include" allows you to specify the file name patterns.
 
-  If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is.
-  This can save volume server's gzipped processing and allow customizable gzip compression level.
-  The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js".
-
-  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly.
+  If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separately.
   The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned.
 
   `,
 }
 
 func runUpload(cmd *Command, args []string) bool {
-	secret := security.Secret(*upload.secretKey)
+
+	weed_server.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(viper.Sub("grpc"), "client")
+
 	if len(args) == 0 {
 		if *upload.dir == "" {
 			return false
@@ -81,9 +80,9 @@ func runUpload(cmd *Command, args []string) bool {
 					if e != nil {
 						return e
 					}
-					results, e := operation.SubmitFiles(*upload.master, parts,
+					results, e := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
 						*upload.replication, *upload.collection, *upload.dataCenter,
-						*upload.ttl, *upload.maxMB, secret)
+						*upload.ttl, *upload.maxMB)
 					bytes, _ := json.Marshal(results)
 					fmt.Println(string(bytes))
 					if e != nil {
@@ -100,9 +99,9 @@ func runUpload(cmd *Command, args []string) bool {
 		if e != nil {
 			fmt.Println(e.Error())
 		}
-		results, _ := operation.SubmitFiles(*upload.master, parts,
+		results, _ := operation.SubmitFiles(*upload.master, grpcDialOption, parts,
 			*upload.replication, *upload.collection, *upload.dataCenter,
-			*upload.ttl, *upload.maxMB, secret)
+			*upload.ttl, *upload.maxMB)
 		bytes, _ := json.Marshal(results)
 		fmt.Println(string(bytes))
 	}

weed/command/volume.go

@@ -1,6 +1,8 @@
 package command
 
 import (
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
 	"net/http"
 	"os"
 	"runtime"
@@ -78,6 +80,9 @@ var (
 )
 
 func runVolume(cmd *Command, args []string) bool {
+
+	weed_server.LoadConfiguration("security", false)
+
 	if *v.maxCpu < 1 {
 		*v.maxCpu = runtime.NumCPU()
 	}
@@ -155,7 +160,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	)
 
 	listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port)
-	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)
+	glog.V(0).Infof("Start Seaweed volume server %s at %s", util.VERSION, listeningAddress)
 	listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second)
 	if e != nil {
 		glog.Fatalf("Volume server listener error:%v", e)
@@ -185,13 +190,20 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
 	}
-	grpcS := util.NewGrpcServer()
+	grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "volume"))
 	volume_server_pb.RegisterVolumeServerServer(grpcS, volumeServer)
 	reflection.Register(grpcS)
 	go grpcS.Serve(grpcL)
 
-	if e := http.Serve(listener, volumeMux); e != nil {
-		glog.Fatalf("Volume server fail to serve: %v", e)
+	if viper.GetString("https.volume.key") != "" {
+		if e := http.ServeTLS(listener, volumeMux,
+			viper.GetString("https.volume.cert"), viper.GetString("https.volume.key")); e != nil {
+			glog.Fatalf("Volume server fail to serve: %v", e)
+		}
+	} else {
+		if e := http.Serve(listener, volumeMux); e != nil {
+			glog.Fatalf("Volume server fail to serve: %v", e)
+		}
 	}
 
 }

weed/filer2/cassandra/cassandra_store.go

@@ -48,8 +48,8 @@ func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) {
 	}
 
 	if err := store.session.Query(
-		"INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?)",
-		dir, name, meta).Exec(); err != nil {
+		"INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?) USING TTL ? ",
+		dir, name, meta, entry.TtlSec).Exec(); err != nil {
 		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
 	}
 

weed/filer2/entry.go

@@ -8,17 +8,18 @@ import (
 )
 
 type Attr struct {
-	Mtime       time.Time   // time of last modification
-	Crtime      time.Time   // time of creation (OS X only)
-	Mode        os.FileMode // file mode
-	Uid         uint32      // owner uid
-	Gid         uint32      // group gid
-	Mime        string      // mime type
-	Replication string      // replication
-	Collection  string      // collection name
-	TtlSec      int32       // ttl in seconds
-	UserName    string
-	GroupNames  []string
+	Mtime         time.Time   // time of last modification
+	Crtime        time.Time   // time of creation (OS X only)
+	Mode          os.FileMode // file mode
+	Uid           uint32      // owner uid
+	Gid           uint32      // group gid
+	Mime          string      // mime type
+	Replication   string      // replication
+	Collection    string      // collection name
+	TtlSec        int32       // ttl in seconds
+	UserName      string
+	GroupNames    []string
+	SymlinkTarget string
 }
 
 func (attr Attr) IsDirectory() bool {

weed/filer2/entry_codec.go

@@ -6,7 +6,7 @@ import (
 
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/gogo/protobuf/proto"
+	"github.com/golang/protobuf/proto"
 )
 
 func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
@@ -35,17 +35,18 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
 func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
 
 	return &filer_pb.FuseAttributes{
-		Crtime:      entry.Attr.Crtime.Unix(),
-		Mtime:       entry.Attr.Mtime.Unix(),
-		FileMode:    uint32(entry.Attr.Mode),
-		Uid:         entry.Uid,
-		Gid:         entry.Gid,
-		Mime:        entry.Mime,
-		Collection:  entry.Attr.Collection,
-		Replication: entry.Attr.Replication,
-		TtlSec:      entry.Attr.TtlSec,
-		UserName:    entry.Attr.UserName,
-		GroupName:   entry.Attr.GroupNames,
+		Crtime:        entry.Attr.Crtime.Unix(),
+		Mtime:         entry.Attr.Mtime.Unix(),
+		FileMode:      uint32(entry.Attr.Mode),
+		Uid:           entry.Uid,
+		Gid:           entry.Gid,
+		Mime:          entry.Mime,
+		Collection:    entry.Attr.Collection,
+		Replication:   entry.Attr.Replication,
+		TtlSec:        entry.Attr.TtlSec,
+		UserName:      entry.Attr.UserName,
+		GroupName:     entry.Attr.GroupNames,
+		SymlinkTarget: entry.Attr.SymlinkTarget,
 	}
 }
 
@@ -64,6 +65,7 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
 	t.TtlSec = attr.TtlSec
 	t.UserName = attr.UserName
 	t.GroupNames = attr.GroupName
+	t.SymlinkTarget = attr.SymlinkTarget
 
 	return t
 }

weed/filer2/filechunks.go

@@ -33,7 +33,7 @@ func ETag(chunks []*filer_pb.FileChunk) (etag string) {
 
 func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
 
-	visibles := nonOverlappingVisibleIntervals(chunks)
+	visibles := NonOverlappingVisibleIntervals(chunks)
 
 	fileIds := make(map[string]bool)
 	for _, interval := range visibles {
@@ -75,7 +75,13 @@ type ChunkView struct {
 
 func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) {
 
-	visibles := nonOverlappingVisibleIntervals(chunks)
+	visibles := NonOverlappingVisibleIntervals(chunks)
+
+	return ViewFromVisibleIntervals(visibles, offset, size)
+
+}
+
+func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int) (views []*ChunkView) {
 
 	stop := offset + int64(size)
 
@@ -97,7 +103,7 @@ func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views
 
 }
 
-func logPrintf(name string, visibles []*visibleInterval) {
+func logPrintf(name string, visibles []VisibleInterval) {
 	/*
 		log.Printf("%s len %d", name, len(visibles))
 		for _, v := range visibles {
@@ -108,11 +114,11 @@ func logPrintf(name string, visibles []*visibleInterval) {
 
 var bufPool = sync.Pool{
 	New: func() interface{} {
-		return new(visibleInterval)
+		return new(VisibleInterval)
 	},
 }
 
-func mergeIntoVisibles(visibles, newVisibles []*visibleInterval, chunk *filer_pb.FileChunk) []*visibleInterval {
+func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
 
 	newV := newVisibleInterval(
 		chunk.Offset,
@@ -173,18 +179,18 @@ func mergeIntoVisibles(visibles, newVisibles []*visibleInterval, chunk *filer_pb
 	return newVisibles
 }
 
-func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) {
+func NonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []VisibleInterval) {
 
 	sort.Slice(chunks, func(i, j int) bool {
 		return chunks[i].Mtime < chunks[j].Mtime
 	})
 
-	var newVislbles []*visibleInterval
+	var newVisibles []VisibleInterval
 	for _, chunk := range chunks {
-		newVislbles = mergeIntoVisibles(visibles, newVislbles, chunk)
+		newVisibles = MergeIntoVisibles(visibles, newVisibles, chunk)
 		t := visibles[:0]
-		visibles = newVislbles
-		newVislbles = t
+		visibles = newVisibles
+		newVisibles = t
 
 		logPrintf("add", visibles)
 
@@ -196,7 +202,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v
 // find non-overlapping visible intervals
 // visible interval map to one file chunk
 
-type visibleInterval struct {
+type VisibleInterval struct {
 	start        int64
 	stop         int64
 	modifiedTime int64
@@ -204,8 +210,8 @@ type visibleInterval struct {
 	isFullChunk  bool
 }
 
-func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) *visibleInterval {
-	return &visibleInterval{
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval {
+	return VisibleInterval{
 		start:        start,
 		stop:         stop,
 		fileId:       fileId,

weed/filer2/filechunks_test.go

@@ -63,7 +63,7 @@ func TestIntervalMerging(t *testing.T) {
 
 	testcases := []struct {
 		Chunks   []*filer_pb.FileChunk
-		Expected []*visibleInterval
+		Expected []*VisibleInterval
 	}{
 		// case 0: normal
 		{
@@ -72,7 +72,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 100, Size: 100, FileId: "asdf", Mtime: 134},
 				{Offset: 200, Size: 100, FileId: "fsad", Mtime: 353},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 100, fileId: "abc"},
 				{start: 100, stop: 200, fileId: "asdf"},
 				{start: 200, stop: 300, fileId: "fsad"},
@@ -84,7 +84,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 200, fileId: "asdf"},
 			},
 		},
@@ -94,7 +94,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 				{Offset: 0, Size: 50, FileId: "asdf", Mtime: 134},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 50, fileId: "asdf"},
 				{start: 50, stop: 100, fileId: "abc"},
 			},
@@ -106,7 +106,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
 				{Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 50, fileId: "asdf"},
 				{start: 50, stop: 300, fileId: "xxxx"},
 			},
@@ -118,7 +118,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 200, FileId: "asdf", Mtime: 134},
 				{Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 200, fileId: "asdf"},
 				{start: 250, stop: 500, fileId: "xxxx"},
 			},
@@ -131,7 +131,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 70, Size: 150, FileId: "abc", Mtime: 143},
 				{Offset: 80, Size: 100, FileId: "xxxx", Mtime: 134},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 200, fileId: "asdf"},
 				{start: 200, stop: 220, fileId: "abc"},
 			},
@@ -143,7 +143,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 100, fileId: "abc"},
 			},
 		},
@@ -157,7 +157,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 8388608, Size: 3145728, FileId: "5,02982f80de50", Mtime: 160},
 				{Offset: 11534336, Size: 2842193, FileId: "7,0299ad723803", Mtime: 170},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 2097152, fileId: "3,029565bf3092"},
 				{start: 2097152, stop: 5242880, fileId: "6,029632f47ae2"},
 				{start: 5242880, stop: 8388608, fileId: "2,029734c5aa10"},
@@ -174,7 +174,7 @@ func TestIntervalMerging(t *testing.T) {
 				{Offset: 208896, Size: 339968 - 208896, FileId: "2,0b4031a72689", Mtime: 150},
 				{Offset: 339968, Size: 471040 - 339968, FileId: "3,0b416a557362", Mtime: 160},
 			},
-			Expected: []*visibleInterval{
+			Expected: []*VisibleInterval{
 				{start: 0, stop: 77824, fileId: "4,0b3df938e301"},
 				{start: 77824, stop: 208896, fileId: "4,0b3f0c7202f0"},
 				{start: 208896, stop: 339968, fileId: "2,0b4031a72689"},
@@ -186,7 +186,7 @@ func TestIntervalMerging(t *testing.T) {
 
 	for i, testcase := range testcases {
 		log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i)
-		intervals := nonOverlappingVisibleIntervals(testcase.Chunks)
+		intervals := NonOverlappingVisibleIntervals(testcase.Chunks)
 		for x, interval := range intervals {
 			log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s",
 				i, x, interval.start, interval.stop, interval.fileId)

weed/filer2/filer.go

@@ -3,6 +3,7 @@ package filer2
 import (
 	"context"
 	"fmt"
+	"google.golang.org/grpc"
 	"math"
 	"os"
 	"path/filepath"
@@ -24,13 +25,15 @@ type Filer struct {
 	directoryCache     *ccache.Cache
 	MasterClient       *wdclient.MasterClient
 	fileIdDeletionChan chan string
+	GrpcDialOption     grpc.DialOption
 }
 
-func NewFiler(masters []string) *Filer {
+func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
 	f := &Filer{
 		directoryCache:     ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
-		MasterClient:       wdclient.NewMasterClient(context.Background(), "filer", masters),
+		MasterClient:       wdclient.NewMasterClient(context.Background(), grpcDialOption, "filer", masters),
 		fileIdDeletionChan: make(chan string, 4096),
+		GrpcDialOption:     grpcDialOption,
 	}
 
 	go f.loopProcessingDeletion()
@@ -67,7 +70,7 @@ func (f *Filer) CreateEntry(entry *Entry) error {
 	var lastDirectoryEntry *Entry
 
 	for i := 1; i < len(dirParts); i++ {
-		dirPath := "/" + filepath.Join(dirParts[:i]...)
+		dirPath := "/" + filepath.ToSlash(filepath.Join(dirParts[:i]...))
 		// fmt.Printf("%d directory: %+v\n", i, dirPath)
 
 		// first check local cache
@@ -195,20 +198,36 @@ func (f *Filer) DeleteEntryMetaAndData(p FullPath, isRecursive bool, shouldDelet
 		if isRecursive {
 			limit = math.MaxInt32
 		}
-		entries, err := f.ListDirectoryEntries(p, "", false, limit)
-		if err != nil {
-			return fmt.Errorf("list folder %s: %v", p, err)
-		}
-		if isRecursive {
-			for _, sub := range entries {
-				f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
+		lastFileName := ""
+		includeLastFile := false
+		for limit > 0 {
+			entries, err := f.ListDirectoryEntries(p, lastFileName, includeLastFile, 1024)
+			if err != nil {
+				return fmt.Errorf("list folder %s: %v", p, err)
 			}
-		} else {
-			if len(entries) > 0 {
-				return fmt.Errorf("folder %s is not empty", p)
+			if len(entries) == 0 {
+				break
+			} else {
+				if isRecursive {
+					for _, sub := range entries {
+						lastFileName = sub.Name()
+						f.DeleteEntryMetaAndData(sub.FullPath, isRecursive, shouldDeleteChunks)
+						limit--
+						if limit <= 0 {
+							break
+						}
+					}
+				} else {
+					if len(entries) > 0 {
+						return fmt.Errorf("folder %s is not empty", p)
+					}
+				}
+				f.cacheDelDirectory(string(p))
+				if len(entries) < 1024 {
+					break
+				}
 			}
 		}
-		f.cacheDelDirectory(string(p))
 	}
 
 	if shouldDeleteChunks {

weed/filer2/filer_deletion.go

@@ -38,13 +38,13 @@ func (f *Filer) loopProcessingDeletion() {
 			fileIds = append(fileIds, fid)
 			if len(fileIds) >= 4096 {
 				glog.V(1).Infof("deleting fileIds len=%d", len(fileIds))
-				operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+				operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
 				fileIds = fileIds[:0]
 			}
 		case <-ticker.C:
 			if len(fileIds) > 0 {
 				glog.V(1).Infof("timed deletion fileIds len=%d", len(fileIds))
-				operation.DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+				operation.DeleteFilesWithLookupVolumeId(f.GrpcDialOption, fileIds, lookupFunc)
 				fileIds = fileIds[:0]
 			}
 		}

weed/filer2/leveldb/leveldb_store_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -61,7 +61,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}

weed/filer2/memdb/memdb_store_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	store := &MemDbStore{}
 	store.Initialize(nil)
 	filer.SetStore(store)
@@ -43,7 +43,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestCreateFileAndList(t *testing.T) {
-	filer := filer2.NewFiler(nil)
+	filer := filer2.NewFiler(nil, nil)
 	store := &MemDbStore{}
 	store.Initialize(nil)
 	filer.SetStore(store)

weed/filer2/redis/universal_redis_store.go

@@ -7,6 +7,7 @@ import (
 	"github.com/go-redis/redis"
 	"sort"
 	"strings"
+	"time"
 )
 
 const (
@@ -24,7 +25,7 @@ func (store *UniversalRedisStore) InsertEntry(entry *filer2.Entry) (err error) {
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	_, err = store.Client.Set(string(entry.FullPath), value, 0).Result()
+	_, err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result()
 
 	if err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)

weed/filesys/dir.go

@@ -7,11 +7,11 @@ import (
 	"path/filepath"
 	"time"
 
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 type Dir struct {
@@ -34,8 +34,10 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 	// https://github.com/bazil/fuse/issues/196
 	attr.Valid = time.Second
 
-	if dir.Path == "/" {
-		attr.Mode = os.ModeDir | 0777
+	if dir.Path == dir.wfs.option.FilerMountRootPath {
+		attr.Uid = dir.wfs.option.MountUid
+		attr.Gid = dir.wfs.option.MountGid
+		attr.Mode = dir.wfs.option.MountMode
 		return nil
 	}
 
@@ -64,6 +66,9 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
 		resp, err := client.LookupDirectoryEntry(context, request)
 		if err != nil {
+			if err == filer2.ErrNotFound {
+				return nil
+			}
 			glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
 			return err
 		}
@@ -72,7 +77,7 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 			dir.attributes = resp.Entry.Attributes
 		}
 
-		dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
+		// dir.wfs.listDirectoryEntriesCache.Set(dir.Path, resp.Entry, dir.wfs.option.EntryCacheTtl)
 
 		return nil
 	})
@@ -96,10 +101,11 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
 
 func (dir *Dir) newFile(name string, entry *filer_pb.Entry) *File {
 	return &File{
-		Name:  name,
-		dir:   dir,
-		wfs:   dir.wfs,
-		entry: entry,
+		Name:           name,
+		dir:            dir,
+		wfs:            dir.wfs,
+		entry:          entry,
+		entryViewCache: nil,
 	}
 }
 
@@ -209,7 +215,7 @@ func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.
 
 			entry = resp.Entry
 
-			dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
+			// dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, dir.wfs.option.EntryCacheTtl)
 
 			return nil
 		})
@@ -239,29 +245,46 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
-		request := &filer_pb.ListEntriesRequest{
-			Directory: dir.Path,
-			Limit:     uint32(dir.wfs.option.DirListingLimit),
-		}
+		paginationLimit := 1024
+		remaining := dir.wfs.option.DirListingLimit
 
-		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("list %s: %v", dir.Path, err)
-			return fuse.EIO
-		}
+		lastEntryName := ""
 
-		cacheTtl := estimatedCacheTtl(len(resp.Entries))
+		for remaining >= 0 {
 
-		for _, entry := range resp.Entries {
-			if entry.IsDirectory {
-				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
-				ret = append(ret, dirent)
-			} else {
-				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
-				ret = append(ret, dirent)
+			request := &filer_pb.ListEntriesRequest{
+				Directory:         dir.Path,
+				StartFromFileName: lastEntryName,
+				Limit:             uint32(paginationLimit),
 			}
-			dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
+
+			glog.V(4).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
+			if err != nil {
+				glog.V(0).Infof("list %s: %v", dir.Path, err)
+				return fuse.EIO
+			}
+
+			cacheTtl := estimatedCacheTtl(len(resp.Entries))
+
+			for _, entry := range resp.Entries {
+				if entry.IsDirectory {
+					dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
+					ret = append(ret, dirent)
+				} else {
+					dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
+					ret = append(ret, dirent)
+				}
+				dir.wfs.listDirectoryEntriesCache.Set(path.Join(dir.Path, entry.Name), entry, cacheTtl)
+				lastEntryName = entry.Name
+			}
+
+			remaining -= len(resp.Entries)
+
+			if len(resp.Entries) < paginationLimit {
+				break
+			}
+
 		}
 
 		return nil
@@ -272,20 +295,79 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {
 
 func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 
+	if !req.Dir {
+		return dir.removeOneFile(ctx, req)
+	}
+
+	return dir.removeFolder(ctx, req)
+
+}
+
+func (dir *Dir) removeOneFile(ctx context.Context, req *fuse.RemoveRequest) error {
+
+	var entry *filer_pb.Entry
+	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir.Path,
+			Name:      req.Name,
+		}
+
+		glog.V(4).Infof("lookup to-be-removed entry: %v", request)
+		resp, err := client.LookupDirectoryEntry(ctx, request)
+		if err != nil {
+			// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
+			return fuse.ENOENT
+		}
+
+		entry = resp.Entry
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	dir.wfs.deleteFileChunks(entry.Chunks)
+
+	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.DeleteEntryRequest{
+			Directory:    dir.Path,
+			Name:         req.Name,
+			IsDeleteData: false,
+		}
+
+		glog.V(3).Infof("remove file: %v", request)
+		_, err := client.DeleteEntry(ctx, request)
+		if err != nil {
+			glog.V(3).Infof("remove file %s/%s: %v", dir.Path, req.Name, err)
+			return fuse.ENOENT
+		}
+
+		dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
+
+		return nil
+	})
+
+}
+
+func (dir *Dir) removeFolder(ctx context.Context, req *fuse.RemoveRequest) error {
+
 	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir.Path,
 			Name:         req.Name,
-			IsDirectory:  req.Dir,
 			IsDeleteData: true,
 		}
 
-		glog.V(1).Infof("remove directory entry: %v", request)
+		glog.V(3).Infof("remove directory entry: %v", request)
 		_, err := client.DeleteEntry(ctx, request)
 		if err != nil {
-			glog.V(0).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
-			return fuse.EIO
+			glog.V(3).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
+			return fuse.ENOENT
 		}
 
 		dir.wfs.listDirectoryEntriesCache.Delete(path.Join(dir.Path, req.Name))
@@ -297,6 +379,10 @@ func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
 
 func (dir *Dir) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
 
+	if dir.attributes == nil {
+		return nil
+	}
+
 	glog.V(3).Infof("%v dir setattr %+v, fh=%d", dir.Path, req, req.Handle)
 	if req.Valid.Mode() {
 		dir.attributes.FileMode = uint32(req.Mode)
@@ -352,10 +438,7 @@ func estimatedCacheTtl(numEntries int) time.Duration {
 		// 10 ms per entry
 		return 100 * time.Second
 	}
-	if numEntries < 100000 {
-		// 10 ms per entry
-		return 1000 * time.Second
-	}
+
 	// 2 ms per entry
 	return time.Duration(numEntries*2) * time.Millisecond
 }

weed/filesys/dir_link.go

@@ -0,0 +1,66 @@
+package filesys
+
+import (
+	"context"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
+)
+
+var _ = fs.NodeSymlinker(&Dir{})
+var _ = fs.NodeReadlinker(&File{})
+
+func (dir *Dir) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {
+
+	glog.V(3).Infof("Symlink: %v/%v to %v", dir.Path, req.NewName, req.Target)
+
+	request := &filer_pb.CreateEntryRequest{
+		Directory: dir.Path,
+		Entry: &filer_pb.Entry{
+			Name:        req.NewName,
+			IsDirectory: false,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:         time.Now().Unix(),
+				Crtime:        time.Now().Unix(),
+				FileMode:      uint32(os.FileMode(0755) | os.ModeSymlink),
+				Uid:           req.Uid,
+				Gid:           req.Gid,
+				SymlinkTarget: req.Target,
+			},
+		},
+	}
+
+	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		if _, err := client.CreateEntry(ctx, request); err != nil {
+			glog.V(0).Infof("symlink %s/%s: %v", dir.Path, req.NewName, err)
+			return fuse.EIO
+		}
+		return nil
+	})
+
+	symlink := dir.newFile(req.NewName, request.Entry)
+
+	return symlink, err
+
+}
+
+func (file *File) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
+
+	if err := file.maybeLoadAttributes(ctx); err != nil {
+		return "", err
+	}
+
+	if os.FileMode(file.entry.Attributes.FileMode)&os.ModeSymlink == 0 {
+		return "", fuse.Errno(syscall.EINVAL)
+	}
+
+	glog.V(3).Infof("Readlink: %v/%v => %v", file.dir.Path, file.Name, file.entry.Attributes.SymlinkTarget)
+
+	return file.entry.Attributes.SymlinkTarget, nil
+
+}

weed/filesys/dir_rename.go

@@ -1,12 +1,14 @@
 package filesys
 
 import (
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"context"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
+	"math"
+	"path/filepath"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"path/filepath"
 )
 
 func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {
@@ -24,7 +26,7 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 		glog.V(4).Infof("find existing directory entry: %v", request)
 		resp, err := client.LookupDirectoryEntry(ctx, request)
 		if err != nil {
-			glog.V(0).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
+			glog.V(3).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
 			return fuse.ENOENT
 		}
 
@@ -39,22 +41,38 @@ func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirector
 
 func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParent string, entry *filer_pb.Entry, newParent, newName string) error {
 	if entry.IsDirectory {
-		currentDirPath := filepath.Join(oldParent, entry.Name)
-		request := &filer_pb.ListEntriesRequest{
-			Directory: currentDirPath,
-		}
+		currentDirPath := filepath.ToSlash(filepath.Join(oldParent, entry.Name))
 
-		glog.V(4).Infof("read directory: %v", request)
-		resp, err := client.ListEntries(ctx, request)
-		if err != nil {
-			glog.V(0).Infof("list %s: %v", oldParent, err)
-			return fuse.EIO
-		}
-
-		for _, item := range resp.Entries {
-			err := moveEntry(ctx, client, currentDirPath, item, filepath.Join(newParent, newName), item.Name)
+		lastFileName := ""
+		includeLastFile := false
+		limit := math.MaxInt32
+		for limit > 0 {
+			request := &filer_pb.ListEntriesRequest{
+				Directory:          currentDirPath,
+				StartFromFileName:  lastFileName,
+				InclusiveStartFrom: includeLastFile,
+				Limit:              1024,
+			}
+			glog.V(4).Infof("read directory: %v", request)
+			resp, err := client.ListEntries(ctx, request)
 			if err != nil {
-				return err
+				glog.V(0).Infof("list %s: %v", oldParent, err)
+				return fuse.EIO
+			}
+			if len(resp.Entries) == 0 {
+				break
+			}
+
+			for _, item := range resp.Entries {
+				lastFileName = item.Name
+				err := moveEntry(ctx, client, currentDirPath, item, filepath.ToSlash(filepath.Join(newParent, newName)), item.Name)
+				if err != nil {
+					return err
+				}
+				limit--
+			}
+			if len(resp.Entries) < 1024 {
+				break
 			}
 		}
 
@@ -84,7 +102,6 @@ func moveEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, oldParen
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    oldParent,
 			Name:         entry.Name,
-			IsDirectory:  entry.IsDirectory,
 			IsDeleteData: false,
 		}
 

weed/filesys/dirty_page.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"sync"
+	"github.com/chrislusf/seaweedfs/weed/security"
 )
 
 type ContinuousDirtyPages struct {
@@ -23,19 +25,22 @@ type ContinuousDirtyPages struct {
 
 func newDirtyPages(file *File) *ContinuousDirtyPages {
 	return &ContinuousDirtyPages{
-		Data: make([]byte, file.wfs.option.ChunkSizeLimit),
+		Data: nil,
 		f:    file,
 	}
 }
 
-func (pages *ContinuousDirtyPages) InitializeToFile(file *File) *ContinuousDirtyPages {
-	if len(pages.Data) != int(file.wfs.option.ChunkSizeLimit) {
-		pages.Data = make([]byte, file.wfs.option.ChunkSizeLimit)
+func (pages *ContinuousDirtyPages) releaseResource() {
+	if pages.Data != nil {
+		pages.f.wfs.bufPool.Put(pages.Data)
+		pages.Data = nil
+		atomic.AddInt32(&counter, -1)
+		glog.V(3).Infof("%s/%s releasing resource %d", pages.f.dir.Path, pages.f.Name, counter)
 	}
-	pages.f = file
-	return pages
 }
 
+var counter = int32(0)
+
 func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) {
 
 	pages.lock.Lock()
@@ -43,11 +48,17 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da
 
 	var chunk *filer_pb.FileChunk
 
-	if len(data) > len(pages.Data) {
+	if len(data) > int(pages.f.wfs.option.ChunkSizeLimit) {
 		// this is more than what buffer can hold.
 		return pages.flushAndSave(ctx, offset, data)
 	}
 
+	if pages.Data == nil {
+		pages.Data = pages.f.wfs.bufPool.Get().([]byte)
+		atomic.AddInt32(&counter, 1)
+		glog.V(3).Infof("%s/%s acquire resource %d", pages.f.dir.Path, pages.f.Name, counter)
+	}
+
 	if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) ||
 		pages.Offset+int64(len(pages.Data)) < offset+int64(len(data)) {
 		// if the data is out of range,
@@ -154,6 +165,7 @@ func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Contex
 func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) {
 
 	var fileId, host string
+	var auth security.EncodedJwt
 
 	if err := pages.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
@@ -171,7 +183,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 			return err
 		}
 
-		fileId, host = resp.FileId, resp.Url
+		fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
 
 		return nil
 	}); err != nil {
@@ -180,7 +192,7 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte
 
 	fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)
 	bufReader := bytes.NewReader(buf)
-	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, "")
+	uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, auth)
 	if err != nil {
 		glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err)
 		return nil, fmt.Errorf("upload data: %v", err)

weed/filesys/file.go

@@ -1,15 +1,17 @@
 package filesys
 
 import (
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"context"
+	"os"
+	"path/filepath"
+	"sort"
+	"time"
+
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"os"
-	"path/filepath"
-	"time"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 )
 
 const blockSize = 512
@@ -20,11 +22,12 @@ var _ = fs.NodeFsyncer(&File{})
 var _ = fs.NodeSetattrer(&File{})
 
 type File struct {
-	Name   string
-	dir    *Dir
-	wfs    *WFS
-	entry  *filer_pb.Entry
-	isOpen bool
+	Name           string
+	dir            *Dir
+	wfs            *WFS
+	entry          *filer_pb.Entry
+	entryViewCache []filer2.VisibleInterval
+	isOpen         bool
 }
 
 func (file *File) fullpath() string {
@@ -71,10 +74,6 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		return err
 	}
 
-	if file.isOpen {
-		return nil
-	}
-
 	glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
 	if req.Valid.Size() {
 
@@ -82,6 +81,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		if req.Size == 0 {
 			// fmt.Printf("truncate %v \n", fullPath)
 			file.entry.Chunks = nil
+			file.entryViewCache = nil
 		}
 		file.entry.Attributes.FileSize = req.Size
 	}
@@ -97,10 +97,18 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
 		file.entry.Attributes.Gid = req.Gid
 	}
 
+	if req.Valid.Crtime() {
+		file.entry.Attributes.Crtime = req.Crtime.Unix()
+	}
+
 	if req.Valid.Mtime() {
 		file.entry.Attributes.Mtime = req.Mtime.Unix()
 	}
 
+	if file.isOpen {
+		return nil
+	}
+
 	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.UpdateEntryRequest{
@@ -133,7 +141,7 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 		item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
 		if item != nil && !item.Expired() {
 			entry := item.Value().(*filer_pb.Entry)
-			file.entry = entry
+			file.setEntry(entry)
 			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
 		} else {
 			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
@@ -145,15 +153,15 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 
 				resp, err := client.LookupDirectoryEntry(ctx, request)
 				if err != nil {
-					glog.V(0).Infof("file attr read file %v: %v", request, err)
-					return err
+					glog.V(3).Infof("file attr read file %v: %v", request, err)
+					return fuse.ENOENT
 				}
 
-				file.entry = resp.Entry
+				file.setEntry(resp.Entry)
 
-				glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))
+				glog.V(3).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))
 
-				file.wfs.listDirectoryEntriesCache.Set(file.fullpath(), file.entry, file.wfs.option.EntryCacheTtl)
+				// file.wfs.listDirectoryEntriesCache.Set(file.fullpath(), file.entry, file.wfs.option.EntryCacheTtl)
 
 				return nil
 			})
@@ -165,3 +173,31 @@ func (file *File) maybeLoadAttributes(ctx context.Context) error {
 	}
 	return nil
 }
+
+func (file *File) addChunk(chunk *filer_pb.FileChunk) {
+	if chunk != nil {
+		file.addChunks([]*filer_pb.FileChunk{chunk})
+	}
+}
+
+func (file *File) addChunks(chunks []*filer_pb.FileChunk) {
+
+	sort.Slice(chunks, func(i, j int) bool {
+		return chunks[i].Mtime < chunks[j].Mtime
+	})
+
+	var newVisibles []filer2.VisibleInterval
+	for _, chunk := range chunks {
+		newVisibles = filer2.MergeIntoVisibles(file.entryViewCache, newVisibles, chunk)
+		t := file.entryViewCache[:0]
+		file.entryViewCache = newVisibles
+		newVisibles = t
+	}
+
+	file.entry.Chunks = append(file.entry.Chunks, chunks...)
+}
+
+func (file *File) setEntry(entry *filer_pb.Entry) {
+	file.entry = entry
+	file.entryViewCache = filer2.NonOverlappingVisibleIntervals(file.entry.Chunks)
+}

weed/filesys/filehandle.go

@@ -1,17 +1,20 @@
 package filesys
 
 import (
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
+	"google.golang.org/grpc"
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 )
 
 type FileHandle struct {
@@ -37,16 +40,6 @@ func newFileHandle(file *File, uid, gid uint32) *FileHandle {
 	}
 }
 
-func (fh *FileHandle) InitializeToFile(file *File, uid, gid uint32) *FileHandle {
-	newHandle := &FileHandle{
-		f:          file,
-		dirtyPages: fh.dirtyPages.InitializeToFile(file),
-		Uid:        uid,
-		Gid:        gid,
-	}
-	return newHandle
-}
-
 var _ = fs.Handle(&FileHandle{})
 
 // var _ = fs.HandleReadAller(&FileHandle{})
@@ -67,7 +60,11 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus
 
 	buff := make([]byte, req.Size)
 
-	chunkViews := filer2.ViewFromChunks(fh.f.entry.Chunks, req.Offset, req.Size)
+	if fh.f.entryViewCache == nil {
+		fh.f.entryViewCache = filer2.NonOverlappingVisibleIntervals(fh.f.entry.Chunks)
+	}
+
+	chunkViews := filer2.ViewFromVisibleIntervals(fh.f.entryViewCache, req.Offset, req.Size)
 
 	var vids []string
 	for _, chunkView := range chunkViews {
@@ -161,9 +158,9 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 		fh.dirtyMetadata = true
 	}
 
-	for _, chunk := range chunks {
-		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
-		glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+	fh.f.addChunks(chunks)
+
+	if len(chunks) > 0 {
 		fh.dirtyMetadata = true
 	}
 
@@ -174,6 +171,8 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 
 	glog.V(4).Infof("%v release fh %d", fh.f.fullpath(), fh.handle)
 
+	fh.dirtyPages.releaseResource()
+
 	fh.f.wfs.ReleaseHandle(fh.f.fullpath(), fuse.HandleID(fh.handle))
 
 	fh.f.isOpen = false
@@ -181,8 +180,6 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 	return nil
 }
 
-// Flush - experimenting with uploading at flush, this slows operations down till it has been
-// completely flushed
 func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	// fflush works at fh level
 	// send the data to the OS
@@ -193,9 +190,8 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 		glog.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
 		return fmt.Errorf("flush %s/%s: %v", fh.f.dir.Path, fh.f.Name, err)
 	}
-	if chunk != nil {
-		fh.f.entry.Chunks = append(fh.f.entry.Chunks, chunk)
-	}
+
+	fh.f.addChunk(chunk)
 
 	if !fh.dirtyMetadata {
 		return nil
@@ -207,6 +203,9 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 			fh.f.entry.Attributes.Mime = fh.contentType
 			fh.f.entry.Attributes.Uid = req.Uid
 			fh.f.entry.Attributes.Gid = req.Gid
+			fh.f.entry.Attributes.Mtime = time.Now().Unix()
+			fh.f.entry.Attributes.Crtime = time.Now().Unix()
+			fh.f.entry.Attributes.FileMode = uint32(0770)
 		}
 
 		request := &filer_pb.CreateEntryRequest{
@@ -214,10 +213,16 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 			Entry:     fh.f.entry,
 		}
 
-		glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
-		for i, chunk := range fh.f.entry.Chunks {
-			glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
-		}
+		//glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.entry.Chunks))
+		//for i, chunk := range fh.f.entry.Chunks {
+		//	glog.V(4).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size))
+		//}
+
+		chunks, garbages := filer2.CompactFileChunks(fh.f.entry.Chunks)
+		fh.f.entry.Chunks = chunks
+		// fh.f.entryViewCache = nil
+		fh.f.wfs.deleteFileChunks(garbages)
+
 		if _, err := client.CreateEntry(ctx, request); err != nil {
 			return fmt.Errorf("update fh: %v", err)
 		}
@@ -226,6 +231,48 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 	})
 }
 
+func deleteFileIds(ctx context.Context, grpcDialOption grpc.DialOption, client filer_pb.SeaweedFilerClient, fileIds []string) error {
+
+	var vids []string
+	for _, fileId := range fileIds {
+		vids = append(vids, volumeId(fileId))
+	}
+
+	lookupFunc := func(vids []string) (map[string]operation.LookupResult, error) {
+
+		m := make(map[string]operation.LookupResult)
+
+		glog.V(4).Infof("remove file lookup volume id locations: %v", vids)
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+			VolumeIds: vids,
+		})
+		if err != nil {
+			return m, err
+		}
+
+		for _, vid := range vids {
+			lr := operation.LookupResult{
+				VolumeId:  vid,
+				Locations: nil,
+			}
+			locations := resp.LocationsMap[vid]
+			for _, loc := range locations.Locations {
+				lr.Locations = append(lr.Locations, operation.Location{
+					Url:       loc.Url,
+					PublicUrl: loc.PublicUrl,
+				})
+			}
+			m[vid] = lr
+		}
+
+		return m, err
+	}
+
+	_, err := operation.DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
+
+	return err
+}
+
 func volumeId(fileId string) string {
 	lastCommaIndex := strings.LastIndex(fileId, ",")
 	if lastCommaIndex > 0 {

weed/filesys/wfs.go

@@ -4,20 +4,22 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"os"
 	"sync"
 	"time"
 
-	"bazil.org/fuse"
-	"bazil.org/fuse/fs"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/karlseguin/ccache"
+	"github.com/seaweedfs/fuse"
+	"github.com/seaweedfs/fuse/fs"
 	"google.golang.org/grpc"
 )
 
 type Option struct {
 	FilerGrpcAddress   string
+	GrpcDialOption     grpc.DialOption
 	FilerMountRootPath string
 	Collection         string
 	Replication        string
@@ -26,6 +28,10 @@ type Option struct {
 	DataCenter         string
 	DirListingLimit    int
 	EntryCacheTtl      time.Duration
+
+	MountUid  uint32
+	MountGid  uint32
+	MountMode os.FileMode
 }
 
 var _ = fs.FS(&WFS{})
@@ -39,6 +45,7 @@ type WFS struct {
 	handles           []*FileHandle
 	pathToHandleIndex map[string]int
 	pathToHandleLock  sync.Mutex
+	bufPool           sync.Pool
 
 	stats statsCache
 }
@@ -48,11 +55,18 @@ type statsCache struct {
 }
 
 func NewSeaweedFileSystem(option *Option) *WFS {
-	return &WFS{
+	wfs := &WFS{
 		option:                    option,
-		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(int64(option.DirListingLimit) + 200).ItemsToPrune(100)),
+		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(1024 * 8).ItemsToPrune(100)),
 		pathToHandleIndex:         make(map[string]int),
+		bufPool: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, option.ChunkSizeLimit)
+			},
+		},
 	}
+
+	return wfs
 }
 
 func (wfs *WFS) Root() (fs.Node, error) {
@@ -64,7 +78,7 @@ func (wfs *WFS) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) erro
 	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
 		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
 		return fn(client)
-	}, wfs.option.FilerGrpcAddress)
+	}, wfs.option.FilerGrpcAddress, wfs.option.GrpcDialOption)
 
 }
 
@@ -76,17 +90,10 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
 
 	index, found := wfs.pathToHandleIndex[fullpath]
 	if found && wfs.handles[index] != nil {
-		glog.V(4).Infoln(fullpath, "found fileHandle id", index)
+		glog.V(2).Infoln(fullpath, "found fileHandle id", index)
 		return wfs.handles[index]
 	}
 
-	if found && wfs.handles[index] != nil {
-		glog.V(4).Infoln(fullpath, "reuse previous fileHandle id", index)
-		wfs.handles[index].InitializeToFile(file, uid, gid)
-		fileHandle.handle = uint64(index)
-		return
-	}
-
 	fileHandle = newFileHandle(file, uid, gid)
 	for i, h := range wfs.handles {
 		if h == nil {
@@ -100,7 +107,7 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
 
 	wfs.handles = append(wfs.handles, fileHandle)
 	fileHandle.handle = uint64(len(wfs.handles) - 1)
-	glog.V(4).Infoln(fullpath, "new fileHandle id", fileHandle.handle)
+	glog.V(2).Infoln(fullpath, "new fileHandle id", fileHandle.handle)
 	wfs.pathToHandleIndex[fullpath] = int(fileHandle.handle)
 
 	return
@@ -163,11 +170,11 @@ func (wfs *WFS) Statfs(ctx context.Context, req *fuse.StatfsRequest, resp *fuse.
 	resp.Blocks = totalDiskSize / blockSize
 
 	// Compute the number of used blocks
-	numblocks := uint64(usedDiskSize / blockSize)
+	numBlocks := uint64(usedDiskSize / blockSize)
 
 	// Report the number of free and available blocks for the block size
-	resp.Bfree = resp.Blocks - numblocks
-	resp.Bavail = resp.Blocks - numblocks
+	resp.Bfree = resp.Blocks - numBlocks
+	resp.Bavail = resp.Blocks - numBlocks
 	resp.Bsize = uint32(blockSize)
 
 	// Report the total number of possible files in the file system (and those free)

weed/filesys/wfs_deletion.go

@@ -0,0 +1,22 @@
+package filesys
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+)
+
+func (wfs *WFS) deleteFileChunks(chunks []*filer_pb.FileChunk) {
+	if len(chunks) == 0 {
+		return
+	}
+
+	var fileIds []string
+	for _, chunk := range chunks {
+		fileIds = append(fileIds, chunk.FileId)
+	}
+
+	wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+		deleteFileIds(context.Background(), wfs.option.GrpcDialOption, client, fileIds)
+		return nil
+	})
+}

weed/glide.lock

@@ -1,66 +1,139 @@
-hash: 2e3a065472829938d25e879451b6d1aa43e55270e1166a9c044803ef8a3b9eb1
-updated: 2018-06-28T22:01:35.910567-07:00
+hash: 306ab43df769fe8072733ba28c3d2b6308288f248ee65806ac7d0bfa3349ab73
+updated: 2019-02-25T00:25:56.429024-08:00
 imports:
-- name: bazil.org/fuse
-  version: 65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e
+- name: cloud.google.com/go
+  version: e2c125ceac8b663cfcf4610477d4d67827377cb7
   subpackages:
-  - fs
-  - fuseutil
+  - compute/metadata
+  - iam
+  - internal
+  - internal/optional
+  - internal/trace
+  - internal/version
+  - pubsub
+  - pubsub/apiv1
+  - pubsub/internal/distribution
+  - storage
+- name: github.com/aws/aws-sdk-go
+  version: 5604f1add1ce6b18465fd50f7fe8de7561cc8a62
+  subpackages:
+  - aws
+  - aws/awserr
+  - aws/awsutil
+  - aws/client
+  - aws/client/metadata
+  - aws/corehandlers
+  - aws/credentials
+  - aws/credentials/ec2rolecreds
+  - aws/credentials/endpointcreds
+  - aws/credentials/processcreds
+  - aws/credentials/stscreds
+  - aws/csm
+  - aws/defaults
+  - aws/ec2metadata
+  - aws/endpoints
+  - aws/request
+  - aws/session
+  - aws/signer/v4
+  - internal/ini
+  - internal/s3err
+  - internal/sdkio
+  - internal/sdkrand
+  - internal/sdkuri
+  - internal/shareddefaults
+  - private/protocol
+  - private/protocol/eventstream
+  - private/protocol/eventstream/eventstreamapi
+  - private/protocol/query
+  - private/protocol/query/queryutil
+  - private/protocol/rest
+  - private/protocol/restxml
+  - private/protocol/xml/xmlutil
+  - service/s3
+  - service/s3/s3iface
+  - service/sqs
+  - service/sts
+- name: github.com/Azure/azure-pipeline-go
+  version: 76b57228f36adfbb7e6990ba1347a7fbbf3043da
+  subpackages:
+  - pipeline
+- name: github.com/Azure/azure-storage-blob-go
+  version: 457680cc0804810f6d02958481e0ffdda51d5c60
+  subpackages:
+  - azblob
 - name: github.com/boltdb/bolt
-  version: 2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8
+  version: fd01fc79c553a8e99d512a07e8e0c63d4a3ccfc5
 - name: github.com/chrislusf/raft
-  version: 5f7ddd8f479583daf05879d3d3b174aa202c8fb7
+  version: 10d6e2182d923e93ec0cc1aa1d556e5b1f8a39e7
   subpackages:
   - protobuf
+- name: github.com/DataDog/zstd
+  version: 1e382f59b41eebd6f592c5db4fd1958ec38a0eba
+- name: github.com/davecgh/go-spew
+  version: d8f796af33cc11cb798c1aaeb27a4ebc5099927d
+  subpackages:
+  - spew
 - name: github.com/dgrijalva/jwt-go
-  version: 06ea1031745cb8b3dab3f6a236daf2b0aa468b7e
+  version: 3af4c746e1c248ee8491a3e0c6f7a9cd831e95f8
 - name: github.com/disintegration/imaging
-  version: bbcee2f5c9d5e94ca42c8b50ec847fec64a6c134
+  version: 5362c131d56305ce787e79a5b94ffc956df00d62
+- name: github.com/dustin/go-humanize
+  version: 9f541cc9db5d55bce703bd99987c9d5cb8eea45e
+- name: github.com/eapache/go-resiliency
+  version: 842e16ec2c98ef0c59eebfe60d2d3500a793ba19
+  subpackages:
+  - breaker
+- name: github.com/eapache/go-xerial-snappy
+  version: 776d5712da21bc4762676d614db1d8a64f4238b0
+- name: github.com/eapache/queue
+  version: 093482f3f8ce946c05bcba64badd2c82369e084d
 - name: github.com/fsnotify/fsnotify
-  version: c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9
+  version: ccc981bf80385c528a65fbfdd49bf2d8da22aa23
 - name: github.com/go-redis/redis
-  version: 83fb42932f6145ce52df09860384a4653d2d332a
+  version: bd542089bb6e776e6fced5038edac8a0f526aa53
   subpackages:
   - internal
   - internal/consistenthash
   - internal/hashtag
   - internal/pool
   - internal/proto
-  - internal/singleflight
   - internal/util
 - name: github.com/go-sql-driver/mysql
-  version: d523deb1b23d913de5bdada721a6071e71283618
+  version: 972a708cf97995463843c08c8585b26997daf0e1
 - name: github.com/gocql/gocql
-  version: e06f8c1bcd787e6bf0608288b314522f08cc7848
+  version: ec4793573d1447b6f92a1b359a0594566fad9d0e
   subpackages:
   - internal/lru
   - internal/murmur
   - internal/streams
-- name: github.com/gogo/protobuf
-  version: 30cf7ac33676b5786e78c746683f0d4cd64fa75b
-  subpackages:
-  - proto
 - name: github.com/golang/protobuf
-  version: b4deda0973fb4c70b50d226b1af49f3da59f5265
+  version: c823c79ea1570fb5ff454033735a8e68575d1d0f
   subpackages:
   - proto
   - protoc-gen-go/descriptor
   - ptypes
   - ptypes/any
   - ptypes/duration
+  - ptypes/empty
   - ptypes/timestamp
 - name: github.com/golang/snappy
-  version: 2e65f85255dbc3072edf28d6b5b8efc472979f5a
+  version: 2a8bb927dd31d8daada140a5d09578521ce5c36a
 - name: github.com/google/btree
-  version: e89373fe6b4a7413d7acd6da1725b83ef713e6e4
-- name: github.com/gorilla/context
-  version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
+  version: 4030bb1f1f0c35b30ca7009e9ebd06849dd45306
+- name: github.com/googleapis/gax-go
+  version: ddfab93c3faef4935403ac75a7c11f0e731dc181
+  subpackages:
+  - v2
 - name: github.com/gorilla/mux
-  version: e3702bed27f0d39777b0b37b664b6280e8ef8fbf
+  version: 8559a4f775fc329165fe32bd4c2543de8ada8fce
 - name: github.com/hailocab/go-hostpool
   version: e80d13ce29ede4452c43dea11e79b9bc8a15b478
+- name: github.com/hashicorp/golang-lru
+  version: 20f1fb78b0740ba8c3cb143a61e86ba5c8669768
+  subpackages:
+  - simplelru
 - name: github.com/hashicorp/hcl
-  version: ef8a98b0bbce4a65b5aa4c368430a80ddc533168
+  version: 65a6292f0157eff210d03ed1bf6c59b190b8b906
   subpackages:
   - hcl/ast
   - hcl/parser
@@ -71,41 +144,71 @@ imports:
   - json/parser
   - json/scanner
   - json/token
+- name: github.com/jmespath/go-jmespath
+  version: c2b33e8439af944379acbdd9c3a5fe0bc44bd8a5
 - name: github.com/karlseguin/ccache
-  version: b425c9ca005a2050ebe723f6a0cddcb907354ab7
+  version: ec06cd93a07565b373789b0078ba88fe697fddd9
 - name: github.com/klauspost/crc32
-  version: cb6bfca970f6908083f26f39a79009d608efd5cd
+  version: bab58d77464aa9cf4e84200c3276da0831fe0c03
+- name: github.com/kurin/blazer
+  version: f20ef4f2aa8ccc2e94a1981dc37f199e90fa4ba5
+  subpackages:
+  - b2
+  - base
+  - internal/b2assets
+  - internal/b2types
+  - internal/blog
+  - x/window
 - name: github.com/lib/pq
-  version: 90697d60dd844d5ef6ff15135d0203f65d2f53b8
+  version: 9eb73efc1fcc404148b56765b0d3f61d9a5ef8ee
   subpackages:
   - oid
 - name: github.com/magiconair/properties
-  version: c2353362d570a7bfa228149c62842019201cfb71
+  version: 7757cc9fdb852f7579b24170bcacda2c7471bb6a
 - name: github.com/mitchellh/mapstructure
-  version: bb74f1db0675b241733089d5a1faa5dd8b0ef57b
+  version: 3536a929edddb9a5b34bd6861dc4a9647cb459fe
 - name: github.com/pelletier/go-toml
-  version: c01d1270ff3e442a8a57cddc1c92dc1138598194
+  version: 27c6b39a135b7dc87a14afb068809132fb7a9a8f
+- name: github.com/pierrec/lz4
+  version: 062282ea0dcff40c9fb8525789eef9644b1fbd6e
+  subpackages:
+  - internal/xxh32
+- name: github.com/rakyll/statik
+  version: 79258177a57a85a8ab2eca7ce0936aad80307f4e
+  subpackages:
+  - fs
+- name: github.com/rcrowley/go-metrics
+  version: 3113b8401b8a98917cde58f8bbd42a1b1c03b1fd
 - name: github.com/rwcarlsen/goexif
-  version: 8d986c03457a2057c7b0fb0a48113f7dd48f9619
+  version: b1fd11e07dc5bc0d2ca3b79d28cbdf3c6d186247
   subpackages:
   - exif
   - tiff
-- name: github.com/soheilhy/cmux
-  version: e09e9389d85d8492d313d73d1469c029e710623f
+- name: github.com/satori/go.uuid
+  version: b2ce2384e17bbe0c6d34077efa39dbab3e09123b
+- name: github.com/seaweedfs/fuse
+  version: 1aae43e32cadcfa182fc60777f20fb02673e8f82
+  subpackages:
+  - fs
+  - fuseutil
+- name: github.com/Shopify/sarama
+  version: 4602b5a8c6e826f9e0737865818dd43b2339a092
+- name: github.com/spaolacci/murmur3
+  version: f09979ecbc725b9e6d41a297405f65e7e8804acc
 - name: github.com/spf13/afero
-  version: 787d034dfe70e44075ccc060d346146ef53270ad
+  version: f4711e4db9e9a1d3887343acb72b2bbfc2f686f5
   subpackages:
   - mem
 - name: github.com/spf13/cast
-  version: 8965335b8c7107321228e3e3702cab9832751bac
+  version: 8c9545af88b134710ab1cd196795e7f2388358d7
 - name: github.com/spf13/jwalterweatherman
-  version: 7c0cea34c8ece3fbeb2b27ab9b59511d360fb394
+  version: 94f6ae3ed3bceceafa716478c5fbf8d29ca601a1
 - name: github.com/spf13/pflag
-  version: 3ebe029320b2676d667ae88da602a5f854788a8a
+  version: 24fa6976df40757dce6aea913e7b81ade90530e1
 - name: github.com/spf13/viper
-  version: 15738813a09db5c8e5b60a19d67d3f9bd38da3a4
+  version: d104d259b3380cb653bb793756823c3c41b37b53
 - name: github.com/syndtr/goleveldb
-  version: 0d5a0ceb10cf9ab89fdd744cc8c50a83134f6697
+  version: 9d007e481048296f09f59bd19bb7ae584563cd95
   subpackages:
   - leveldb
   - leveldb/cache
@@ -119,57 +222,117 @@ imports:
   - leveldb/storage
   - leveldb/table
   - leveldb/util
+- name: github.com/willf/bitset
+  version: 20ad246f50b49590afcb1ed8ad143da7163869cb
+- name: github.com/willf/bloom
+  version: 54e3b963ee1652b06c4562cb9b6020ebc6e36e59
+- name: go.opencensus.io
+  version: beafb2a85a579a4918ba259877a1625e9213a263
+  subpackages:
+  - exemplar
+  - internal
+  - internal/tagencoding
+  - plugin/ocgrpc
+  - plugin/ochttp
+  - plugin/ochttp/propagation/b3
+  - stats
+  - stats/internal
+  - stats/view
+  - tag
+  - trace
+  - trace/internal
+  - trace/propagation
+  - trace/tracestate
 - name: golang.org/x/image
-  version: cc896f830cedae125428bc9fe1b0362aa91b3fb1
+  version: 31aff87c08e9a5e5d524279a564f96968336f886
   subpackages:
   - bmp
   - tiff
   - tiff/lzw
 - name: golang.org/x/net
-  version: 4cb1c02c05b0e749b0365f61ae859a8e0cfceed9
+  version: 3a22650c66bd7f4fb6d1e8072ffd7b75c8a27898
   subpackages:
   - context
+  - context/ctxhttp
   - http/httpguts
   - http2
   - http2/hpack
   - idna
   - internal/timeseries
   - trace
-- name: golang.org/x/sys
-  version: 7138fd3d9dc8335c567ca206f4333fb75eb05d56
+- name: golang.org/x/oauth2
+  version: 9b3c75971fc92dd27c6436a37c05c831498658f1
   subpackages:
-  - unix
+  - google
+  - internal
+  - jws
+  - jwt
+- name: golang.org/x/sync
+  version: 37e7f081c4d4c64e13b10787722085407fe5d15f
+  subpackages:
+  - errgroup
+  - semaphore
 - name: golang.org/x/text
-  version: 5cec4b58c438bd98288aeb248bab2c1840713d21
+  version: 6c92c7dc7f53607809182301b96e4cc1975143f1
   subpackages:
   - secure/bidirule
   - transform
   - unicode/bidi
   - unicode/norm
-- name: google.golang.org/appengine
-  version: b1f26356af11148e710935ed1ac8a7f5702c7612
+- name: golang.org/x/tools
+  version: 83362c3779f5f48611068d488a03ea7bbaddc81e
   subpackages:
-  - cloudsql
+  - godoc/util
+  - godoc/vfs
+- name: google.golang.org/api
+  version: 8a550ba84cafabe9b2262c41303f31e5a4626318
+  subpackages:
+  - gensupport
+  - googleapi
+  - googleapi/internal/uritemplates
+  - googleapi/transport
+  - internal
+  - iterator
+  - option
+  - storage/v1
+  - support/bundler
+  - transport
+  - transport/grpc
+  - transport/http
+  - transport/http/internal/propagation
 - name: google.golang.org/genproto
-  version: ff3583edef7de132f219f0efc00e097cabcc0ec0
+  version: 082222b4a5c572e33e82ee9162d1352c7cf38682
   subpackages:
+  - googleapis/api/annotations
+  - googleapis/iam/v1
+  - googleapis/pubsub/v1
+  - googleapis/rpc/code
   - googleapis/rpc/status
+  - protobuf/field_mask
 - name: google.golang.org/grpc
-  version: 168a6198bcb0ef175f7dacec0b8691fc141dc9b8
+  version: 2773c7bbcf81cf358d3f0038b1469b2d44062acb
   subpackages:
   - balancer
   - balancer/base
   - balancer/roundrobin
+  - binarylog/grpc_binarylog_v1
   - codes
   - connectivity
   - credentials
+  - credentials/internal
+  - credentials/oauth
   - encoding
   - encoding/proto
   - grpclog
   - internal
   - internal/backoff
+  - internal/binarylog
   - internal/channelz
+  - internal/envconfig
   - internal/grpcrand
+  - internal/grpcsync
+  - internal/syscall
+  - internal/transport
   - keepalive
   - metadata
   - naming
@@ -182,9 +345,8 @@ imports:
   - stats
   - status
   - tap
-  - transport
 - name: gopkg.in/inf.v0
   version: d2d2541c53f18d2a059457998ce2876cc8e67cbf
 - name: gopkg.in/yaml.v2
-  version: 5420a8b6744d3b0345ab293f6fcba19c978f1183
+  version: 51d6538a90f86fe93ac480b35f37b2be17fef232
 testImports: []

weed/glide.yaml

@@ -1,44 +1,71 @@
 package: github.com/chrislusf/seaweedfs/weed
 import:
-- package: bazil.org/fuse
+- package: cloud.google.com/go
   subpackages:
-  - fs
+  - pubsub
+  - storage
+- package: github.com/Azure/azure-storage-blob-go
+  subpackages:
+  - azblob
+- package: github.com/Shopify/sarama
+- package: github.com/aws/aws-sdk-go
+  subpackages:
+  - aws
+  - aws/awserr
+  - aws/credentials
+  - aws/session
+  - service/s3
+  - service/s3/s3iface
+  - service/sqs
 - package: github.com/boltdb/bolt
-  version: ^1.3.1
 - package: github.com/chrislusf/raft
+  subpackages:
+  - protobuf
 - package: github.com/dgrijalva/jwt-go
-  version: ^3.2.0
 - package: github.com/disintegration/imaging
-  version: ^1.4.1
+- package: github.com/dustin/go-humanize
 - package: github.com/go-redis/redis
-  version: ^6.10.2
 - package: github.com/go-sql-driver/mysql
-  version: ^1.3.0
 - package: github.com/gocql/gocql
 - package: github.com/golang/protobuf
-  version: ^1.0.0
   subpackages:
   - proto
 - package: github.com/google/btree
 - package: github.com/gorilla/mux
-  version: ^1.6.1
+- package: github.com/karlseguin/ccache
 - package: github.com/klauspost/crc32
-  version: ^1.1.0
+- package: github.com/kurin/blazer
+  subpackages:
+  - b2
 - package: github.com/lib/pq
+- package: github.com/rakyll/statik
+  subpackages:
+  - fs
 - package: github.com/rwcarlsen/goexif
   subpackages:
   - exif
-- package: github.com/soheilhy/cmux
-  version: ^0.1.4
+- package: github.com/satori/go.uuid
+- package: github.com/seaweedfs/fuse
+  subpackages:
+  - fs
+- package: github.com/spf13/viper
 - package: github.com/syndtr/goleveldb
   subpackages:
   - leveldb
   - leveldb/util
+- package: github.com/willf/bloom
 - package: golang.org/x/net
   subpackages:
   - context
-- package: google.golang.org/grpc
-  version: ^1.11.3
+- package: golang.org/x/tools
   subpackages:
+  - godoc/util
+- package: google.golang.org/api
+  subpackages:
+  - option
+- package: google.golang.org/grpc
+  subpackages:
+  - credentials
+  - keepalive
   - peer
   - reflection

weed/glog/glog.go

@@ -46,7 +46,7 @@
 //	-stderrthreshold=ERROR
 //		Log events at or above this severity are logged to standard
 //		error as well as to files.
-//	-log_dir=""
+//	-logdir=""
 //		Log files will be written to this directory instead of the
 //		default temporary directory.
 //

weed/glog/glog_file.go

@@ -38,7 +38,7 @@ var logDirs []string
 
 // If non-empty, overrides the choice of directory in which to write logs.
 // See createLogDirs for the full list of possible destinations.
-var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory")
+var logDir = flag.String("logdir", "", "If non-empty, write log files in this directory")
 
 func createLogDirs() {
 	if *logDir != "" {

weed/operation/assign_file_id.go

@@ -3,8 +3,11 @@ package operation
 import (
 	"context"
 	"fmt"
-
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"strings"
 )
 
 type VolumeAssignRequest struct {
@@ -18,14 +21,15 @@ type VolumeAssignRequest struct {
 }
 
 type AssignResult struct {
-	Fid       string `json:"fid,omitempty"`
-	Url       string `json:"url,omitempty"`
-	PublicUrl string `json:"publicUrl,omitempty"`
-	Count     uint64 `json:"count,omitempty"`
-	Error     string `json:"error,omitempty"`
+	Fid       string              `json:"fid,omitempty"`
+	Url       string              `json:"url,omitempty"`
+	PublicUrl string              `json:"publicUrl,omitempty"`
+	Count     uint64              `json:"count,omitempty"`
+	Error     string              `json:"error,omitempty"`
+	Auth      security.EncodedJwt `json:"auth,omitempty"`
 }
 
-func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
+func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
 
 	var requests []*VolumeAssignRequest
 	requests = append(requests, primaryRequest)
@@ -39,7 +43,8 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 			continue
 		}
 
-		lastError = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+		lastError = withMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+
 			req := &master_pb.AssignRequest{
 				Count:       primaryRequest.Count,
 				Replication: primaryRequest.Replication,
@@ -59,6 +64,7 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 			ret.Url = resp.Url
 			ret.PublicUrl = resp.PublicUrl
 			ret.Error = resp.Error
+			ret.Auth = security.EncodedJwt(resp.Auth)
 
 			return nil
 
@@ -77,3 +83,17 @@ func Assign(server string, primaryRequest *VolumeAssignRequest, alternativeReque
 
 	return ret, lastError
 }
+
+func LookupJwt(master string, fileId string) security.EncodedJwt {
+
+	tokenStr := ""
+
+	if h, e := util.Head(fmt.Sprintf("http://%s/dir/lookup?fileId=%s", master, fileId)); e == nil {
+		bearer := h.Get("Authorization")
+		if len(bearer) > 7 && strings.ToUpper(bearer[0:6]) == "BEARER" {
+			tokenStr = bearer[7:]
+		}
+	}
+
+	return security.EncodedJwt(tokenStr)
+}

weed/operation/chunked_file.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"google.golang.org/grpc"
 	"io"
 	"net/http"
 	"sort"
@@ -69,12 +70,12 @@ func (cm *ChunkManifest) Marshal() ([]byte, error) {
 	return json.Marshal(cm)
 }
 
-func (cm *ChunkManifest) DeleteChunks(master string) error {
+func (cm *ChunkManifest) DeleteChunks(master string, grpcDialOption grpc.DialOption) error {
 	var fileIds []string
 	for _, ci := range cm.Chunks {
 		fileIds = append(fileIds, ci.Fid)
 	}
-	results, err := DeleteFiles(master, fileIds)
+	results, err := DeleteFiles(master, grpcDialOption, fileIds)
 	if err != nil {
 		glog.V(0).Infof("delete %+v: %v", fileIds, err)
 		return fmt.Errorf("chunk delete: %v", err)

weed/operation/compress.go

@@ -8,21 +8,41 @@ import (
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"golang.org/x/tools/godoc/util"
 )
 
 /*
 * Default more not to gzip since gzip can be done on client side.
  */
-func IsGzippable(ext, mtype string) bool {
+func IsGzippable(ext, mtype string, data []byte) bool {
+
+	// text
 	if strings.HasPrefix(mtype, "text/") {
 		return true
 	}
+
+	// images
+	switch ext {
+	case ".svg", ".bmp":
+		return true
+	}
+	if strings.HasPrefix(mtype, "image/") {
+		return false
+	}
+
+	// by file name extension
 	switch ext {
 	case ".zip", ".rar", ".gz", ".bz2", ".xz":
 		return false
 	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
 		return true
+	case ".php", ".java", ".go", ".rb", ".c", ".cpp", ".h", ".hpp":
+		return true
+	case ".png", ".jpg", ".jpeg":
+		return false
 	}
+
+	// by mime type
 	if strings.HasPrefix(mtype, "application/") {
 		if strings.HasSuffix(mtype, "xml") {
 			return true
@@ -31,7 +51,10 @@ func IsGzippable(ext, mtype string) bool {
 			return true
 		}
 	}
-	return false
+
+	isMostlyText := util.IsText(data)
+
+	return isMostlyText
 }
 
 func GzipData(input []byte) ([]byte, error) {

weed/operation/data_struts.go

@@ -2,6 +2,5 @@ package operation
 
 type JoinResult struct {
 	VolumeSizeLimit uint64 `json:"VolumeSizeLimit,omitempty"`
-	SecretKey       string `json:"secretKey,omitempty"`
 	Error           string `json:"error,omitempty"`
 }

weed/operation/delete_content.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
 	"net/http"
 	"strings"
 	"sync"
-
-	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 )
 
 type DeleteResult struct {
@@ -27,17 +27,17 @@ func ParseFileId(fid string) (vid string, key_cookie string, err error) {
 }
 
 // DeleteFiles batch deletes a list of fileIds
-func DeleteFiles(master string, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
+func DeleteFiles(master string, grpcDialOption grpc.DialOption, fileIds []string) ([]*volume_server_pb.DeleteResult, error) {
 
 	lookupFunc := func(vids []string) (map[string]LookupResult, error) {
-		return LookupVolumeIds(master, vids)
+		return LookupVolumeIds(master, grpcDialOption, vids)
 	}
 
-	return DeleteFilesWithLookupVolumeId(fileIds, lookupFunc)
+	return DeleteFilesWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
 
 }
 
-func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []string) (map[string]LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
+func DeleteFilesWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]LookupResult, error)) ([]*volume_server_pb.DeleteResult, error) {
 
 	var ret []*volume_server_pb.DeleteResult
 
@@ -47,7 +47,7 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 		vid, _, err := ParseFileId(fileId)
 		if err != nil {
 			ret = append(ret, &volume_server_pb.DeleteResult{
-				FileId: vid,
+				FileId: fileId,
 				Status: http.StatusBadRequest,
 				Error:  err.Error()},
 			)
@@ -91,7 +91,7 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 		go func(server string, fidList []string) {
 			defer wg.Done()
 
-			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, fidList); deleteErr != nil {
+			if deleteResults, deleteErr := DeleteFilesAtOneVolumeServer(server, grpcDialOption, fidList); deleteErr != nil {
 				err = deleteErr
 			} else {
 				ret = append(ret, deleteResults...)
@@ -105,9 +105,9 @@ func DeleteFilesWithLookupVolumeId(fileIds []string, lookupFunc func(vid []strin
 }
 
 // DeleteFilesAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
-func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
+func DeleteFilesAtOneVolumeServer(volumeServer string, grpcDialOption grpc.DialOption, fileIds []string) (ret []*volume_server_pb.DeleteResult, err error) {
 
-	err = WithVolumeServerClient(volumeServer, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+	err = WithVolumeServerClient(volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
 
 		req := &volume_server_pb.BatchDeleteRequest{
 			FileIds: fileIds,
@@ -131,7 +131,7 @@ func DeleteFilesAtOneVolumeServer(volumeServer string, fileIds []string) (ret []
 	}
 
 	for _, result := range ret {
-		if result.Error != "" && result.Error != "Not Found" {
+		if result.Error != "" && result.Error != "not found" {
 			return nil, fmt.Errorf("delete fileId %s: %v", result.FileId, result.Error)
 		}
 	}

weed/operation/grpc_client.go

@@ -2,23 +2,16 @@ package operation
 
 import (
 	"fmt"
-	"strconv"
-	"strings"
-	"sync"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"strconv"
+	"strings"
 )
 
-var (
-	grpcClients     = make(map[string]*grpc.ClientConn)
-	grpcClientsLock sync.Mutex
-)
-
-func WithVolumeServerClient(volumeServer string, fn func(volume_server_pb.VolumeServerClient) error) error {
+func WithVolumeServerClient(volumeServer string, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error {
 
 	grpcAddress, err := toVolumeServerGrpcAddress(volumeServer)
 	if err != nil {
@@ -28,7 +21,7 @@ func WithVolumeServerClient(volumeServer string, fn func(volume_server_pb.Volume
 	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
 		client := volume_server_pb.NewVolumeServerClient(grpcConnection)
 		return fn(client)
-	}, grpcAddress)
+	}, grpcAddress, grpcDialOption)
 
 }
 
@@ -42,11 +35,16 @@ func toVolumeServerGrpcAddress(volumeServer string) (grpcAddress string, err err
 	return fmt.Sprintf("%s:%d", volumeServer[0:sepIndex], port+10000), nil
 }
 
-func withMasterServerClient(masterServer string, fn func(masterClient master_pb.SeaweedClient) error) error {
+func withMasterServerClient(masterServer string, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
+
+	masterGrpcAddress, parseErr := util.ParseServerToGrpcAddress(masterServer, 0)
+	if parseErr != nil {
+		return fmt.Errorf("failed to parse master grpc %v", masterServer)
+	}
 
 	return util.WithCachedGrpcClient(func(grpcConnection *grpc.ClientConn) error {
 		client := master_pb.NewSeaweedClient(grpcConnection)
 		return fn(client)
-	}, masterServer)
+	}, masterGrpcAddress, grpcDialOption)
 
 }

weed/operation/list_masters.go

@@ -1,32 +0,0 @@
-package operation
-
-import (
-	"encoding/json"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-type ClusterStatusResult struct {
-	IsLeader bool     `json:"IsLeader,omitempty"`
-	Leader   string   `json:"Leader,omitempty"`
-	Peers    []string `json:"Peers,omitempty"`
-}
-
-func ListMasters(server string) (leader string, peers []string, err error) {
-	jsonBlob, err := util.Get("http://" + server + "/cluster/status")
-	glog.V(2).Info("list masters result :", string(jsonBlob))
-	if err != nil {
-		return "", nil, err
-	}
-	var ret ClusterStatusResult
-	err = json.Unmarshal(jsonBlob, &ret)
-	if err != nil {
-		return "", nil, err
-	}
-	peers = ret.Peers
-	if ret.IsLeader {
-		peers = append(peers, ret.Leader)
-	}
-	return ret.Leader, peers, nil
-}

weed/operation/lookup.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"google.golang.org/grpc"
 	"math/rand"
 	"net/url"
 	"strings"
@@ -78,7 +79,7 @@ func LookupFileId(server string, fileId string) (fullUrl string, err error) {
 }
 
 // LookupVolumeIds find volume locations by cache and actual lookup
-func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, error) {
+func LookupVolumeIds(server string, grpcDialOption grpc.DialOption, vids []string) (map[string]LookupResult, error) {
 	ret := make(map[string]LookupResult)
 	var unknown_vids []string
 
@@ -98,7 +99,8 @@ func LookupVolumeIds(server string, vids []string) (map[string]LookupResult, err
 
 	//only query unknown_vids
 
-	err := withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
+	err := withMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+
 		req := &master_pb.LookupVolumeRequest{
 			VolumeIds: unknown_vids,
 		}

weed/operation/lookup_vid_cache.go

@@ -9,6 +9,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 )
 
+var ErrorNotFound = errors.New("not found")
+
 type VidInfo struct {
 	Locations       []Location
 	NextRefreshTime time.Time
@@ -28,14 +30,14 @@ func (vc *VidCache) Get(vid string) ([]Location, error) {
 	defer vc.RUnlock()
 	if 0 < id && id <= len(vc.cache) {
 		if vc.cache[id-1].Locations == nil {
-			return nil, errors.New("Not Set")
+			return nil, errors.New("not set")
 		}
 		if vc.cache[id-1].NextRefreshTime.Before(time.Now()) {
-			return nil, errors.New("Expired")
+			return nil, errors.New("expired")
 		}
 		return vc.cache[id-1].Locations, nil
 	}
-	return nil, errors.New("Not Found")
+	return nil, ErrorNotFound
 }
 func (vc *VidCache) Set(vid string, locations []Location, duration time.Duration) {
 	id, err := strconv.Atoi(vid)

weed/operation/stats.go

@@ -2,13 +2,15 @@ package operation
 
 import (
 	"context"
+	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 )
 
-func Statistics(server string, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
+func Statistics(server string, grpcDialOption grpc.DialOption, req *master_pb.StatisticsRequest) (resp *master_pb.StatisticsResponse, err error) {
+
+	err = withMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
-	err = withMasterServerClient(server, func(masterClient master_pb.SeaweedClient) error {
 		grpcResponse, grpcErr := masterClient.Statistics(context.Background(), req)
 		if grpcErr != nil {
 			return grpcErr

weed/operation/submit.go

@@ -2,6 +2,7 @@ package operation
 
 import (
 	"bytes"
+	"google.golang.org/grpc"
 	"io"
 	"mime"
 	"net/url"
@@ -18,7 +19,6 @@ type FilePart struct {
 	Reader      io.Reader
 	FileName    string
 	FileSize    int64
-	IsGzipped   bool
 	MimeType    string
 	ModTime     int64 //in seconds
 	Replication string
@@ -37,10 +37,8 @@ type SubmitResult struct {
 	Error    string `json:"error,omitempty"`
 }
 
-func SubmitFiles(master string, files []FilePart,
-	replication string, collection string, dataCenter string, ttl string, maxMB int,
-	secret security.Secret,
-) ([]SubmitResult, error) {
+func SubmitFiles(master string, grpcDialOption grpc.DialOption, files []FilePart,
+	replication string, collection string, dataCenter string, ttl string, maxMB int) ([]SubmitResult, error) {
 	results := make([]SubmitResult, len(files))
 	for index, file := range files {
 		results[index].FileName = file.FileName
@@ -52,7 +50,7 @@ func SubmitFiles(master string, files []FilePart,
 		DataCenter:  dataCenter,
 		Ttl:         ttl,
 	}
-	ret, err := Assign(master, ar)
+	ret, err := Assign(master, grpcDialOption, ar)
 	if err != nil {
 		for index, _ := range files {
 			results[index].Error = err.Error()
@@ -68,7 +66,7 @@ func SubmitFiles(master string, files []FilePart,
 		file.Replication = replication
 		file.Collection = collection
 		file.DataCenter = dataCenter
-		results[index].Size, err = file.Upload(maxMB, master, secret)
+		results[index].Size, err = file.Upload(maxMB, master, ret.Auth, grpcDialOption)
 		if err != nil {
 			results[index].Error = err.Error()
 		}
@@ -103,7 +101,6 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
 	ret.ModTime = fi.ModTime().UTC().Unix()
 	ret.FileSize = fi.Size()
 	ext := strings.ToLower(path.Ext(fullPathFilename))
-	ret.IsGzipped = ext == ".gz"
 	ret.FileName = fi.Name()
 	if ext != "" {
 		ret.MimeType = mime.TypeByExtension(ext)
@@ -112,8 +109,7 @@ func newFilePart(fullPathFilename string) (ret FilePart, err error) {
 	return ret, nil
 }
 
-func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (retSize uint32, err error) {
-	jwt := security.GenJwt(secret, fi.Fid)
+func (fi FilePart) Upload(maxMB int, master string, jwt security.EncodedJwt, grpcDialOption grpc.DialOption) (retSize uint32, err error) {
 	fileUrl := "http://" + fi.Server + "/" + fi.Fid
 	if fi.ModTime != 0 {
 		fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
@@ -141,7 +137,7 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 				Collection:  fi.Collection,
 				Ttl:         fi.Ttl,
 			}
-			ret, err = Assign(master, ar)
+			ret, err = Assign(master, grpcDialOption, ar)
 			if err != nil {
 				return
 			}
@@ -154,10 +150,10 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 					Collection:  fi.Collection,
 					Ttl:         fi.Ttl,
 				}
-				ret, err = Assign(master, ar)
+				ret, err = Assign(master, grpcDialOption, ar)
 				if err != nil {
 					// delete all uploaded chunks
-					cm.DeleteChunks(master)
+					cm.DeleteChunks(master, grpcDialOption)
 					return
 				}
 				id = ret.Fid
@@ -172,10 +168,10 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 				baseName+"-"+strconv.FormatInt(i+1, 10),
 				io.LimitReader(fi.Reader, chunkSize),
 				master, fileUrl,
-				jwt)
+				ret.Auth)
 			if e != nil {
 				// delete all uploaded chunks
-				cm.DeleteChunks(master)
+				cm.DeleteChunks(master, grpcDialOption)
 				return 0, e
 			}
 			cm.Chunks = append(cm.Chunks,
@@ -190,10 +186,10 @@ func (fi FilePart) Upload(maxMB int, master string, secret security.Secret) (ret
 		err = upload_chunked_file_manifest(fileUrl, &cm, jwt)
 		if err != nil {
 			// delete all uploaded chunks
-			cm.DeleteChunks(master)
+			cm.DeleteChunks(master, grpcDialOption)
 		}
 	} else {
-		ret, e := Upload(fileUrl, baseName, fi.Reader, fi.IsGzipped, fi.MimeType, nil, jwt)
+		ret, e := Upload(fileUrl, baseName, fi.Reader, false, fi.MimeType, nil, jwt)
 		if e != nil {
 			return 0, e
 		}

weed/operation/sync_volume.go

@@ -2,15 +2,18 @@ package operation
 
 import (
 	"context"
-
+	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	. "github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"io"
 )
 
-func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
+func GetVolumeSyncStatus(server string, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
+
+	WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
 
-	WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
 		resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
 			VolumdId: vid,
 		})
@@ -20,20 +23,33 @@ func GetVolumeSyncStatus(server string, vid uint32) (resp *volume_server_pb.Volu
 	return
 }
 
-func GetVolumeIdxEntries(server string, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
+func GetVolumeIdxEntries(server string, grpcDialOption grpc.DialOption, vid uint32, eachEntryFn func(key NeedleId, offset Offset, size uint32)) error {
 
-	return WithVolumeServerClient(server, func(client volume_server_pb.VolumeServerClient) error {
-		resp, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
+	return WithVolumeServerClient(server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		stream, err := client.VolumeSyncIndex(context.Background(), &volume_server_pb.VolumeSyncIndexRequest{
 			VolumdId: vid,
 		})
 		if err != nil {
 			return err
 		}
 
-		dataSize := len(resp.IndexFileContent)
+		var indexFileContent []byte
+
+		for {
+			resp, err := stream.Recv()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return fmt.Errorf("read index entries: %v", err)
+			}
+			indexFileContent = append(indexFileContent, resp.IndexFileContent...)
+		}
+
+		dataSize := len(indexFileContent)
 
 		for idx := 0; idx+NeedleEntrySize <= dataSize; idx += NeedleEntrySize {
-			line := resp.IndexFileContent[idx : idx+NeedleEntrySize]
+			line := indexFileContent[idx : idx+NeedleEntrySize]
 			key := BytesToNeedleId(line[:NeedleIdSize])
 			offset := BytesToOffset(line[NeedleIdSize : NeedleIdSize+OffsetSize])
 			size := util.BytesToUint32(line[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])

weed/operation/upload_content.go

@@ -22,7 +22,7 @@ type UploadResult struct {
 	Name  string `json:"name,omitempty"`
 	Size  uint32 `json:"size,omitempty"`
 	Error string `json:"error,omitempty"`
-	ETag  string `json:"error,omitempty"`
+	ETag  string `json:"eTag,omitempty"`
 }
 
 var (
@@ -58,9 +58,6 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	if isGzipped {
 		h.Set("Content-Encoding", "gzip")
 	}
-	if jwt != "" {
-		h.Set("Authorization", "BEARER "+string(jwt))
-	}
 
 	file_writer, cp_err := body_writer.CreatePart(h)
 	if cp_err != nil {
@@ -86,18 +83,15 @@ func upload_content(uploadUrl string, fillBufferFunction func(w io.Writer) error
 	for k, v := range pairMap {
 		req.Header.Set(k, v)
 	}
+	if jwt != "" {
+		req.Header.Set("Authorization", "BEARER "+string(jwt))
+	}
 	resp, post_err := client.Do(req)
 	if post_err != nil {
 		glog.V(0).Infoln("failing to upload to", uploadUrl, post_err.Error())
 		return nil, post_err
 	}
 	defer resp.Body.Close()
-
-	if resp.StatusCode < http.StatusOK ||
-		resp.StatusCode > http.StatusIMUsed {
-		return nil, errors.New(http.StatusText(resp.StatusCode))
-	}
-
 	etag := getEtag(resp)
 	resp_body, ra_err := ioutil.ReadAll(resp.Body)
 	if ra_err != nil {

weed/pb/filer.proto

@@ -97,6 +97,7 @@ message FuseAttributes {
     int32 ttl_sec = 10;
     string user_name = 11; // for hdfs
     repeated string group_name = 12; // for hdfs
+    string symlink_target = 13;
 }
 
 message CreateEntryRequest {
@@ -117,7 +118,7 @@ message UpdateEntryResponse {
 message DeleteEntryRequest {
     string directory = 1;
     string name = 2;
-    bool is_directory = 3;
+    // bool is_directory = 3;
     bool is_delete_data = 4;
     bool is_recursive = 5;
 }
@@ -138,6 +139,7 @@ message AssignVolumeResponse {
     string url = 2;
     string public_url = 3;
     int32 count = 4;
+    string auth = 5;
 }
 
 message LookupVolumeRequest {

weed/pb/filer_pb/filer.pb.go

@@ -297,18 +297,19 @@ func (m *FileChunk) GetSourceFileId() string {
 }
 
 type FuseAttributes struct {
-	FileSize    uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
-	Mtime       int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
-	FileMode    uint32   `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"`
-	Uid         uint32   `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"`
-	Gid         uint32   `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"`
-	Crtime      int64    `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"`
-	Mime        string   `protobuf:"bytes,7,opt,name=mime" json:"mime,omitempty"`
-	Replication string   `protobuf:"bytes,8,opt,name=replication" json:"replication,omitempty"`
-	Collection  string   `protobuf:"bytes,9,opt,name=collection" json:"collection,omitempty"`
-	TtlSec      int32    `protobuf:"varint,10,opt,name=ttl_sec,json=ttlSec" json:"ttl_sec,omitempty"`
-	UserName    string   `protobuf:"bytes,11,opt,name=user_name,json=userName" json:"user_name,omitempty"`
-	GroupName   []string `protobuf:"bytes,12,rep,name=group_name,json=groupName" json:"group_name,omitempty"`
+	FileSize      uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
+	Mtime         int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
+	FileMode      uint32   `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"`
+	Uid           uint32   `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"`
+	Gid           uint32   `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"`
+	Crtime        int64    `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"`
+	Mime          string   `protobuf:"bytes,7,opt,name=mime" json:"mime,omitempty"`
+	Replication   string   `protobuf:"bytes,8,opt,name=replication" json:"replication,omitempty"`
+	Collection    string   `protobuf:"bytes,9,opt,name=collection" json:"collection,omitempty"`
+	TtlSec        int32    `protobuf:"varint,10,opt,name=ttl_sec,json=ttlSec" json:"ttl_sec,omitempty"`
+	UserName      string   `protobuf:"bytes,11,opt,name=user_name,json=userName" json:"user_name,omitempty"`
+	GroupName     []string `protobuf:"bytes,12,rep,name=group_name,json=groupName" json:"group_name,omitempty"`
+	SymlinkTarget string   `protobuf:"bytes,13,opt,name=symlink_target,json=symlinkTarget" json:"symlink_target,omitempty"`
 }
 
 func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
@@ -400,6 +401,13 @@ func (m *FuseAttributes) GetGroupName() []string {
 	return nil
 }
 
+func (m *FuseAttributes) GetSymlinkTarget() string {
+	if m != nil {
+		return m.SymlinkTarget
+	}
+	return ""
+}
+
 type CreateEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
 	Entry     *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
@@ -465,11 +473,11 @@ func (*UpdateEntryResponse) ProtoMessage()               {}
 func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
 
 type DeleteEntryRequest struct {
-	Directory    string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
-	Name         string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
-	IsDirectory  bool   `protobuf:"varint,3,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"`
-	IsDeleteData bool   `protobuf:"varint,4,opt,name=is_delete_data,json=isDeleteData" json:"is_delete_data,omitempty"`
-	IsRecursive  bool   `protobuf:"varint,5,opt,name=is_recursive,json=isRecursive" json:"is_recursive,omitempty"`
+	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
+	Name      string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
+	// bool is_directory = 3;
+	IsDeleteData bool `protobuf:"varint,4,opt,name=is_delete_data,json=isDeleteData" json:"is_delete_data,omitempty"`
+	IsRecursive  bool `protobuf:"varint,5,opt,name=is_recursive,json=isRecursive" json:"is_recursive,omitempty"`
 }
 
 func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
@@ -491,13 +499,6 @@ func (m *DeleteEntryRequest) GetName() string {
 	return ""
 }
 
-func (m *DeleteEntryRequest) GetIsDirectory() bool {
-	if m != nil {
-		return m.IsDirectory
-	}
-	return false
-}
-
 func (m *DeleteEntryRequest) GetIsDeleteData() bool {
 	if m != nil {
 		return m.IsDeleteData
@@ -573,6 +574,7 @@ type AssignVolumeResponse struct {
 	Url       string `protobuf:"bytes,2,opt,name=url" json:"url,omitempty"`
 	PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"`
 	Count     int32  `protobuf:"varint,4,opt,name=count" json:"count,omitempty"`
+	Auth      string `protobuf:"bytes,5,opt,name=auth" json:"auth,omitempty"`
 }
 
 func (m *AssignVolumeResponse) Reset()                    { *m = AssignVolumeResponse{} }
@@ -608,6 +610,13 @@ func (m *AssignVolumeResponse) GetCount() int32 {
 	return 0
 }
 
+func (m *AssignVolumeResponse) GetAuth() string {
+	if m != nil {
+		return m.Auth
+	}
+	return ""
+}
+
 type LookupVolumeRequest struct {
 	VolumeIds []string `protobuf:"bytes,1,rep,name=volume_ids,json=volumeIds" json:"volume_ids,omitempty"`
 }
@@ -1158,85 +1167,87 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1272 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xcd, 0x6f, 0xdc, 0x44,
-	0x14, 0xaf, 0xf7, 0x2b, 0xeb, 0xb7, 0xbb, 0x21, 0x99, 0x0d, 0xd4, 0xda, 0x7c, 0xb0, 0x35, 0x2d,
-	0x4a, 0x45, 0xb4, 0xaa, 0x02, 0x87, 0x96, 0x0a, 0x89, 0x36, 0x1f, 0x52, 0xa4, 0xb4, 0x45, 0x4e,
-	0x83, 0x84, 0x38, 0x58, 0x8e, 0x3d, 0xbb, 0x8c, 0xe2, 0xb5, 0x8d, 0x67, 0x9c, 0xb4, 0xfc, 0x1d,
-	0x5c, 0x39, 0x70, 0xe2, 0xce, 0x1f, 0xc0, 0x85, 0x03, 0x7f, 0x13, 0x37, 0x34, 0x1f, 0xf6, 0x8e,
-	0xd7, 0x4e, 0x0a, 0x42, 0xbd, 0xcd, 0xfc, 0xde, 0x9b, 0xf7, 0x35, 0xbf, 0x79, 0xcf, 0x86, 0xde,
-	0x94, 0x84, 0x38, 0x9d, 0x24, 0x69, 0xcc, 0x62, 0xd4, 0x15, 0x1b, 0x37, 0xb9, 0xb0, 0x5f, 0xc1,
-	0xe6, 0x69, 0x1c, 0x5f, 0x66, 0xc9, 0x21, 0x49, 0xb1, 0xcf, 0xe2, 0xf4, 0xed, 0x51, 0xc4, 0xd2,
-	0xb7, 0x0e, 0xfe, 0x31, 0xc3, 0x94, 0xa1, 0x2d, 0x30, 0x83, 0x5c, 0x60, 0x19, 0x63, 0x63, 0xd7,
-	0x74, 0x16, 0x00, 0x42, 0xd0, 0x8a, 0xbc, 0x39, 0xb6, 0x1a, 0x42, 0x20, 0xd6, 0xf6, 0x11, 0x6c,
-	0xd5, 0x1b, 0xa4, 0x49, 0x1c, 0x51, 0x8c, 0x1e, 0x40, 0x1b, 0x73, 0x40, 0x58, 0xeb, 0xed, 0x7f,
-	0x30, 0xc9, 0x43, 0x99, 0x48, 0x3d, 0x29, 0xb5, 0xff, 0x30, 0x00, 0x9d, 0x12, 0xca, 0x38, 0x48,
-	0x30, 0xfd, 0x77, 0xf1, 0x7c, 0x04, 0x9d, 0x24, 0xc5, 0x53, 0xf2, 0x46, 0x45, 0xa4, 0x76, 0x68,
-	0x0f, 0xd6, 0x29, 0xf3, 0x52, 0x76, 0x9c, 0xc6, 0xf3, 0x63, 0x12, 0xe2, 0x97, 0x3c, 0xe8, 0xa6,
-	0x50, 0xa9, 0x0a, 0xd0, 0x04, 0x10, 0x89, 0xfc, 0x30, 0xa3, 0xe4, 0x0a, 0x9f, 0xe5, 0x52, 0xab,
-	0x35, 0x36, 0x76, 0xbb, 0x4e, 0x8d, 0x04, 0x6d, 0x40, 0x3b, 0x24, 0x73, 0xc2, 0xac, 0xf6, 0xd8,
-	0xd8, 0x1d, 0x38, 0x72, 0x63, 0x7f, 0x0d, 0xc3, 0x52, 0xfc, 0x2a, 0xfd, 0x87, 0xb0, 0x82, 0x25,
-	0x64, 0x19, 0xe3, 0x66, 0x5d, 0x01, 0x72, 0xb9, 0xfd, 0x4b, 0x03, 0xda, 0x02, 0x2a, 0xea, 0x6c,
-	0x2c, 0xea, 0x8c, 0xee, 0x41, 0x9f, 0x50, 0x77, 0x51, 0x8c, 0x86, 0x88, 0xaf, 0x47, 0x68, 0x51,
-	0x77, 0xf4, 0x19, 0x74, 0xfc, 0x1f, 0xb2, 0xe8, 0x92, 0x5a, 0x4d, 0xe1, 0x6a, 0xb8, 0x70, 0xc5,
-	0x93, 0x3d, 0xe0, 0x32, 0x47, 0xa9, 0xa0, 0xc7, 0x00, 0x1e, 0x63, 0x29, 0xb9, 0xc8, 0x18, 0xa6,
-	0x22, 0xdb, 0xde, 0xbe, 0xa5, 0x1d, 0xc8, 0x28, 0x7e, 0x56, 0xc8, 0x1d, 0x4d, 0x17, 0x3d, 0x81,
-	0x2e, 0x7e, 0xc3, 0x70, 0x14, 0xe0, 0xc0, 0x6a, 0x0b, 0x47, 0xdb, 0x4b, 0x39, 0x4d, 0x8e, 0x94,
-	0x5c, 0x66, 0x58, 0xa8, 0x8f, 0x9e, 0xc2, 0xa0, 0x24, 0x42, 0x6b, 0xd0, 0xbc, 0xc4, 0xf9, 0xcd,
-	0xf2, 0x25, 0xaf, 0xee, 0x95, 0x17, 0x66, 0x92, 0x64, 0x7d, 0x47, 0x6e, 0xbe, 0x6c, 0x3c, 0x36,
-	0xec, 0x9f, 0x0d, 0x58, 0x3f, 0xba, 0xc2, 0x11, 0x7b, 0x19, 0x33, 0x32, 0x25, 0xbe, 0xc7, 0x48,
-	0x1c, 0xa1, 0x3d, 0x30, 0xe3, 0x30, 0x70, 0x6f, 0xe5, 0x58, 0x37, 0x0e, 0x95, 0xbf, 0x3d, 0x30,
-	0x23, 0x7c, 0xad, 0xb4, 0x1b, 0x37, 0x68, 0x47, 0xf8, 0x5a, 0x6a, 0x7f, 0x02, 0x83, 0x00, 0x87,
-	0x98, 0x61, 0xb7, 0xa8, 0x2b, 0x2f, 0x7a, 0x5f, 0x82, 0xa2, 0x9e, 0xd4, 0xfe, 0xd5, 0x00, 0xb3,
-	0x28, 0x2f, 0xba, 0x0b, 0x2b, 0xdc, 0x9c, 0x4b, 0x02, 0x95, 0x54, 0x87, 0x6f, 0x4f, 0x02, 0xce,
-	0xd5, 0x78, 0x3a, 0xa5, 0x98, 0x09, 0xb7, 0x4d, 0x47, 0xed, 0xf8, 0x5d, 0x53, 0xf2, 0x93, 0xa4,
-	0x67, 0xcb, 0x11, 0x6b, 0x5e, 0x83, 0x39, 0x23, 0x73, 0x2c, 0xae, 0xa5, 0xe9, 0xc8, 0x0d, 0x1a,
-	0x42, 0x1b, 0xbb, 0xcc, 0x9b, 0x09, 0xde, 0x99, 0x4e, 0x0b, 0xbf, 0xf6, 0x66, 0xe8, 0x3e, 0xac,
-	0xd2, 0x38, 0x4b, 0x7d, 0xec, 0xe6, 0x6e, 0x3b, 0x42, 0xda, 0x97, 0xe8, 0xb1, 0x70, 0x6e, 0xff,
-	0xd5, 0x80, 0xd5, 0xf2, 0x8d, 0xa2, 0x4d, 0x30, 0xc5, 0x09, 0xe1, 0xdc, 0x10, 0xce, 0x45, 0x97,
-	0x38, 0x2b, 0x05, 0xd0, 0xd0, 0x03, 0xc8, 0x8f, 0xcc, 0xe3, 0x40, 0xc6, 0x3b, 0x90, 0x47, 0x5e,
-	0xc4, 0x01, 0xe6, 0x37, 0x99, 0x91, 0x40, 0x44, 0x3c, 0x70, 0xf8, 0x92, 0x23, 0x33, 0x12, 0xa8,
-	0x57, 0xc2, 0x97, 0xbc, 0x06, 0x7e, 0x2a, 0xec, 0x76, 0x64, 0x0d, 0xe4, 0x8e, 0xd7, 0x60, 0xce,
-	0xd1, 0x15, 0x99, 0x18, 0x5f, 0xa3, 0x31, 0xf4, 0x52, 0x9c, 0x84, 0xea, 0x9a, 0xad, 0xae, 0x10,
-	0xe9, 0x10, 0xda, 0x01, 0xf0, 0xe3, 0x30, 0xc4, 0xbe, 0x50, 0x30, 0x85, 0x82, 0x86, 0xf0, 0xab,
-	0x60, 0x2c, 0x74, 0x29, 0xf6, 0x2d, 0x18, 0x1b, 0xbb, 0x6d, 0xa7, 0xc3, 0x58, 0x78, 0x86, 0x7d,
-	0x9e, 0x47, 0x46, 0x71, 0xea, 0x8a, 0x37, 0xd6, 0x13, 0xe7, 0xba, 0x1c, 0x10, 0xdd, 0x60, 0x1b,
-	0x60, 0x96, 0xc6, 0x59, 0x22, 0xa5, 0xfd, 0x71, 0x93, 0xb7, 0x1c, 0x81, 0x70, 0xb1, 0xfd, 0x1d,
-	0xa0, 0x83, 0x14, 0x7b, 0x0c, 0xff, 0x87, 0xb6, 0x59, 0xb4, 0xc0, 0xc6, 0xad, 0x2d, 0xf0, 0x43,
-	0x18, 0x96, 0x4c, 0xcb, 0x0e, 0xc2, 0x3d, 0x9e, 0x27, 0xc1, 0xfb, 0xf2, 0x58, 0x32, 0xad, 0x3c,
-	0xfe, 0x6e, 0x00, 0x3a, 0x14, 0x14, 0xff, 0x7f, 0xb3, 0xa1, 0xd2, 0xb3, 0x9a, 0xd5, 0x9e, 0x75,
-	0x1f, 0x56, 0xb9, 0x8a, 0x7c, 0x65, 0x81, 0xc7, 0x3c, 0xd5, 0x78, 0xfb, 0x84, 0xca, 0x10, 0x0e,
-	0x3d, 0xe6, 0x29, 0x43, 0x29, 0xf6, 0xb3, 0x94, 0xf7, 0x62, 0xc1, 0x29, 0x61, 0xc8, 0xc9, 0x21,
-	0x9e, 0x4b, 0x29, 0x66, 0x95, 0xcb, 0x6f, 0x06, 0x0c, 0x9f, 0x51, 0x4a, 0x66, 0xd1, 0xb7, 0x71,
-	0x98, 0xcd, 0x71, 0x9e, 0xcc, 0x06, 0xb4, 0xfd, 0x38, 0x8b, 0x98, 0x48, 0xa4, 0xed, 0xc8, 0xcd,
-	0x12, 0xa5, 0x1a, 0x15, 0x4a, 0x2d, 0x91, 0xb2, 0x59, 0x25, 0xa5, 0x46, 0xba, 0x56, 0x89, 0x74,
-	0x1f, 0x43, 0x8f, 0xa7, 0xe7, 0xfa, 0x38, 0x62, 0x38, 0x55, 0x6f, 0x18, 0x38, 0x74, 0x20, 0x10,
-	0xfb, 0x0a, 0x36, 0xca, 0x81, 0xaa, 0x09, 0x72, 0x63, 0x47, 0xe1, 0x2f, 0x2e, 0x0d, 0x55, 0x94,
-	0x7c, 0xc9, 0xb9, 0x9b, 0x64, 0x17, 0x21, 0xf1, 0x5d, 0x2e, 0x90, 0xd1, 0x99, 0x12, 0x39, 0x4f,
-	0xc3, 0x45, 0xce, 0x2d, 0x2d, 0x67, 0xfb, 0x0b, 0x18, 0xca, 0x01, 0x5e, 0x2e, 0xd0, 0x36, 0xc0,
-	0x95, 0x00, 0x5c, 0x12, 0xc8, 0xd9, 0x65, 0x3a, 0xa6, 0x44, 0x4e, 0x02, 0x6a, 0x7f, 0x05, 0xe6,
-	0x69, 0x2c, 0x73, 0xa6, 0xe8, 0x11, 0x98, 0x61, 0xbe, 0x51, 0x63, 0x0e, 0x2d, 0x28, 0x97, 0xeb,
-	0x39, 0x0b, 0x25, 0xfb, 0x29, 0x74, 0x73, 0x38, 0xcf, 0xc3, 0xb8, 0x29, 0x8f, 0xc6, 0x52, 0x1e,
-	0xf6, 0x9f, 0x06, 0x6c, 0x94, 0x43, 0x56, 0xa5, 0x3a, 0x87, 0x41, 0xe1, 0xc2, 0x9d, 0x7b, 0x89,
-	0x8a, 0xe5, 0x91, 0x1e, 0x4b, 0xf5, 0x58, 0x11, 0x20, 0x7d, 0xe1, 0x25, 0x92, 0x3d, 0xfd, 0x50,
-	0x83, 0x46, 0xaf, 0x61, 0xbd, 0xa2, 0x52, 0x33, 0xb9, 0x1e, 0xea, 0x93, 0xab, 0x34, 0x7d, 0x8b,
-	0xd3, 0xfa, 0x38, 0x7b, 0x02, 0x77, 0x25, 0x61, 0x0f, 0x0a, 0x7e, 0xe5, 0xb5, 0x2f, 0xd3, 0xd0,
-	0x58, 0xa6, 0xa1, 0x3d, 0x02, 0xab, 0x7a, 0x54, 0x11, 0x7e, 0x06, 0xeb, 0x67, 0xcc, 0x63, 0x84,
-	0x32, 0xe2, 0x17, 0x9f, 0x51, 0x4b, 0xbc, 0x35, 0xde, 0xd5, 0x4c, 0xab, 0xcc, 0x5f, 0x83, 0x26,
-	0x63, 0x39, 0xa7, 0xf8, 0x92, 0xdf, 0x02, 0xd2, 0x3d, 0xa9, 0x3b, 0x78, 0x0f, 0xae, 0x38, 0x1f,
-	0x58, 0xcc, 0xbc, 0x50, 0x0e, 0xab, 0x96, 0x18, 0x56, 0xa6, 0x40, 0xc4, 0xb4, 0x92, 0xfd, 0x3c,
-	0x90, 0xd2, 0xb6, 0x1c, 0x65, 0x1c, 0x10, 0xc2, 0x6d, 0x00, 0xf1, 0x7c, 0x24, 0xf3, 0x3b, 0xf2,
-	0x2c, 0x47, 0x0e, 0x38, 0xb0, 0xff, 0x77, 0x1b, 0xfa, 0x67, 0xd8, 0xbb, 0xc6, 0x38, 0xe0, 0xb3,
-	0x32, 0x45, 0xb3, 0x9c, 0x5b, 0xe5, 0xef, 0x59, 0xf4, 0x60, 0x99, 0x44, 0xb5, 0x1f, 0xd0, 0xa3,
-	0x4f, 0xdf, 0xa5, 0xa6, 0xae, 0xe9, 0x0e, 0x3a, 0x85, 0x9e, 0xf6, 0xc1, 0x88, 0xb6, 0xb4, 0x83,
-	0x95, 0xef, 0xe0, 0xd1, 0xf6, 0x0d, 0x52, 0xdd, 0x9a, 0x36, 0x3c, 0x74, 0x6b, 0xd5, 0x71, 0xa5,
-	0x5b, 0xab, 0x9b, 0x38, 0xc2, 0x9a, 0x36, 0x18, 0x74, 0x6b, 0xd5, 0x51, 0xa4, 0x5b, 0xab, 0x9b,
-	0x26, 0xc2, 0x9a, 0xd6, 0x9a, 0x75, 0x6b, 0xd5, 0x29, 0xa3, 0x5b, 0xab, 0xeb, 0xe7, 0x77, 0xd0,
-	0x2b, 0xe8, 0xeb, 0x7d, 0x12, 0x69, 0x07, 0x6a, 0x1a, 0xfd, 0x68, 0xe7, 0x26, 0xb1, 0x6e, 0x50,
-	0x6f, 0x0b, 0xba, 0xc1, 0x9a, 0xc6, 0xa8, 0x1b, 0xac, 0xeb, 0x26, 0xf6, 0x1d, 0xf4, 0x3d, 0xac,
-	0x2d, 0x3f, 0x4f, 0x74, 0x6f, 0x39, 0xad, 0xca, 0xab, 0x1f, 0xd9, 0xb7, 0xa9, 0x14, 0xc6, 0x4f,
-	0x00, 0x16, 0xaf, 0x0e, 0x6d, 0x2e, 0xce, 0x54, 0x5e, 0xfd, 0x68, 0xab, 0x5e, 0x98, 0x9b, 0x7a,
-	0xbe, 0x03, 0x6b, 0x54, 0x52, 0x7f, 0x4a, 0x27, 0x7e, 0x48, 0x70, 0xc4, 0x9e, 0x83, 0x78, 0x05,
-	0xdf, 0xf0, 0xbf, 0xc6, 0x8b, 0x8e, 0xf8, 0x79, 0xfc, 0xfc, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff,
-	0x58, 0x55, 0x30, 0x36, 0x4b, 0x0e, 0x00, 0x00,
+	// 1301 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0x4d, 0x8f, 0xd3, 0xc6,
+	0x1b, 0xc7, 0x79, 0x23, 0x7e, 0x92, 0xf0, 0xdf, 0x9d, 0xec, 0xbf, 0x58, 0x61, 0x97, 0x06, 0xb7,
+	0x54, 0x8b, 0x8a, 0x22, 0x44, 0x7b, 0x80, 0xa2, 0x4a, 0x85, 0x65, 0x91, 0x90, 0x16, 0xa8, 0xbc,
+	0x50, 0xa9, 0xea, 0xc1, 0xf2, 0xda, 0x93, 0x30, 0x5a, 0xc7, 0x4e, 0x3d, 0xe3, 0x05, 0xfa, 0x11,
+	0x7a, 0xe9, 0xa5, 0xd7, 0x1e, 0x7a, 0xea, 0xb7, 0xe8, 0xa5, 0xdf, 0xa7, 0xf7, 0xde, 0xaa, 0x79,
+	0x66, 0xec, 0x8c, 0x63, 0x2f, 0xb4, 0xaa, 0xb8, 0xcd, 0xfc, 0x9e, 0xf7, 0x67, 0x9e, 0x17, 0x1b,
+	0x06, 0x73, 0x16, 0xd3, 0x6c, 0xb6, 0xca, 0x52, 0x91, 0x92, 0x3e, 0x5e, 0xfc, 0xd5, 0x89, 0xfb,
+	0x0c, 0xae, 0x1c, 0xa5, 0xe9, 0x69, 0xbe, 0x7a, 0xc8, 0x32, 0x1a, 0x8a, 0x34, 0x7b, 0x73, 0x98,
+	0x88, 0xec, 0x8d, 0x47, 0xbf, 0xcf, 0x29, 0x17, 0x64, 0x17, 0xec, 0xa8, 0x20, 0x38, 0xd6, 0xd4,
+	0xda, 0xb7, 0xbd, 0x35, 0x40, 0x08, 0x74, 0x92, 0x60, 0x49, 0x9d, 0x16, 0x12, 0xf0, 0xec, 0x1e,
+	0xc2, 0x6e, 0xb3, 0x42, 0xbe, 0x4a, 0x13, 0x4e, 0xc9, 0x75, 0xe8, 0x52, 0x09, 0xa0, 0xb6, 0xc1,
+	0xed, 0xff, 0xcd, 0x0a, 0x57, 0x66, 0x8a, 0x4f, 0x51, 0xdd, 0xdf, 0x2d, 0x20, 0x47, 0x8c, 0x0b,
+	0x09, 0x32, 0xca, 0xff, 0x99, 0x3f, 0x1f, 0x40, 0x6f, 0x95, 0xd1, 0x39, 0x7b, 0xad, 0x3d, 0xd2,
+	0x37, 0x72, 0x13, 0xb6, 0xb9, 0x08, 0x32, 0xf1, 0x28, 0x4b, 0x97, 0x8f, 0x58, 0x4c, 0x9f, 0x4a,
+	0xa7, 0xdb, 0xc8, 0x52, 0x27, 0x90, 0x19, 0x10, 0x96, 0x84, 0x71, 0xce, 0xd9, 0x19, 0x3d, 0x2e,
+	0xa8, 0x4e, 0x67, 0x6a, 0xed, 0xf7, 0xbd, 0x06, 0x0a, 0xd9, 0x81, 0x6e, 0xcc, 0x96, 0x4c, 0x38,
+	0xdd, 0xa9, 0xb5, 0x3f, 0xf2, 0xd4, 0xc5, 0xfd, 0x0a, 0xc6, 0x15, 0xff, 0x75, 0xf8, 0x37, 0xe0,
+	0x22, 0x55, 0x90, 0x63, 0x4d, 0xdb, 0x4d, 0x09, 0x28, 0xe8, 0xee, 0x2f, 0x2d, 0xe8, 0x22, 0x54,
+	0xe6, 0xd9, 0x5a, 0xe7, 0x99, 0x5c, 0x83, 0x21, 0xe3, 0xfe, 0x3a, 0x19, 0x2d, 0xf4, 0x6f, 0xc0,
+	0x78, 0x99, 0x77, 0xf2, 0x29, 0xf4, 0xc2, 0x97, 0x79, 0x72, 0xca, 0x9d, 0x36, 0x9a, 0x1a, 0xaf,
+	0x4d, 0xc9, 0x60, 0x0f, 0x24, 0xcd, 0xd3, 0x2c, 0xe4, 0x0e, 0x40, 0x20, 0x44, 0xc6, 0x4e, 0x72,
+	0x41, 0x39, 0x46, 0x3b, 0xb8, 0xed, 0x18, 0x02, 0x39, 0xa7, 0xf7, 0x4b, 0xba, 0x67, 0xf0, 0x92,
+	0xbb, 0xd0, 0xa7, 0xaf, 0x05, 0x4d, 0x22, 0x1a, 0x39, 0x5d, 0x34, 0xb4, 0xb7, 0x11, 0xd3, 0xec,
+	0x50, 0xd3, 0x55, 0x84, 0x25, 0xfb, 0xe4, 0x1e, 0x8c, 0x2a, 0x24, 0xb2, 0x05, 0xed, 0x53, 0x5a,
+	0xbc, 0xac, 0x3c, 0xca, 0xec, 0x9e, 0x05, 0x71, 0xae, 0x8a, 0x6c, 0xe8, 0xa9, 0xcb, 0x17, 0xad,
+	0x3b, 0x96, 0xfb, 0xb3, 0x05, 0xdb, 0x87, 0x67, 0x34, 0x11, 0x4f, 0x53, 0xc1, 0xe6, 0x2c, 0x0c,
+	0x04, 0x4b, 0x13, 0x72, 0x13, 0xec, 0x34, 0x8e, 0xfc, 0xb7, 0xd6, 0x58, 0x3f, 0x8d, 0xb5, 0xbd,
+	0x9b, 0x60, 0x27, 0xf4, 0x95, 0xe6, 0x6e, 0x9d, 0xc3, 0x9d, 0xd0, 0x57, 0x8a, 0xfb, 0x23, 0x18,
+	0x45, 0x34, 0xa6, 0x82, 0xfa, 0x65, 0x5e, 0x65, 0xd2, 0x87, 0x0a, 0xc4, 0x7c, 0x72, 0xf7, 0x57,
+	0x0b, 0xec, 0x32, 0xbd, 0xe4, 0x32, 0x5c, 0x94, 0xea, 0x7c, 0x16, 0xe9, 0xa0, 0x7a, 0xf2, 0xfa,
+	0x38, 0x92, 0xb5, 0x9a, 0xce, 0xe7, 0x9c, 0x0a, 0x34, 0xdb, 0xf6, 0xf4, 0x4d, 0xbe, 0x35, 0x67,
+	0x3f, 0xa8, 0xf2, 0xec, 0x78, 0x78, 0x96, 0x39, 0x58, 0x0a, 0xb6, 0xa4, 0xf8, 0x2c, 0x6d, 0x4f,
+	0x5d, 0xc8, 0x18, 0xba, 0xd4, 0x17, 0xc1, 0x02, 0xeb, 0xce, 0xf6, 0x3a, 0xf4, 0x79, 0xb0, 0x20,
+	0x1f, 0xc3, 0x25, 0x9e, 0xe6, 0x59, 0x48, 0xfd, 0xc2, 0x6c, 0x0f, 0xa9, 0x43, 0x85, 0x3e, 0x42,
+	0xe3, 0xee, 0x9f, 0x2d, 0xb8, 0x54, 0x7d, 0x51, 0x72, 0x05, 0x6c, 0x94, 0x40, 0xe3, 0x16, 0x1a,
+	0xc7, 0x29, 0x71, 0x5c, 0x71, 0xa0, 0x65, 0x3a, 0x50, 0x88, 0x2c, 0xd3, 0x48, 0xf9, 0x3b, 0x52,
+	0x22, 0x4f, 0xd2, 0x88, 0xca, 0x97, 0xcc, 0x59, 0x84, 0x1e, 0x8f, 0x3c, 0x79, 0x94, 0xc8, 0x82,
+	0x45, 0xba, 0x4b, 0xe4, 0x51, 0xe6, 0x20, 0xcc, 0x50, 0x6f, 0x4f, 0xe5, 0x40, 0xdd, 0x64, 0x0e,
+	0x96, 0x12, 0xbd, 0xa8, 0x02, 0x93, 0x67, 0x32, 0x85, 0x41, 0x46, 0x57, 0xb1, 0x7e, 0x66, 0xa7,
+	0x8f, 0x24, 0x13, 0x22, 0x57, 0x01, 0xc2, 0x34, 0x8e, 0x69, 0x88, 0x0c, 0x36, 0x32, 0x18, 0x88,
+	0x7c, 0x0a, 0x21, 0x62, 0x9f, 0xd3, 0xd0, 0x81, 0xa9, 0xb5, 0xdf, 0xf5, 0x7a, 0x42, 0xc4, 0xc7,
+	0x34, 0x94, 0x71, 0xe4, 0x9c, 0x66, 0x3e, 0xf6, 0xd8, 0x00, 0xe5, 0xfa, 0x12, 0xc0, 0x69, 0xb0,
+	0x07, 0xb0, 0xc8, 0xd2, 0x7c, 0xa5, 0xa8, 0xc3, 0x69, 0x5b, 0x8e, 0x1c, 0x44, 0x90, 0x7c, 0x1d,
+	0x2e, 0xf1, 0x37, 0xcb, 0x98, 0x25, 0xa7, 0xbe, 0x08, 0xb2, 0x05, 0x15, 0xce, 0x08, 0x15, 0x8c,
+	0x34, 0xfa, 0x1c, 0x41, 0xf7, 0x5b, 0x20, 0x07, 0x19, 0x0d, 0x04, 0xfd, 0x17, 0xd3, 0xb5, 0x9c,
+	0x94, 0xad, 0xb7, 0x4e, 0xca, 0xff, 0xc3, 0xb8, 0xa2, 0x5a, 0x0d, 0x1a, 0x69, 0xf1, 0xc5, 0x2a,
+	0x7a, 0x5f, 0x16, 0x2b, 0xaa, 0xb5, 0xc5, 0x9f, 0x2c, 0x20, 0x0f, 0xb1, 0x13, 0xfe, 0xdb, 0x0a,
+	0x91, 0x35, 0x2c, 0x47, 0x9b, 0xea, 0xb4, 0x28, 0x10, 0x81, 0x1e, 0xbe, 0x43, 0xc6, 0x95, 0xfe,
+	0x87, 0x81, 0x08, 0xf4, 0x00, 0xcc, 0x68, 0x98, 0x67, 0x72, 0x1e, 0x63, 0x5d, 0xe1, 0x00, 0xf4,
+	0x0a, 0x48, 0x3a, 0x5a, 0x71, 0x48, 0x3b, 0xfa, 0x9b, 0x05, 0xe3, 0xfb, 0x9c, 0xb3, 0x45, 0xf2,
+	0x4d, 0x1a, 0xe7, 0x4b, 0x5a, 0x78, 0xba, 0x03, 0xdd, 0x30, 0xcd, 0x13, 0x81, 0x5e, 0x76, 0x3d,
+	0x75, 0xd9, 0x28, 0xab, 0x56, 0xad, 0xac, 0x36, 0x0a, 0xb3, 0x5d, 0x2f, 0x4c, 0xa3, 0xf0, 0x3a,
+	0x95, 0xc2, 0xfb, 0x10, 0x06, 0x32, 0x3c, 0x3f, 0xa4, 0x89, 0xa0, 0x99, 0xee, 0x63, 0x90, 0xd0,
+	0x01, 0x22, 0xee, 0x8f, 0x16, 0xec, 0x54, 0x3d, 0xd5, 0x6b, 0xe4, 0xdc, 0xb1, 0x22, 0xdb, 0x2e,
+	0x8b, 0xb5, 0x9b, 0xf2, 0x28, 0x0b, 0x78, 0x95, 0x9f, 0xc4, 0x2c, 0xf4, 0x25, 0x41, 0xb9, 0x67,
+	0x2b, 0xe4, 0x45, 0x16, 0xaf, 0x83, 0xee, 0x98, 0x41, 0x13, 0xe8, 0x04, 0xb9, 0x78, 0x59, 0x8c,
+	0x16, 0x79, 0x76, 0x3f, 0x87, 0xb1, 0xda, 0xec, 0xd5, 0xac, 0xed, 0x01, 0x9c, 0x21, 0xe0, 0xb3,
+	0x48, 0x2d, 0x35, 0xdb, 0xb3, 0x15, 0xf2, 0x38, 0xe2, 0xee, 0x97, 0x60, 0x1f, 0xa5, 0x2a, 0x11,
+	0x9c, 0xdc, 0x02, 0x3b, 0x2e, 0x2e, 0x7a, 0xff, 0x91, 0x75, 0x91, 0x15, 0x7c, 0xde, 0x9a, 0xc9,
+	0xbd, 0x07, 0xfd, 0x02, 0x2e, 0x62, 0xb3, 0xce, 0x8b, 0xad, 0xb5, 0x11, 0x9b, 0xfb, 0x87, 0x05,
+	0x3b, 0x55, 0x97, 0x75, 0xfa, 0x5e, 0xc0, 0xa8, 0x34, 0xe1, 0x2f, 0x83, 0x95, 0xf6, 0xe5, 0x96,
+	0xe9, 0x4b, 0x5d, 0xac, 0x74, 0x90, 0x3f, 0x09, 0x56, 0xaa, 0xa4, 0x86, 0xb1, 0x01, 0x4d, 0x9e,
+	0xc3, 0x76, 0x8d, 0xa5, 0x61, 0xa5, 0xdd, 0x30, 0x57, 0x5a, 0x65, 0x2d, 0x97, 0xd2, 0xe6, 0x9e,
+	0xbb, 0x0b, 0x97, 0x55, 0x15, 0x1f, 0x94, 0x45, 0x57, 0xe4, 0xbe, 0x5a, 0x9b, 0xd6, 0x66, 0x6d,
+	0xba, 0x13, 0x70, 0xea, 0xa2, 0xba, 0x0b, 0x16, 0xb0, 0x7d, 0x2c, 0x02, 0xc1, 0xb8, 0x60, 0x61,
+	0xf9, 0x7d, 0xb5, 0x51, 0xcc, 0xd6, 0xbb, 0xa6, 0x6c, 0xbd, 0x1d, 0xb6, 0xa0, 0x2d, 0x44, 0x51,
+	0x67, 0xf2, 0x28, 0x5f, 0x81, 0x98, 0x96, 0xf4, 0x1b, 0xbc, 0x07, 0x53, 0xb2, 0x1e, 0x44, 0x2a,
+	0x82, 0x58, 0x6d, 0xb1, 0x0e, 0x6e, 0x31, 0x1b, 0x11, 0x5c, 0x63, 0x6a, 0xd0, 0x47, 0x8a, 0xda,
+	0x55, 0x3b, 0x4e, 0x02, 0x48, 0xdc, 0x03, 0xc0, 0x96, 0x52, 0xdd, 0xd0, 0x53, 0xb2, 0x12, 0x39,
+	0x90, 0xc0, 0xed, 0xbf, 0xba, 0x30, 0x3c, 0xa6, 0xc1, 0x2b, 0x4a, 0x23, 0xb9, 0x44, 0x33, 0xb2,
+	0x28, 0x6a, 0xab, 0xfa, 0xa1, 0x4b, 0xae, 0x6f, 0x16, 0x51, 0xe3, 0x97, 0xf5, 0xe4, 0x93, 0x77,
+	0xb1, 0xe9, 0x67, 0xba, 0x40, 0x8e, 0x60, 0x60, 0x7c, 0x49, 0x92, 0x5d, 0x43, 0xb0, 0xf6, 0x81,
+	0x3c, 0xd9, 0x3b, 0x87, 0x6a, 0x6a, 0x33, 0xd6, 0x85, 0xa9, 0xad, 0xbe, 0xa0, 0x4c, 0x6d, 0x4d,
+	0x3b, 0x06, 0xb5, 0x19, 0xab, 0xc0, 0xd4, 0x56, 0x5f, 0x3e, 0xa6, 0xb6, 0xa6, 0xfd, 0x81, 0xda,
+	0x8c, 0x79, 0x6d, 0x6a, 0xab, 0xef, 0x15, 0x53, 0x5b, 0xd3, 0x90, 0xbf, 0x40, 0x9e, 0xc1, 0xd0,
+	0x9c, 0x9d, 0xc4, 0x10, 0x68, 0x98, 0xfe, 0x93, 0xab, 0xe7, 0x91, 0x4d, 0x85, 0xe6, 0x58, 0x30,
+	0x15, 0x36, 0x0c, 0x46, 0x53, 0x61, 0xd3, 0x34, 0x71, 0x2f, 0x90, 0xef, 0x60, 0x6b, 0xb3, 0x3d,
+	0xc9, 0xb5, 0xcd, 0xb0, 0x6a, 0x5d, 0x3f, 0x71, 0xdf, 0xc6, 0x52, 0x2a, 0x7f, 0x0c, 0xb0, 0xee,
+	0x3a, 0x72, 0x65, 0x2d, 0x53, 0xeb, 0xfa, 0xc9, 0x6e, 0x33, 0xb1, 0x50, 0xf5, 0xe0, 0x2a, 0x6c,
+	0x71, 0x55, 0xfa, 0x73, 0x3e, 0x0b, 0x63, 0x46, 0x13, 0xf1, 0x00, 0xb0, 0x0b, 0xbe, 0x96, 0xbf,
+	0x93, 0x27, 0x3d, 0xfc, 0xab, 0xfc, 0xec, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8c, 0x6d, 0xf7,
+	0x42, 0x64, 0x0e, 0x00, 0x00,
 }

weed/pb/master.proto

@@ -36,7 +36,6 @@ message Heartbeat {
 
 message HeartbeatResponse {
     uint64 volumeSizeLimit = 1;
-    string secretKey = 2;
     string leader = 3;
 }
 
@@ -109,6 +108,7 @@ message AssignResponse {
     string public_url = 3;
     uint64 count = 4;
     string error = 5;
+    string auth = 6;
 }
 
 message StatisticsRequest {

weed/pb/master_pb/master.pb.go

@@ -145,7 +145,6 @@ func (m *Heartbeat) GetDeletedVids() []uint32 {
 
 type HeartbeatResponse struct {
 	VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"`
-	SecretKey       string `protobuf:"bytes,2,opt,name=secretKey" json:"secretKey,omitempty"`
 	Leader          string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"`
 }
 
@@ -161,13 +160,6 @@ func (m *HeartbeatResponse) GetVolumeSizeLimit() uint64 {
 	return 0
 }
 
-func (m *HeartbeatResponse) GetSecretKey() string {
-	if m != nil {
-		return m.SecretKey
-	}
-	return ""
-}
-
 func (m *HeartbeatResponse) GetLeader() string {
 	if m != nil {
 		return m.Leader
@@ -545,6 +537,7 @@ type AssignResponse struct {
 	PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"`
 	Count     uint64 `protobuf:"varint,4,opt,name=count" json:"count,omitempty"`
 	Error     string `protobuf:"bytes,5,opt,name=error" json:"error,omitempty"`
+	Auth      string `protobuf:"bytes,6,opt,name=auth" json:"auth,omitempty"`
 }
 
 func (m *AssignResponse) Reset()                    { *m = AssignResponse{} }
@@ -587,6 +580,13 @@ func (m *AssignResponse) GetError() string {
 	return ""
 }
 
+func (m *AssignResponse) GetAuth() string {
+	if m != nil {
+		return m.Auth
+	}
+	return ""
+}
+
 type StatisticsRequest struct {
 	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
 	Collection  string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
@@ -966,71 +966,71 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("master.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1055 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x4b, 0x6f, 0xe4, 0x44,
-	0x10, 0x5e, 0x7b, 0x9e, 0xae, 0xd9, 0xc9, 0x4e, 0x3a, 0x11, 0xf2, 0xce, 0xbe, 0x06, 0x73, 0x19,
-	0x04, 0x8a, 0x96, 0x70, 0x44, 0x08, 0xb1, 0xd1, 0x22, 0xa2, 0x04, 0x36, 0x38, 0xb0, 0x07, 0x2e,
-	0xa6, 0x63, 0x57, 0xa2, 0x56, 0xfc, 0xa2, 0xbb, 0x27, 0x99, 0xd9, 0x0b, 0x47, 0xfe, 0x15, 0x17,
-	0xb8, 0xf1, 0x53, 0xb8, 0xf1, 0x0b, 0x50, 0x3f, 0xec, 0xf1, 0x38, 0x09, 0x91, 0x90, 0xb8, 0xb5,
-	0xbf, 0xae, 0xee, 0xaa, 0xfe, 0xbe, 0x7a, 0x18, 0x1e, 0x66, 0x54, 0x48, 0xe4, 0x7b, 0x25, 0x2f,
-	0x64, 0x41, 0x3c, 0xf3, 0x15, 0x95, 0x67, 0xc1, 0x5f, 0x2e, 0x78, 0x5f, 0x23, 0xe5, 0xf2, 0x0c,
-	0xa9, 0x24, 0x5b, 0xe0, 0xb2, 0xd2, 0x77, 0x66, 0xce, 0xdc, 0x0b, 0x5d, 0x56, 0x12, 0x02, 0xdd,
-	0xb2, 0xe0, 0xd2, 0x77, 0x67, 0xce, 0x7c, 0x1c, 0xea, 0x35, 0x79, 0x06, 0x50, 0x2e, 0xce, 0x52,
-	0x16, 0x47, 0x0b, 0x9e, 0xfa, 0x1d, 0x6d, 0xeb, 0x19, 0xe4, 0x07, 0x9e, 0x92, 0x39, 0x4c, 0x32,
-	0xba, 0x8c, 0xae, 0x8a, 0x74, 0x91, 0x61, 0x14, 0x17, 0x8b, 0x5c, 0xfa, 0x5d, 0x7d, 0x7c, 0x2b,
-	0xa3, 0xcb, 0xb7, 0x1a, 0x3e, 0x50, 0x28, 0x99, 0xa9, 0xa8, 0x96, 0xd1, 0x39, 0x4b, 0x31, 0xba,
-	0xc4, 0x95, 0xdf, 0x9b, 0x39, 0xf3, 0x6e, 0x08, 0x19, 0x5d, 0x7e, 0xc5, 0x52, 0x3c, 0xc2, 0x15,
-	0x79, 0x01, 0xa3, 0x84, 0x4a, 0x1a, 0xc5, 0x98, 0x4b, 0xe4, 0x7e, 0x5f, 0xfb, 0x02, 0x05, 0x1d,
-	0x68, 0x44, 0xc5, 0xc7, 0x69, 0x7c, 0xe9, 0x0f, 0xf4, 0x8e, 0x5e, 0xab, 0xf8, 0x68, 0x92, 0xb1,
-	0x3c, 0xd2, 0x91, 0x0f, 0xb5, 0x6b, 0x4f, 0x23, 0x27, 0x2a, 0xfc, 0xcf, 0x61, 0x60, 0x62, 0x13,
-	0xbe, 0x37, 0xeb, 0xcc, 0x47, 0xfb, 0x1f, 0xec, 0xd5, 0x6c, 0xec, 0x99, 0xf0, 0x0e, 0xf3, 0xf3,
-	0x82, 0x67, 0x54, 0xb2, 0x22, 0xff, 0x06, 0x85, 0xa0, 0x17, 0x18, 0x56, 0x67, 0xc8, 0x63, 0x18,
-	0xe6, 0x78, 0x1d, 0x5d, 0xb1, 0x44, 0xf8, 0x30, 0xeb, 0xcc, 0xc7, 0xe1, 0x20, 0xc7, 0xeb, 0xb7,
-	0x2c, 0x11, 0xe4, 0x7d, 0x78, 0x98, 0x60, 0x8a, 0x12, 0x13, 0xb3, 0x3d, 0xd2, 0xdb, 0x23, 0x8b,
-	0x29, 0x93, 0x40, 0xc0, 0x76, 0x4d, 0x76, 0x88, 0xa2, 0x2c, 0x72, 0x81, 0x64, 0x0e, 0x8f, 0xcc,
-	0xed, 0xa7, 0xec, 0x1d, 0x1e, 0xb3, 0x8c, 0x49, 0xad, 0x40, 0x37, 0x6c, 0xc3, 0xe4, 0x29, 0x78,
-	0x02, 0x63, 0x8e, 0xf2, 0x08, 0x57, 0x5a, 0x13, 0x2f, 0x5c, 0x03, 0xe4, 0x3d, 0xe8, 0xa7, 0x48,
-	0x13, 0xe4, 0x56, 0x14, 0xfb, 0x15, 0xfc, 0xe1, 0x82, 0x7f, 0xd7, 0xc3, 0xb4, 0xe2, 0x89, 0xf6,
-	0x37, 0x0e, 0x5d, 0x96, 0x28, 0x46, 0x05, 0x7b, 0x87, 0xfa, 0xf6, 0x6e, 0xa8, 0xd7, 0xe4, 0x39,
-	0x40, 0x5c, 0xa4, 0x29, 0xc6, 0xea, 0xa0, 0xbd, 0xbc, 0x81, 0x28, 0xc6, 0xb5, 0x88, 0x6b, 0xb1,
-	0xbb, 0xa1, 0xa7, 0x10, 0xa3, 0x73, 0xcd, 0x8b, 0x35, 0x30, 0x3a, 0x5b, 0x5e, 0x8c, 0xc9, 0xc7,
-	0x40, 0x2a, 0xea, 0xce, 0x56, 0xb5, 0x61, 0x5f, 0x1b, 0x4e, 0xec, 0xce, 0xab, 0x55, 0x65, 0xfd,
-	0x04, 0x3c, 0x8e, 0x34, 0x89, 0x8a, 0x3c, 0x5d, 0x69, 0xe9, 0x87, 0xe1, 0x50, 0x01, 0x6f, 0xf2,
-	0x74, 0x45, 0x3e, 0x82, 0x6d, 0x8e, 0x65, 0xca, 0x62, 0x1a, 0x95, 0x29, 0x8d, 0x31, 0xc3, 0xbc,
-	0xca, 0x82, 0x89, 0xdd, 0x38, 0xa9, 0x70, 0xe2, 0xc3, 0xe0, 0x0a, 0xb9, 0x50, 0xcf, 0xf2, 0xb4,
-	0x49, 0xf5, 0x49, 0x26, 0xd0, 0x91, 0x32, 0xf5, 0x41, 0xa3, 0x6a, 0x19, 0x0c, 0xa0, 0xf7, 0x3a,
-	0x2b, 0xe5, 0x2a, 0xf8, 0xcd, 0x81, 0x47, 0xa7, 0x8b, 0x12, 0xf9, 0xab, 0xb4, 0x88, 0x2f, 0x5f,
-	0x2f, 0x25, 0xa7, 0xe4, 0x0d, 0x6c, 0x21, 0xa7, 0x62, 0xc1, 0x55, 0xec, 0x09, 0xcb, 0x2f, 0x34,
-	0xa5, 0xa3, 0xfd, 0x79, 0x23, 0xb9, 0x5a, 0x67, 0xf6, 0x5e, 0x9b, 0x03, 0x07, 0xda, 0x3e, 0x1c,
-	0x63, 0xf3, 0x73, 0xfa, 0x23, 0x8c, 0x37, 0xf6, 0x95, 0x30, 0x2a, 0xf1, 0xad, 0x54, 0x7a, 0xad,
-	0x14, 0x2f, 0x29, 0x67, 0x72, 0x65, 0x0b, 0xd4, 0x7e, 0x29, 0x41, 0x6c, 0xfd, 0xa9, 0x3c, 0xec,
-	0xe8, 0x3c, 0xf4, 0x0c, 0x72, 0x98, 0x88, 0xe0, 0x43, 0xd8, 0x39, 0x48, 0x19, 0xe6, 0xf2, 0x98,
-	0x09, 0x89, 0x79, 0x88, 0x3f, 0x2f, 0x50, 0x48, 0xe5, 0x21, 0xa7, 0x19, 0xda, 0xf2, 0xd7, 0xeb,
-	0xe0, 0x17, 0xd8, 0x32, 0xa9, 0x73, 0x5c, 0xc4, 0x3a, 0x6f, 0x14, 0x31, 0xaa, 0xee, 0x8d, 0x91,
-	0x5a, 0xb6, 0x1a, 0x82, 0xdb, 0x6e, 0x08, 0xcd, 0x8a, 0xe9, 0xfc, 0x7b, 0xc5, 0x74, 0x6f, 0x56,
-	0xcc, 0xf7, 0xb0, 0x73, 0x5c, 0x14, 0x97, 0x8b, 0xd2, 0x84, 0x51, 0xc5, 0xba, 0xf9, 0x42, 0x67,
-	0xd6, 0x51, 0x3e, 0xeb, 0x17, 0xb6, 0x32, 0xd6, 0x6d, 0x67, 0x6c, 0xf0, 0xb7, 0x03, 0xbb, 0x9b,
-	0xd7, 0xda, 0x5a, 0xfc, 0x09, 0x76, 0xea, 0x7b, 0xa3, 0xd4, 0xbe, 0xd9, 0x38, 0x18, 0xed, 0xbf,
-	0x6c, 0x88, 0x79, 0xdb, 0xe9, 0xaa, 0x7d, 0x24, 0x15, 0x59, 0xe1, 0xf6, 0x55, 0x0b, 0x11, 0xd3,
-	0x25, 0x4c, 0xda, 0x66, 0x2a, 0xa1, 0x6b, 0xaf, 0x96, 0xd9, 0x61, 0x75, 0x92, 0x7c, 0x02, 0xde,
-	0x3a, 0x10, 0x57, 0x07, 0xb2, 0xb3, 0x11, 0x88, 0xf5, 0xb5, 0xb6, 0x22, 0xbb, 0xd0, 0x43, 0xce,
-	0x8b, 0xaa, 0x11, 0x98, 0x8f, 0xe0, 0x33, 0x18, 0xfe, 0x67, 0x15, 0x83, 0x3f, 0x1d, 0x18, 0x7f,
-	0x29, 0x04, 0xbb, 0xa8, 0xd3, 0x65, 0x17, 0x7a, 0xa6, 0x4c, 0x4d, 0xb3, 0x32, 0x1f, 0x64, 0x06,
-	0x23, 0x5b, 0x65, 0x0d, 0xea, 0x9b, 0xd0, 0xbd, 0xdd, 0xc4, 0x56, 0x5e, 0xd7, 0x84, 0x26, 0x65,
-	0xda, 0x1e, 0x03, 0xbd, 0x3b, 0xc7, 0x40, 0xbf, 0x31, 0x06, 0x9e, 0x80, 0xa7, 0x0f, 0xe5, 0x45,
-	0x82, 0x76, 0x3e, 0x0c, 0x15, 0xf0, 0x6d, 0x91, 0xe8, 0xb4, 0xae, 0x1e, 0x63, 0x85, 0x9f, 0x40,
-	0xe7, 0xbc, 0x26, 0x5f, 0x2d, 0x2b, 0x8a, 0xdc, 0xbb, 0x28, 0xba, 0x31, 0xf9, 0x6a, 0x42, 0xba,
-	0x4d, 0x42, 0x6a, 0x2d, 0x7a, 0x4d, 0x2d, 0x2e, 0x60, 0xfb, 0x54, 0x52, 0xc9, 0x84, 0x64, 0xb1,
-	0xa8, 0x18, 0x6d, 0x71, 0xe7, 0xdc, 0xc7, 0x9d, 0x7b, 0x17, 0x77, 0x9d, 0x9a, 0xbb, 0xe0, 0x77,
-	0x07, 0x48, 0xd3, 0x93, 0x7d, 0xee, 0xff, 0xe0, 0x4a, 0xd1, 0x23, 0x0b, 0x49, 0xd3, 0x48, 0x0f,
-	0x10, 0x3b, 0x06, 0x34, 0xa2, 0x26, 0x98, 0x12, 0x64, 0x21, 0x30, 0x31, 0xbb, 0x66, 0x06, 0x0c,
-	0x15, 0xa0, 0x37, 0x37, 0x47, 0x48, 0xbf, 0x35, 0x42, 0xf6, 0x7f, 0xed, 0xc0, 0xe0, 0x14, 0xe9,
-	0x35, 0x62, 0x42, 0x0e, 0x61, 0x7c, 0x8a, 0x79, 0xb2, 0xfe, 0x69, 0xd9, 0x6d, 0x54, 0x43, 0x8d,
-	0x4e, 0x9f, 0xde, 0x86, 0x56, 0xef, 0x0f, 0x1e, 0xcc, 0x9d, 0x97, 0x0e, 0x39, 0x81, 0xf1, 0x11,
-	0x62, 0x79, 0x50, 0xe4, 0x39, 0xc6, 0x12, 0x13, 0xf2, 0xbc, 0x71, 0xe8, 0x96, 0x16, 0x39, 0x7d,
-	0x7c, 0xe3, 0x5f, 0xa1, 0xaa, 0x28, 0x7b, 0xe3, 0x77, 0xf0, 0xb0, 0xd9, 0x19, 0x36, 0x2e, 0xbc,
-	0xa5, 0x8f, 0x4d, 0x5f, 0xdc, 0xd3, 0x52, 0x82, 0x07, 0xe4, 0x0b, 0xe8, 0x9b, 0x5c, 0x25, 0x7e,
-	0xc3, 0x78, 0xa3, 0x16, 0x37, 0xe2, 0xda, 0x4c, 0xec, 0xe0, 0x01, 0x39, 0x02, 0x58, 0x67, 0x00,
-	0x69, 0xf2, 0x72, 0x23, 0x05, 0xa7, 0xcf, 0xee, 0xd8, 0xad, 0x2e, 0x3b, 0xeb, 0xeb, 0x3f, 0xc8,
-	0x4f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x9f, 0x0a, 0x25, 0x51, 0x0a, 0x00, 0x00,
+	// 1056 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xcd, 0x6e, 0xe4, 0x44,
+	0x10, 0x8e, 0x3d, 0xbf, 0xae, 0xc9, 0x64, 0x27, 0x9d, 0x08, 0x79, 0x67, 0xd9, 0xdd, 0xc1, 0x5c,
+	0x06, 0x81, 0xa2, 0x25, 0x1c, 0x11, 0x42, 0x6c, 0x14, 0x44, 0x94, 0xc0, 0x06, 0x87, 0xdd, 0x03,
+	0x17, 0xd3, 0xb1, 0x2b, 0xa1, 0x15, 0xff, 0xe1, 0x6e, 0x27, 0x33, 0x7b, 0xe1, 0xc8, 0x03, 0xf0,
+	0x3e, 0x5c, 0xe0, 0xc6, 0xa3, 0x70, 0xe3, 0x09, 0x50, 0xff, 0xd8, 0xf1, 0x38, 0x19, 0x22, 0x21,
+	0x71, 0xeb, 0xfe, 0xba, 0xba, 0xab, 0xfa, 0xfb, 0xaa, 0xaa, 0x1b, 0x36, 0x13, 0xca, 0x05, 0x16,
+	0x7b, 0x79, 0x91, 0x89, 0x8c, 0x38, 0x7a, 0x16, 0xe4, 0xe7, 0xde, 0x5f, 0x36, 0x38, 0x5f, 0x21,
+	0x2d, 0xc4, 0x39, 0x52, 0x41, 0xb6, 0xc0, 0x66, 0xb9, 0x6b, 0xcd, 0xac, 0xb9, 0xe3, 0xdb, 0x2c,
+	0x27, 0x04, 0xba, 0x79, 0x56, 0x08, 0xd7, 0x9e, 0x59, 0xf3, 0xb1, 0xaf, 0xc6, 0xe4, 0x29, 0x40,
+	0x5e, 0x9e, 0xc7, 0x2c, 0x0c, 0xca, 0x22, 0x76, 0x3b, 0xca, 0xd6, 0xd1, 0xc8, 0xeb, 0x22, 0x26,
+	0x73, 0x98, 0x24, 0x74, 0x11, 0x5c, 0x67, 0x71, 0x99, 0x60, 0x10, 0x66, 0x65, 0x2a, 0xdc, 0xae,
+	0xda, 0xbe, 0x95, 0xd0, 0xc5, 0x1b, 0x05, 0x1f, 0x48, 0x94, 0xcc, 0x64, 0x54, 0x8b, 0xe0, 0x82,
+	0xc5, 0x18, 0x5c, 0xe1, 0xd2, 0xed, 0xcd, 0xac, 0x79, 0xd7, 0x87, 0x84, 0x2e, 0xbe, 0x64, 0x31,
+	0x1e, 0xe3, 0x92, 0x3c, 0x87, 0x51, 0x44, 0x05, 0x0d, 0x42, 0x4c, 0x05, 0x16, 0x6e, 0x5f, 0xf9,
+	0x02, 0x09, 0x1d, 0x28, 0x44, 0xc6, 0x57, 0xd0, 0xf0, 0xca, 0x1d, 0xa8, 0x15, 0x35, 0x96, 0xf1,
+	0xd1, 0x28, 0x61, 0x69, 0xa0, 0x22, 0x1f, 0x2a, 0xd7, 0x8e, 0x42, 0x4e, 0x65, 0xf8, 0x9f, 0xc1,
+	0x40, 0xc7, 0xc6, 0x5d, 0x67, 0xd6, 0x99, 0x8f, 0xf6, 0xdf, 0xdf, 0xab, 0xd9, 0xd8, 0xd3, 0xe1,
+	0x1d, 0xa5, 0x17, 0x59, 0x91, 0x50, 0xc1, 0xb2, 0xf4, 0x6b, 0xe4, 0x9c, 0x5e, 0xa2, 0x5f, 0xed,
+	0x21, 0x8f, 0x61, 0x98, 0xe2, 0x4d, 0x70, 0xcd, 0x22, 0xee, 0xc2, 0xac, 0x33, 0x1f, 0xfb, 0x83,
+	0x14, 0x6f, 0xde, 0xb0, 0x88, 0x93, 0xf7, 0x60, 0x33, 0xc2, 0x18, 0x05, 0x46, 0x7a, 0x79, 0xa4,
+	0x96, 0x47, 0x06, 0x93, 0x26, 0xde, 0x6b, 0xd8, 0xae, 0xc9, 0xf6, 0x91, 0xe7, 0x59, 0xca, 0x91,
+	0xcc, 0xe1, 0x91, 0x3e, 0xfd, 0x8c, 0xbd, 0xc5, 0x13, 0x96, 0x30, 0xa1, 0x14, 0xe8, 0xfa, 0x6d,
+	0x98, 0xbc, 0x03, 0xfd, 0x18, 0x69, 0x84, 0x85, 0xa1, 0xdd, 0xcc, 0xbc, 0x3f, 0x6c, 0x70, 0xd7,
+	0x85, 0xae, 0x34, 0x8d, 0xd4, 0x89, 0x63, 0xdf, 0x66, 0x91, 0xe4, 0x8c, 0xb3, 0xb7, 0xa8, 0x34,
+	0xed, 0xfa, 0x6a, 0x4c, 0x9e, 0x01, 0x84, 0x59, 0x1c, 0x63, 0x28, 0x37, 0x9a, 0xc3, 0x1b, 0x88,
+	0xe4, 0x54, 0xc9, 0x74, 0x2b, 0x67, 0xd7, 0x77, 0x24, 0xa2, 0x95, 0xac, 0x6f, 0x6e, 0x0c, 0xb4,
+	0x92, 0xe6, 0xe6, 0xda, 0xe4, 0x23, 0x20, 0x15, 0x39, 0xe7, 0xcb, 0xda, 0xb0, 0xaf, 0x0c, 0x27,
+	0x66, 0xe5, 0xe5, 0xb2, 0xb2, 0x7e, 0x02, 0x4e, 0x81, 0x34, 0x0a, 0xb2, 0x34, 0x5e, 0x2a, 0x71,
+	0x87, 0xfe, 0x50, 0x02, 0xaf, 0xd2, 0x78, 0x49, 0x3e, 0x84, 0xed, 0x02, 0xf3, 0x98, 0x85, 0x34,
+	0xc8, 0x63, 0x1a, 0x62, 0x82, 0x69, 0xa5, 0xf3, 0xc4, 0x2c, 0x9c, 0x56, 0x38, 0x71, 0x61, 0x70,
+	0x8d, 0x05, 0x97, 0xd7, 0x72, 0x94, 0x49, 0x35, 0x25, 0x13, 0xe8, 0x08, 0x11, 0xbb, 0xa0, 0x50,
+	0x39, 0xf4, 0x06, 0xd0, 0x3b, 0x4c, 0x72, 0xb1, 0xf4, 0x7e, 0xb3, 0xe0, 0xd1, 0x59, 0x99, 0x63,
+	0xf1, 0x32, 0xce, 0xc2, 0xab, 0xc3, 0x85, 0x28, 0x28, 0x79, 0x05, 0x5b, 0x58, 0x50, 0x5e, 0x16,
+	0x32, 0xf6, 0x88, 0xa5, 0x97, 0x8a, 0xd2, 0xd1, 0xfe, 0xbc, 0x91, 0x3e, 0xad, 0x3d, 0x7b, 0x87,
+	0x7a, 0xc3, 0x81, 0xb2, 0xf7, 0xc7, 0xd8, 0x9c, 0x4e, 0xbf, 0x87, 0xf1, 0xca, 0xba, 0x14, 0x46,
+	0xa6, 0xb6, 0x91, 0x4a, 0x8d, 0xa5, 0xe2, 0x39, 0x2d, 0x98, 0x58, 0x9a, 0x12, 0x34, 0x33, 0x29,
+	0x88, 0xa9, 0x30, 0x99, 0x69, 0x1d, 0x95, 0x69, 0x8e, 0x46, 0x8e, 0x22, 0xee, 0x7d, 0x00, 0x3b,
+	0x07, 0x31, 0xc3, 0x54, 0x9c, 0x30, 0x2e, 0x30, 0xf5, 0xf1, 0xa7, 0x12, 0xb9, 0x90, 0x1e, 0x52,
+	0x9a, 0xa0, 0x29, 0x70, 0x35, 0xf6, 0x7e, 0x86, 0x2d, 0x9d, 0x3a, 0x27, 0x59, 0xa8, 0xf2, 0x46,
+	0x12, 0x23, 0x2b, 0x5b, 0x1b, 0xc9, 0x61, 0xab, 0xe4, 0xed, 0x76, 0xc9, 0x37, 0x6b, 0xa2, 0xf3,
+	0xef, 0x35, 0xd1, 0xbd, 0x5b, 0x13, 0xdf, 0xc1, 0xce, 0x49, 0x96, 0x5d, 0x95, 0xb9, 0x0e, 0xa3,
+	0x8a, 0x75, 0xf5, 0x86, 0xd6, 0xac, 0x23, 0x7d, 0xd6, 0x37, 0x6c, 0x65, 0xac, 0xdd, 0xce, 0x58,
+	0xef, 0x6f, 0x0b, 0x76, 0x57, 0x8f, 0x35, 0xd5, 0xf6, 0x03, 0xec, 0xd4, 0xe7, 0x06, 0xb1, 0xb9,
+	0xb3, 0x76, 0x30, 0xda, 0x7f, 0xd1, 0x10, 0xf3, 0xbe, 0xdd, 0x55, 0x83, 0x88, 0x2a, 0xb2, 0xfc,
+	0xed, 0xeb, 0x16, 0xc2, 0xa7, 0x0b, 0x98, 0xb4, 0xcd, 0x64, 0x42, 0xd7, 0x5e, 0x0d, 0xb3, 0xc3,
+	0x6a, 0x27, 0xf9, 0x18, 0x9c, 0xdb, 0x40, 0x6c, 0x15, 0xc8, 0xce, 0x4a, 0x20, 0xc6, 0xd7, 0xad,
+	0x15, 0xd9, 0x85, 0x1e, 0x16, 0x45, 0x56, 0x35, 0x02, 0x3d, 0xf1, 0x3e, 0x85, 0xe1, 0x7f, 0x56,
+	0xd1, 0xfb, 0xd3, 0x82, 0xf1, 0x17, 0x9c, 0xb3, 0xcb, 0x3a, 0x5d, 0x76, 0xa1, 0xa7, 0xcb, 0x54,
+	0xb7, 0x23, 0x3d, 0x21, 0x33, 0x18, 0x99, 0x2a, 0x6b, 0x50, 0xdf, 0x84, 0x1e, 0xec, 0x26, 0xa6,
+	0xf2, 0xba, 0x3a, 0x34, 0x21, 0xe2, 0x76, 0xa3, 0xef, 0xad, 0x6d, 0xf4, 0xfd, 0x46, 0xa3, 0x7f,
+	0x02, 0x8e, 0xda, 0x94, 0x66, 0x11, 0x9a, 0x17, 0x60, 0x28, 0x81, 0x6f, 0xb2, 0x08, 0xbd, 0x5f,
+	0x2d, 0xd8, 0xaa, 0x6e, 0x63, 0x94, 0x9f, 0x40, 0xe7, 0xa2, 0x66, 0x5f, 0x0e, 0x2b, 0x8e, 0xec,
+	0x75, 0x1c, 0xdd, 0x79, 0xdc, 0x6a, 0x46, 0xba, 0x4d, 0x46, 0x6a, 0x31, 0x7a, 0x0d, 0x31, 0x64,
+	0xc8, 0xb4, 0x14, 0x3f, 0x56, 0x21, 0xcb, 0xb1, 0x77, 0x09, 0xdb, 0x67, 0x82, 0x0a, 0xc6, 0x05,
+	0x0b, 0x79, 0x45, 0x73, 0x8b, 0x50, 0xeb, 0x21, 0x42, 0xed, 0x75, 0x84, 0x76, 0x6a, 0x42, 0xbd,
+	0xdf, 0x2d, 0x20, 0x4d, 0x4f, 0x86, 0x82, 0xff, 0xc1, 0x95, 0xa4, 0x4c, 0x64, 0x82, 0xc6, 0x81,
+	0x7a, 0x55, 0xcc, 0xdb, 0xa0, 0x10, 0xf9, 0x70, 0x49, 0x95, 0x4a, 0x8e, 0x91, 0x5e, 0xd5, 0x0f,
+	0xc3, 0x50, 0x02, 0x6a, 0x71, 0xf5, 0x5d, 0xe9, 0xb7, 0xde, 0x95, 0xfd, 0x5f, 0x3a, 0x30, 0x38,
+	0x43, 0x7a, 0x83, 0x18, 0x91, 0x23, 0x18, 0x9f, 0x61, 0x1a, 0xdd, 0xfe, 0x55, 0x76, 0x1b, 0x25,
+	0x52, 0xa3, 0xd3, 0x77, 0xef, 0x43, 0xab, 0xfb, 0x7b, 0x1b, 0x73, 0xeb, 0x85, 0x45, 0x4e, 0x61,
+	0x7c, 0x8c, 0x98, 0x1f, 0x64, 0x69, 0x8a, 0xa1, 0xc0, 0x88, 0x3c, 0x6b, 0x6c, 0xba, 0xa7, 0x6f,
+	0x4e, 0x1f, 0xdf, 0xf9, 0x22, 0x54, 0x65, 0x66, 0x4e, 0xfc, 0x16, 0x36, 0x9b, 0xed, 0x62, 0xe5,
+	0xc0, 0x7b, 0x9a, 0xdb, 0xf4, 0xf9, 0x03, 0x7d, 0xc6, 0xdb, 0x20, 0x9f, 0x43, 0x5f, 0xe7, 0x2f,
+	0x71, 0x1b, 0xc6, 0x2b, 0x05, 0xba, 0x12, 0xd7, 0x6a, 0xb2, 0x7b, 0x1b, 0xe4, 0x18, 0xe0, 0x36,
+	0x03, 0x48, 0x93, 0x97, 0x3b, 0x29, 0x38, 0x7d, 0xba, 0x66, 0xb5, 0x3a, 0xec, 0xbc, 0xaf, 0x3e,
+	0x8e, 0x9f, 0xfc, 0x13, 0x00, 0x00, 0xff, 0xff, 0x25, 0xfc, 0xb7, 0x54, 0x48, 0x0a, 0x00, 0x00,
 }

weed/pb/volume_server.proto

@@ -24,15 +24,17 @@ service VolumeServer {
 
     rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) {
     }
-    rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (VolumeSyncIndexResponse) {
+    rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) {
     }
-    rpc VolumeSyncData (VolumeSyncDataRequest) returns (VolumeSyncDataResponse) {
+    rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) {
     }
 
     rpc VolumeMount (VolumeMountRequest) returns (VolumeMountResponse) {
     }
     rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
     }
+    rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
+    }
 
     // rpc VolumeUiPage (VolumeUiPageRequest) returns (VolumeUiPageResponse) {}
 
@@ -104,6 +106,7 @@ message VolumeSyncStatusRequest {
 }
 message VolumeSyncStatusResponse {
     uint32 volumd_id = 1;
+    string collection = 2;
     string replication = 4;
     string ttl = 5;
     uint64 tail_offset = 6;
@@ -141,6 +144,12 @@ message VolumeUnmountRequest {
 message VolumeUnmountResponse {
 }
 
+message VolumeDeleteRequest {
+    uint32 volumd_id = 1;
+}
+message VolumeDeleteResponse {
+}
+
 message VolumeUiPageRequest {
 }
 message VolumeUiPageResponse {

weed/pb/volume_server_pb/volume_server.pb.go

@@ -35,6 +35,8 @@ It has these top-level messages:
 	VolumeMountResponse
 	VolumeUnmountRequest
 	VolumeUnmountResponse
+	VolumeDeleteRequest
+	VolumeDeleteResponse
 	VolumeUiPageRequest
 	VolumeUiPageResponse
 	DiskStatus
@@ -352,6 +354,7 @@ func (m *VolumeSyncStatusRequest) GetVolumdId() uint32 {
 
 type VolumeSyncStatusResponse struct {
 	VolumdId        uint32 `protobuf:"varint,1,opt,name=volumd_id,json=volumdId" json:"volumd_id,omitempty"`
+	Collection      string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
 	Replication     string `protobuf:"bytes,4,opt,name=replication" json:"replication,omitempty"`
 	Ttl             string `protobuf:"bytes,5,opt,name=ttl" json:"ttl,omitempty"`
 	TailOffset      uint64 `protobuf:"varint,6,opt,name=tail_offset,json=tailOffset" json:"tail_offset,omitempty"`
@@ -371,6 +374,13 @@ func (m *VolumeSyncStatusResponse) GetVolumdId() uint32 {
 	return 0
 }
 
+func (m *VolumeSyncStatusResponse) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
 func (m *VolumeSyncStatusResponse) GetReplication() string {
 	if m != nil {
 		return m.Replication
@@ -550,13 +560,37 @@ func (m *VolumeUnmountResponse) String() string            { return proto.Compac
 func (*VolumeUnmountResponse) ProtoMessage()               {}
 func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
+type VolumeDeleteRequest struct {
+	VolumdId uint32 `protobuf:"varint,1,opt,name=volumd_id,json=volumdId" json:"volumd_id,omitempty"`
+}
+
+func (m *VolumeDeleteRequest) Reset()                    { *m = VolumeDeleteRequest{} }
+func (m *VolumeDeleteRequest) String() string            { return proto.CompactTextString(m) }
+func (*VolumeDeleteRequest) ProtoMessage()               {}
+func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+
+func (m *VolumeDeleteRequest) GetVolumdId() uint32 {
+	if m != nil {
+		return m.VolumdId
+	}
+	return 0
+}
+
+type VolumeDeleteResponse struct {
+}
+
+func (m *VolumeDeleteResponse) Reset()                    { *m = VolumeDeleteResponse{} }
+func (m *VolumeDeleteResponse) String() string            { return proto.CompactTextString(m) }
+func (*VolumeDeleteResponse) ProtoMessage()               {}
+func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+
 type VolumeUiPageRequest struct {
 }
 
 func (m *VolumeUiPageRequest) Reset()                    { *m = VolumeUiPageRequest{} }
 func (m *VolumeUiPageRequest) String() string            { return proto.CompactTextString(m) }
 func (*VolumeUiPageRequest) ProtoMessage()               {}
-func (*VolumeUiPageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+func (*VolumeUiPageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
 
 type VolumeUiPageResponse struct {
 }
@@ -564,7 +598,7 @@ type VolumeUiPageResponse struct {
 func (m *VolumeUiPageResponse) Reset()                    { *m = VolumeUiPageResponse{} }
 func (m *VolumeUiPageResponse) String() string            { return proto.CompactTextString(m) }
 func (*VolumeUiPageResponse) ProtoMessage()               {}
-func (*VolumeUiPageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+func (*VolumeUiPageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
 
 type DiskStatus struct {
 	Dir  string `protobuf:"bytes,1,opt,name=dir" json:"dir,omitempty"`
@@ -576,7 +610,7 @@ type DiskStatus struct {
 func (m *DiskStatus) Reset()                    { *m = DiskStatus{} }
 func (m *DiskStatus) String() string            { return proto.CompactTextString(m) }
 func (*DiskStatus) ProtoMessage()               {}
-func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
 
 func (m *DiskStatus) GetDir() string {
 	if m != nil {
@@ -619,7 +653,7 @@ type MemStatus struct {
 func (m *MemStatus) Reset()                    { *m = MemStatus{} }
 func (m *MemStatus) String() string            { return proto.CompactTextString(m) }
 func (*MemStatus) ProtoMessage()               {}
-func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
 
 func (m *MemStatus) GetGoroutines() int32 {
 	if m != nil {
@@ -697,6 +731,8 @@ func init() {
 	proto.RegisterType((*VolumeMountResponse)(nil), "volume_server_pb.VolumeMountResponse")
 	proto.RegisterType((*VolumeUnmountRequest)(nil), "volume_server_pb.VolumeUnmountRequest")
 	proto.RegisterType((*VolumeUnmountResponse)(nil), "volume_server_pb.VolumeUnmountResponse")
+	proto.RegisterType((*VolumeDeleteRequest)(nil), "volume_server_pb.VolumeDeleteRequest")
+	proto.RegisterType((*VolumeDeleteResponse)(nil), "volume_server_pb.VolumeDeleteResponse")
 	proto.RegisterType((*VolumeUiPageRequest)(nil), "volume_server_pb.VolumeUiPageRequest")
 	proto.RegisterType((*VolumeUiPageResponse)(nil), "volume_server_pb.VolumeUiPageResponse")
 	proto.RegisterType((*DiskStatus)(nil), "volume_server_pb.DiskStatus")
@@ -723,10 +759,11 @@ type VolumeServerClient interface {
 	DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
 	AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
 	VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error)
-	VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (*VolumeSyncIndexResponse, error)
-	VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (*VolumeSyncDataResponse, error)
+	VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error)
+	VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error)
 	VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error)
 	VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error)
+	VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error)
 }
 
 type volumeServerClient struct {
@@ -809,22 +846,68 @@ func (c *volumeServerClient) VolumeSyncStatus(ctx context.Context, in *VolumeSyn
 	return out, nil
 }
 
-func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (*VolumeSyncIndexResponse, error) {
-	out := new(VolumeSyncIndexResponse)
-	err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeSyncIndex", in, out, c.cc, opts...)
+func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncIndex", opts...)
 	if err != nil {
 		return nil, err
 	}
-	return out, nil
+	x := &volumeServerVolumeSyncIndexClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
 }
 
-func (c *volumeServerClient) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (*VolumeSyncDataResponse, error) {
-	out := new(VolumeSyncDataResponse)
-	err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeSyncData", in, out, c.cc, opts...)
+type VolumeServer_VolumeSyncIndexClient interface {
+	Recv() (*VolumeSyncIndexResponse, error)
+	grpc.ClientStream
+}
+
+type volumeServerVolumeSyncIndexClient struct {
+	grpc.ClientStream
+}
+
+func (x *volumeServerVolumeSyncIndexClient) Recv() (*VolumeSyncIndexResponse, error) {
+	m := new(VolumeSyncIndexResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *volumeServerClient) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncData", opts...)
 	if err != nil {
 		return nil, err
 	}
-	return out, nil
+	x := &volumeServerVolumeSyncDataClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type VolumeServer_VolumeSyncDataClient interface {
+	Recv() (*VolumeSyncDataResponse, error)
+	grpc.ClientStream
+}
+
+type volumeServerVolumeSyncDataClient struct {
+	grpc.ClientStream
+}
+
+func (x *volumeServerVolumeSyncDataClient) Recv() (*VolumeSyncDataResponse, error) {
+	m := new(VolumeSyncDataResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
 }
 
 func (c *volumeServerClient) VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error) {
@@ -845,6 +928,15 @@ func (c *volumeServerClient) VolumeUnmount(ctx context.Context, in *VolumeUnmoun
 	return out, nil
 }
 
+func (c *volumeServerClient) VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error) {
+	out := new(VolumeDeleteResponse)
+	err := grpc.Invoke(ctx, "/volume_server_pb.VolumeServer/VolumeDelete", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // Server API for VolumeServer service
 
 type VolumeServerServer interface {
@@ -857,10 +949,11 @@ type VolumeServerServer interface {
 	DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
 	AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
 	VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error)
-	VolumeSyncIndex(context.Context, *VolumeSyncIndexRequest) (*VolumeSyncIndexResponse, error)
-	VolumeSyncData(context.Context, *VolumeSyncDataRequest) (*VolumeSyncDataResponse, error)
+	VolumeSyncIndex(*VolumeSyncIndexRequest, VolumeServer_VolumeSyncIndexServer) error
+	VolumeSyncData(*VolumeSyncDataRequest, VolumeServer_VolumeSyncDataServer) error
 	VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error)
 	VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error)
+	VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error)
 }
 
 func RegisterVolumeServerServer(s *grpc.Server, srv VolumeServerServer) {
@@ -1011,40 +1104,46 @@ func _VolumeServer_VolumeSyncStatus_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }
 
-func _VolumeServer_VolumeSyncIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(VolumeSyncIndexRequest)
-	if err := dec(in); err != nil {
-		return nil, err
+func _VolumeServer_VolumeSyncIndex_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(VolumeSyncIndexRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
 	}
-	if interceptor == nil {
-		return srv.(VolumeServerServer).VolumeSyncIndex(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/volume_server_pb.VolumeServer/VolumeSyncIndex",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(VolumeServerServer).VolumeSyncIndex(ctx, req.(*VolumeSyncIndexRequest))
-	}
-	return interceptor(ctx, in, info, handler)
+	return srv.(VolumeServerServer).VolumeSyncIndex(m, &volumeServerVolumeSyncIndexServer{stream})
 }
 
-func _VolumeServer_VolumeSyncData_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(VolumeSyncDataRequest)
-	if err := dec(in); err != nil {
-		return nil, err
+type VolumeServer_VolumeSyncIndexServer interface {
+	Send(*VolumeSyncIndexResponse) error
+	grpc.ServerStream
+}
+
+type volumeServerVolumeSyncIndexServer struct {
+	grpc.ServerStream
+}
+
+func (x *volumeServerVolumeSyncIndexServer) Send(m *VolumeSyncIndexResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _VolumeServer_VolumeSyncData_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(VolumeSyncDataRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
 	}
-	if interceptor == nil {
-		return srv.(VolumeServerServer).VolumeSyncData(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/volume_server_pb.VolumeServer/VolumeSyncData",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(VolumeServerServer).VolumeSyncData(ctx, req.(*VolumeSyncDataRequest))
-	}
-	return interceptor(ctx, in, info, handler)
+	return srv.(VolumeServerServer).VolumeSyncData(m, &volumeServerVolumeSyncDataServer{stream})
+}
+
+type VolumeServer_VolumeSyncDataServer interface {
+	Send(*VolumeSyncDataResponse) error
+	grpc.ServerStream
+}
+
+type volumeServerVolumeSyncDataServer struct {
+	grpc.ServerStream
+}
+
+func (x *volumeServerVolumeSyncDataServer) Send(m *VolumeSyncDataResponse) error {
+	return x.ServerStream.SendMsg(m)
 }
 
 func _VolumeServer_VolumeMount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
@@ -1083,6 +1182,24 @@ func _VolumeServer_VolumeUnmount_Handler(srv interface{}, ctx context.Context, d
 	return interceptor(ctx, in, info, handler)
 }
 
+func _VolumeServer_VolumeDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(VolumeDeleteRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VolumeServerServer).VolumeDelete(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/volume_server_pb.VolumeServer/VolumeDelete",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VolumeServerServer).VolumeDelete(ctx, req.(*VolumeDeleteRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _VolumeServer_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "volume_server_pb.VolumeServer",
 	HandlerType: (*VolumeServerServer)(nil),
@@ -1119,14 +1236,6 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
 			MethodName: "VolumeSyncStatus",
 			Handler:    _VolumeServer_VolumeSyncStatus_Handler,
 		},
-		{
-			MethodName: "VolumeSyncIndex",
-			Handler:    _VolumeServer_VolumeSyncIndex_Handler,
-		},
-		{
-			MethodName: "VolumeSyncData",
-			Handler:    _VolumeServer_VolumeSyncData_Handler,
-		},
 		{
 			MethodName: "VolumeMount",
 			Handler:    _VolumeServer_VolumeMount_Handler,
@@ -1135,77 +1244,94 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{
 			MethodName: "VolumeUnmount",
 			Handler:    _VolumeServer_VolumeUnmount_Handler,
 		},
+		{
+			MethodName: "VolumeDelete",
+			Handler:    _VolumeServer_VolumeDelete_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "VolumeSyncIndex",
+			Handler:       _VolumeServer_VolumeSyncIndex_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "VolumeSyncData",
+			Handler:       _VolumeServer_VolumeSyncData_Handler,
+			ServerStreams: true,
+		},
 	},
-	Streams:  []grpc.StreamDesc{},
 	Metadata: "volume_server.proto",
 }
 
 func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1013 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x57, 0xdf, 0x72, 0xdb, 0xc4,
-	0x17, 0x8e, 0x62, 0x3b, 0x76, 0x8e, 0xed, 0x5f, 0xfd, 0x5b, 0xa7, 0x89, 0xaa, 0x42, 0x30, 0x02,
-	0x5a, 0xa7, 0x0d, 0x61, 0x68, 0x07, 0x28, 0xc3, 0x0d, 0x90, 0x00, 0x93, 0x8b, 0x4e, 0x99, 0xcd,
-	0xb4, 0xc3, 0x0c, 0x9d, 0xf1, 0x28, 0xd2, 0x3a, 0xd9, 0x89, 0x2c, 0xa9, 0xda, 0x55, 0x26, 0xe5,
-	0x0d, 0x78, 0x04, 0xae, 0xb9, 0xe1, 0x9d, 0x78, 0x19, 0x66, 0xff, 0x48, 0xd1, 0x3f, 0x57, 0x82,
-	0xbb, 0xd5, 0xb7, 0xe7, 0x9c, 0x6f, 0xcf, 0xd9, 0xb3, 0xe7, 0xb3, 0x61, 0x7a, 0x1d, 0xfa, 0xc9,
-	0x8a, 0x2c, 0x18, 0x89, 0xaf, 0x49, 0x7c, 0x14, 0xc5, 0x21, 0x0f, 0xd1, 0xa4, 0x00, 0x2e, 0xa2,
-	0x73, 0xfb, 0x33, 0x40, 0xdf, 0x3b, 0xdc, 0xbd, 0x3c, 0x21, 0x3e, 0xe1, 0x04, 0x93, 0x37, 0x09,
-	0x61, 0x1c, 0xdd, 0x83, 0xc1, 0x92, 0xfa, 0x64, 0x41, 0x3d, 0x66, 0x1a, 0xb3, 0xce, 0x7c, 0x1b,
-	0xf7, 0xc5, 0xf7, 0xa9, 0xc7, 0xec, 0x17, 0x30, 0x2d, 0x38, 0xb0, 0x28, 0x0c, 0x18, 0x41, 0xcf,
-	0xa0, 0x1f, 0x13, 0x96, 0xf8, 0x5c, 0x39, 0x0c, 0x9f, 0xec, 0x1f, 0x95, 0xb9, 0x8e, 0x32, 0x97,
-	0xc4, 0xe7, 0x38, 0x35, 0xb7, 0x29, 0x8c, 0xf2, 0x1b, 0x68, 0x0f, 0xfa, 0x9a, 0xdb, 0x34, 0x66,
-	0xc6, 0x7c, 0x1b, 0x6f, 0x29, 0x6a, 0xb4, 0x0b, 0x5b, 0x8c, 0x3b, 0x3c, 0x61, 0xe6, 0xe6, 0xcc,
-	0x98, 0xf7, 0xb0, 0xfe, 0x42, 0x3b, 0xd0, 0x23, 0x71, 0x1c, 0xc6, 0x66, 0x47, 0x9a, 0xab, 0x0f,
-	0x84, 0xa0, 0xcb, 0xe8, 0x6f, 0xc4, 0xec, 0xce, 0x8c, 0xf9, 0x18, 0xcb, 0xb5, 0xdd, 0x87, 0xde,
-	0x0f, 0xab, 0x88, 0xbf, 0xb5, 0xbf, 0x02, 0xf3, 0x95, 0xe3, 0x26, 0xc9, 0xea, 0x95, 0x3c, 0xe3,
-	0xf1, 0x25, 0x71, 0xaf, 0xd2, 0xdc, 0xef, 0xc3, 0xb6, 0x3c, 0xb9, 0x97, 0x9e, 0x60, 0x8c, 0x07,
-	0x0a, 0x38, 0xf5, 0xec, 0x6f, 0xe1, 0x5e, 0x8d, 0xa3, 0xae, 0xc1, 0x47, 0x30, 0xbe, 0x70, 0xe2,
-	0x73, 0xe7, 0x82, 0x2c, 0x62, 0x87, 0xd3, 0x50, 0x7a, 0x1b, 0x78, 0xa4, 0x41, 0x2c, 0x30, 0xfb,
-	0x57, 0xb0, 0x0a, 0x11, 0xc2, 0x55, 0xe4, 0xb8, 0xbc, 0x0d, 0x39, 0x9a, 0xc1, 0x30, 0x8a, 0x89,
-	0xe3, 0xfb, 0xa1, 0xeb, 0x70, 0x22, 0xab, 0xd0, 0xc1, 0x79, 0xc8, 0x7e, 0x1f, 0xee, 0xd7, 0x06,
-	0x57, 0x07, 0xb4, 0x9f, 0x95, 0x4e, 0x1f, 0xae, 0x56, 0xb4, 0x15, 0xb5, 0xfd, 0x5e, 0xe5, 0xd4,
-	0xd2, 0x53, 0xc7, 0xfd, 0xba, 0xb4, 0xeb, 0x13, 0x27, 0x48, 0xa2, 0x56, 0x81, 0xcb, 0x27, 0x4e,
-	0x5d, 0xb3, 0xc8, 0x7b, 0xaa, 0x39, 0x8e, 0x43, 0xdf, 0x27, 0x2e, 0xa7, 0x61, 0x90, 0x86, 0xdd,
-	0x07, 0x70, 0x33, 0x50, 0xb7, 0x4a, 0x0e, 0xb1, 0x2d, 0x30, 0xab, 0xae, 0x3a, 0xec, 0x5f, 0x06,
-	0x4c, 0xbf, 0x63, 0x8c, 0x5e, 0x04, 0x8a, 0xb6, 0x55, 0xf9, 0x8b, 0x84, 0x9b, 0x65, 0xc2, 0xf2,
-	0xf5, 0x74, 0x2a, 0xd7, 0x23, 0x2c, 0x62, 0x12, 0xf9, 0xd4, 0x75, 0x64, 0x88, 0xae, 0x0c, 0x91,
-	0x87, 0xd0, 0x04, 0x3a, 0x9c, 0xfb, 0x66, 0x4f, 0xee, 0x88, 0xa5, 0xbd, 0x0b, 0x3b, 0xc5, 0x93,
-	0xea, 0x14, 0xbe, 0x84, 0x3d, 0x85, 0x9c, 0xbd, 0x0d, 0xdc, 0x33, 0xf9, 0x12, 0x5a, 0x15, 0xfc,
-	0x6f, 0x03, 0xcc, 0xaa, 0xa3, 0xee, 0xe0, 0xa6, 0xf6, 0xfb, 0xb7, 0xa7, 0x47, 0x1f, 0xc0, 0x90,
-	0x3b, 0xd4, 0x5f, 0x84, 0xcb, 0x25, 0x23, 0xdc, 0xdc, 0x9a, 0x19, 0xf3, 0x2e, 0x06, 0x01, 0xbd,
-	0x90, 0x08, 0x3a, 0x80, 0x89, 0xab, 0xba, 0x74, 0x11, 0x93, 0x6b, 0xca, 0x44, 0xe4, 0xbe, 0x24,
-	0xbe, 0xe3, 0xa6, 0xdd, 0xab, 0x60, 0x64, 0xc3, 0x98, 0x7a, 0x37, 0x0b, 0x39, 0x1c, 0xe4, 0xd3,
-	0x1e, 0xc8, 0x68, 0x43, 0xea, 0xdd, 0xfc, 0x48, 0x7d, 0x72, 0x26, 0x5e, 0xf8, 0x17, 0xb0, 0x7b,
-	0x9b, 0xdc, 0x69, 0xe0, 0x91, 0x9b, 0x56, 0x45, 0xf9, 0x29, 0x5f, 0x4c, 0xed, 0xa6, 0x4b, 0x72,
-	0x08, 0x88, 0x0a, 0x40, 0xf1, 0xba, 0x61, 0xc0, 0x49, 0xc0, 0x65, 0x80, 0x11, 0x9e, 0xc8, 0x1d,
-	0x41, 0x7e, 0xac, 0x70, 0xfb, 0x0f, 0x03, 0xee, 0xde, 0x46, 0x3a, 0x71, 0xb8, 0xd3, 0xaa, 0xb5,
-	0x2c, 0x18, 0x64, 0xd9, 0x6f, 0xaa, 0xbd, 0xf4, 0x5b, 0x8c, 0x3d, 0x5d, 0xbd, 0x8e, 0xdc, 0xd1,
-	0x5f, 0x75, 0x03, 0x4e, 0x90, 0x04, 0x84, 0x78, 0x6a, 0x7a, 0xaa, 0x6b, 0x18, 0x28, 0xe0, 0xd4,
-	0xb3, 0xbf, 0xc9, 0xd7, 0x46, 0x1d, 0x4d, 0xe7, 0xf8, 0x21, 0x8c, 0x6a, 0xb2, 0x1b, 0x2e, 0x73,
-	0x89, 0x7d, 0x0e, 0x48, 0x39, 0x3f, 0x0f, 0x93, 0xa0, 0xdd, 0xcc, 0xb8, 0x0b, 0xd3, 0x82, 0x8b,
-	0x6e, 0xdc, 0xa7, 0xb0, 0xa3, 0xe0, 0x97, 0xc1, 0xaa, 0x75, 0xac, 0xbd, 0xb4, 0xac, 0x99, 0x93,
-	0x8e, 0x96, 0x91, 0xbc, 0xa4, 0x3f, 0x8b, 0x19, 0xab, 0x82, 0x89, 0x57, 0x53, 0x84, 0xb5, 0xf9,
-	0x2f, 0x00, 0x27, 0x94, 0x5d, 0xa9, 0xb6, 0x17, 0xfd, 0xea, 0xd1, 0x58, 0xcf, 0x0e, 0xb1, 0x14,
-	0x88, 0xe3, 0xfb, 0xf2, 0x0e, 0xba, 0x58, 0x2c, 0x45, 0x99, 0x13, 0x46, 0x3c, 0x59, 0xfc, 0x2e,
-	0x96, 0x6b, 0x81, 0x2d, 0x63, 0xa2, 0x4a, 0xdf, 0xc5, 0x72, 0x6d, 0xff, 0x69, 0xc0, 0xf6, 0x73,
-	0xb2, 0xd2, 0x91, 0xf7, 0x01, 0x2e, 0xc2, 0x38, 0x4c, 0x38, 0x0d, 0x08, 0x93, 0x04, 0x3d, 0x9c,
-	0x43, 0xfe, 0x3b, 0x8f, 0xbc, 0x76, 0xe2, 0x2f, 0xe5, 0xed, 0x76, 0xb1, 0x5c, 0x0b, 0xec, 0x92,
-	0x38, 0x91, 0x7e, 0x5e, 0x72, 0x2d, 0x54, 0x91, 0x71, 0xc7, 0xbd, 0x92, 0xaf, 0xa9, 0x8b, 0xd5,
-	0xc7, 0x93, 0xdf, 0x01, 0x46, 0xba, 0x09, 0xa4, 0x2c, 0xa3, 0xd7, 0x30, 0xcc, 0xc9, 0x39, 0xfa,
-	0xb8, 0xaa, 0xda, 0xd5, 0x9f, 0x07, 0xd6, 0x27, 0x0d, 0x56, 0xba, 0xd8, 0x1b, 0x28, 0x80, 0xff,
-	0x57, 0xe4, 0x12, 0x3d, 0xaa, 0x7a, 0xaf, 0x13, 0x63, 0xeb, 0x71, 0x2b, 0xdb, 0x8c, 0x8f, 0xc3,
-	0xb4, 0x46, 0xff, 0xd0, 0x61, 0x43, 0x94, 0x82, 0x06, 0x5b, 0x9f, 0xb6, 0xb4, 0xce, 0x58, 0xdf,
-	0x00, 0xaa, 0x8a, 0x23, 0x7a, 0xdc, 0x18, 0xe6, 0x56, 0x7c, 0xad, 0xc3, 0x76, 0xc6, 0x6b, 0x13,
-	0x55, 0xb2, 0xd9, 0x98, 0x68, 0x41, 0x98, 0x1b, 0x13, 0x2d, 0x69, 0xf1, 0x06, 0xba, 0x82, 0x49,
-	0x59, 0x52, 0xd1, 0xc1, 0xba, 0xdf, 0x79, 0x15, 0xc5, 0xb6, 0x1e, 0xb5, 0x31, 0xcd, 0xc8, 0x16,
-	0x30, 0xca, 0x0b, 0x1f, 0xaa, 0x69, 0xba, 0x1a, 0x09, 0xb7, 0x1e, 0x34, 0x99, 0xe5, 0xb3, 0x29,
-	0x0b, 0x61, 0x5d, 0x36, 0x6b, 0x54, 0xb6, 0x2e, 0x9b, 0x75, 0xba, 0x6a, 0x6f, 0xa0, 0x4b, 0xb8,
-	0x53, 0x52, 0x18, 0x34, 0x7f, 0x57, 0x80, 0xbc, 0x76, 0x59, 0x07, 0x2d, 0x2c, 0x33, 0x26, 0x02,
-	0xff, 0x2b, 0x8e, 0x79, 0xf4, 0xf0, 0x5d, 0xee, 0x39, 0x8d, 0xb2, 0xe6, 0xcd, 0x86, 0x19, 0xcd,
-	0x6b, 0x18, 0xe6, 0xa6, 0x7b, 0xdd, 0xe0, 0xa8, 0xea, 0x45, 0xdd, 0xe0, 0xa8, 0x93, 0x88, 0x0d,
-	0x74, 0x0e, 0xe3, 0xc2, 0xbc, 0x47, 0x0f, 0xd6, 0x79, 0x16, 0x55, 0xc4, 0x7a, 0xd8, 0x68, 0x97,
-	0x72, 0x9c, 0x6f, 0xc9, 0xff, 0x44, 0x4f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x60, 0xe6, 0x57,
-	0xdf, 0x2a, 0x0d, 0x00, 0x00,
+	// 1044 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x57, 0xdd, 0x72, 0xdb, 0x44,
+	0x14, 0x8e, 0x6a, 0x3b, 0x76, 0x8e, 0x6d, 0x6a, 0xd6, 0x69, 0xa2, 0xaa, 0x10, 0x8c, 0x80, 0xd4,
+	0x69, 0x43, 0x80, 0x74, 0x80, 0x32, 0xdc, 0x00, 0x09, 0x30, 0xb9, 0xe8, 0x94, 0xd9, 0x4c, 0x3b,
+	0xcc, 0xd0, 0x19, 0x8f, 0x22, 0xad, 0x9d, 0x25, 0xb2, 0xe4, 0x6a, 0x57, 0x99, 0x94, 0x37, 0xe1,
+	0x9a, 0x1b, 0x9e, 0x8e, 0x17, 0xe0, 0x86, 0xd9, 0x1f, 0xd9, 0xfa, 0x73, 0x24, 0xe0, 0x6e, 0xf7,
+	0xec, 0x39, 0xdf, 0xf9, 0xd9, 0xa3, 0xf3, 0xad, 0x60, 0x78, 0x1d, 0xfa, 0xf1, 0x9c, 0x4c, 0x18,
+	0x89, 0xae, 0x49, 0x74, 0xb4, 0x88, 0x42, 0x1e, 0xa2, 0x41, 0x46, 0x38, 0x59, 0x5c, 0xd8, 0x9f,
+	0x00, 0xfa, 0xce, 0xe1, 0xee, 0xe5, 0x29, 0xf1, 0x09, 0x27, 0x98, 0xbc, 0x8e, 0x09, 0xe3, 0xe8,
+	0x3e, 0x74, 0xa6, 0xd4, 0x27, 0x13, 0xea, 0x31, 0xd3, 0x18, 0x35, 0xc6, 0x5b, 0xb8, 0x2d, 0xf6,
+	0x67, 0x1e, 0xb3, 0x9f, 0xc3, 0x30, 0x63, 0xc0, 0x16, 0x61, 0xc0, 0x08, 0x7a, 0x0a, 0xed, 0x88,
+	0xb0, 0xd8, 0xe7, 0xca, 0xa0, 0x7b, 0xbc, 0x77, 0x94, 0xf7, 0x75, 0xb4, 0x34, 0x89, 0x7d, 0x8e,
+	0x13, 0x75, 0x9b, 0x42, 0x2f, 0x7d, 0x80, 0x76, 0xa1, 0xad, 0x7d, 0x9b, 0xc6, 0xc8, 0x18, 0x6f,
+	0xe1, 0x4d, 0xe5, 0x1a, 0xed, 0xc0, 0x26, 0xe3, 0x0e, 0x8f, 0x99, 0x79, 0x67, 0x64, 0x8c, 0x5b,
+	0x58, 0xef, 0xd0, 0x36, 0xb4, 0x48, 0x14, 0x85, 0x91, 0xd9, 0x90, 0xea, 0x6a, 0x83, 0x10, 0x34,
+	0x19, 0xfd, 0x8d, 0x98, 0xcd, 0x91, 0x31, 0xee, 0x63, 0xb9, 0xb6, 0xdb, 0xd0, 0xfa, 0x7e, 0xbe,
+	0xe0, 0x6f, 0xec, 0x2f, 0xc1, 0x7c, 0xe9, 0xb8, 0x71, 0x3c, 0x7f, 0x29, 0x63, 0x3c, 0xb9, 0x24,
+	0xee, 0x55, 0x92, 0xfb, 0x03, 0xd8, 0x92, 0x91, 0x7b, 0x49, 0x04, 0x7d, 0xdc, 0x51, 0x82, 0x33,
+	0xcf, 0xfe, 0x06, 0xee, 0x97, 0x18, 0xea, 0x1a, 0x7c, 0x00, 0xfd, 0x99, 0x13, 0x5d, 0x38, 0x33,
+	0x32, 0x89, 0x1c, 0x4e, 0x43, 0x69, 0x6d, 0xe0, 0x9e, 0x16, 0x62, 0x21, 0xb3, 0x7f, 0x01, 0x2b,
+	0x83, 0x10, 0xce, 0x17, 0x8e, 0xcb, 0xeb, 0x38, 0x47, 0x23, 0xe8, 0x2e, 0x22, 0xe2, 0xf8, 0x7e,
+	0xe8, 0x3a, 0x9c, 0xc8, 0x2a, 0x34, 0x70, 0x5a, 0x64, 0xbf, 0x0b, 0x0f, 0x4a, 0xc1, 0x55, 0x80,
+	0xf6, 0xd3, 0x5c, 0xf4, 0xe1, 0x7c, 0x4e, 0x6b, 0xb9, 0xb6, 0xdf, 0x29, 0x44, 0x2d, 0x2d, 0x35,
+	0xee, 0x57, 0xb9, 0x53, 0x9f, 0x38, 0x41, 0xbc, 0xa8, 0x05, 0x9c, 0x8f, 0x38, 0x31, 0x5d, 0x22,
+	0xef, 0xaa, 0xe6, 0x38, 0x09, 0x7d, 0x9f, 0xb8, 0x9c, 0x86, 0x41, 0x02, 0xbb, 0x07, 0xe0, 0x2e,
+	0x85, 0xba, 0x55, 0x52, 0x12, 0xdb, 0x02, 0xb3, 0x68, 0xaa, 0x61, 0xff, 0x34, 0x60, 0xf8, 0x2d,
+	0x63, 0x74, 0x16, 0x28, 0xb7, 0xb5, 0xca, 0x9f, 0x75, 0x78, 0x27, 0xef, 0x30, 0x7f, 0x3d, 0x8d,
+	0xc2, 0xf5, 0x08, 0x8d, 0x88, 0x2c, 0x7c, 0xea, 0x3a, 0x12, 0xa2, 0x29, 0x21, 0xd2, 0x22, 0x34,
+	0x80, 0x06, 0xe7, 0xbe, 0xd9, 0x92, 0x27, 0x62, 0x69, 0xef, 0xc0, 0x76, 0x36, 0x52, 0x9d, 0xc2,
+	0x17, 0xb0, 0xab, 0x24, 0xe7, 0x6f, 0x02, 0xf7, 0x5c, 0x7e, 0x09, 0xb5, 0x0a, 0xfe, 0xb7, 0x01,
+	0x66, 0xd1, 0x50, 0x77, 0xf0, 0xff, 0xcd, 0xff, 0xdf, 0x66, 0x87, 0xde, 0x83, 0x2e, 0x77, 0xa8,
+	0x3f, 0x09, 0xa7, 0x53, 0x46, 0xb8, 0xb9, 0x39, 0x32, 0xc6, 0x4d, 0x0c, 0x42, 0xf4, 0x5c, 0x4a,
+	0xd0, 0x01, 0x0c, 0x5c, 0xd5, 0xc5, 0x93, 0x88, 0x5c, 0x53, 0x26, 0x90, 0xdb, 0x32, 0xb0, 0xbb,
+	0x6e, 0xd2, 0xdd, 0x4a, 0x8c, 0x6c, 0xe8, 0x53, 0xef, 0x66, 0x22, 0x87, 0x87, 0xfc, 0xf4, 0x3b,
+	0x12, 0xad, 0x4b, 0xbd, 0x9b, 0x1f, 0xa8, 0x4f, 0xce, 0xc5, 0x04, 0xf8, 0x1c, 0x76, 0x56, 0xc9,
+	0x9f, 0x05, 0x1e, 0xb9, 0xa9, 0x55, 0xb4, 0x1f, 0xd3, 0xc5, 0xd6, 0x66, 0xba, 0x64, 0x87, 0x80,
+	0xa8, 0x10, 0x28, 0xbf, 0x6e, 0x18, 0x70, 0x12, 0x70, 0x09, 0xd0, 0xc3, 0x03, 0x79, 0x22, 0x9c,
+	0x9f, 0x28, 0xb9, 0xfd, 0xbb, 0x01, 0xf7, 0x56, 0x48, 0xa7, 0x0e, 0x77, 0x6a, 0xb5, 0x9e, 0x05,
+	0x9d, 0x65, 0xf6, 0x77, 0xd4, 0x59, 0xb2, 0x17, 0x63, 0x51, 0x57, 0xaf, 0x21, 0x4f, 0xf4, 0xae,
+	0x6c, 0x00, 0x0a, 0x27, 0x01, 0x21, 0x9e, 0x9a, 0xae, 0xea, 0x1a, 0x3a, 0x4a, 0x70, 0xe6, 0xd9,
+	0x5f, 0xa7, 0x6b, 0xa3, 0x42, 0xd3, 0x39, 0xbe, 0x0f, 0xbd, 0x92, 0xec, 0xba, 0xd3, 0x54, 0x62,
+	0x9f, 0x01, 0x52, 0xc6, 0xcf, 0xc2, 0x38, 0xa8, 0x37, 0x53, 0xee, 0xc1, 0x30, 0x63, 0xa2, 0x1b,
+	0xfb, 0x09, 0x6c, 0x2b, 0xf1, 0x8b, 0x60, 0x5e, 0x1b, 0x6b, 0x37, 0x29, 0xeb, 0xd2, 0x48, 0xa3,
+	0x1d, 0x27, 0x4e, 0xb2, 0x04, 0x77, 0x2b, 0xd8, 0x4e, 0x12, 0x41, 0x96, 0xe3, 0x56, 0x01, 0xbf,
+	0xa0, 0x3f, 0x89, 0x79, 0xae, 0xb0, 0x56, 0xea, 0x89, 0x58, 0xab, 0xff, 0x0c, 0x70, 0x4a, 0xd9,
+	0x95, 0xfa, 0xc4, 0x44, 0xef, 0x7b, 0x34, 0xd2, 0x73, 0x4a, 0x2c, 0x85, 0xc4, 0xf1, 0x7d, 0x79,
+	0x9f, 0x4d, 0x2c, 0x96, 0xe2, 0xca, 0x62, 0x46, 0x3c, 0x79, 0x91, 0x4d, 0x2c, 0xd7, 0x42, 0x36,
+	0x8d, 0x88, 0xba, 0xc6, 0x26, 0x96, 0x6b, 0xfb, 0x0f, 0x03, 0xb6, 0x9e, 0x91, 0xb9, 0x46, 0xde,
+	0x03, 0x98, 0x85, 0x51, 0x18, 0x73, 0x1a, 0x10, 0x26, 0x1d, 0xb4, 0x70, 0x4a, 0xf2, 0xdf, 0xfd,
+	0xc8, 0x16, 0x22, 0xfe, 0x54, 0x76, 0x4a, 0x13, 0xcb, 0xb5, 0x90, 0x5d, 0x12, 0x67, 0xa1, 0x3f,
+	0x55, 0xb9, 0x16, 0x0c, 0xcc, 0xb8, 0xe3, 0x5e, 0xc9, 0x2f, 0xb3, 0x89, 0xd5, 0xe6, 0xf8, 0x2f,
+	0x80, 0x9e, 0x6e, 0x28, 0xf9, 0x04, 0x40, 0xaf, 0xa0, 0x9b, 0x7a, 0x3a, 0xa0, 0x0f, 0x8b, 0x2f,
+	0x84, 0xe2, 0x53, 0xc4, 0xfa, 0xa8, 0x42, 0x4b, 0x17, 0x7b, 0x03, 0x05, 0xf0, 0x76, 0x81, 0x9a,
+	0xd1, 0xa3, 0xa2, 0xf5, 0x3a, 0xe2, 0xb7, 0x1e, 0xd7, 0xd2, 0x5d, 0xfa, 0xe3, 0x30, 0x2c, 0xe1,
+	0x5a, 0x74, 0x58, 0x81, 0x92, 0xe1, 0x7b, 0xeb, 0xe3, 0x9a, 0xda, 0x4b, 0xaf, 0xaf, 0x01, 0x15,
+	0x89, 0x18, 0x3d, 0xae, 0x84, 0x59, 0x11, 0xbd, 0x75, 0x58, 0x4f, 0x79, 0x6d, 0xa2, 0x8a, 0xa2,
+	0x2b, 0x13, 0xcd, 0x3c, 0x02, 0x2a, 0x13, 0xcd, 0xf1, 0xfe, 0x06, 0xba, 0x82, 0x41, 0x9e, 0xbe,
+	0xd1, 0xc1, 0xba, 0x37, 0x65, 0xe1, 0x75, 0x60, 0x3d, 0xaa, 0xa3, 0xba, 0x74, 0x36, 0x81, 0x5e,
+	0x9a, 0x64, 0x51, 0x49, 0xd3, 0x95, 0x3c, 0x17, 0xac, 0xfd, 0x2a, 0xb5, 0x74, 0x36, 0x79, 0xd2,
+	0x2d, 0xcb, 0x66, 0x0d, 0xa3, 0x97, 0x65, 0xb3, 0x8e, 0xc3, 0xed, 0x0d, 0xf4, 0x2b, 0xdc, 0xcd,
+	0xb1, 0x15, 0x1a, 0xdf, 0x06, 0x90, 0xe6, 0x41, 0xeb, 0xa0, 0x86, 0x66, 0xe2, 0xe9, 0x53, 0x03,
+	0xcd, 0xe0, 0xad, 0x2c, 0x69, 0xa0, 0x87, 0xb7, 0x01, 0xa4, 0x18, 0xcf, 0x1a, 0x57, 0x2b, 0xa6,
+	0x1c, 0xbd, 0x82, 0x6e, 0x8a, 0x2d, 0xca, 0x86, 0x47, 0x91, 0x7f, 0xca, 0x86, 0x47, 0x19, 0xe5,
+	0x6c, 0xa0, 0x0b, 0xe8, 0x67, 0xf8, 0x03, 0xed, 0xaf, 0xb3, 0xcc, 0xb2, 0x92, 0xf5, 0xb0, 0x52,
+	0x2f, 0xdd, 0x64, 0x69, 0x5a, 0x41, 0x6b, 0x83, 0xcb, 0x0e, 0xc0, 0xfd, 0x2a, 0xb5, 0xc4, 0xc1,
+	0xc5, 0xa6, 0xfc, 0xc9, 0x7b, 0xf2, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xdb, 0x3c, 0x6d, 0xd7,
+	0xfb, 0x0d, 0x00, 0x00,
 }

weed/replication/replicator.go

@@ -34,7 +34,7 @@ func (r *Replicator) Replicate(key string, message *filer_pb.EventNotification)
 		glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
 		return nil
 	}
-	newKey := filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):])
+	newKey := filepath.ToSlash(filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):]))
 	glog.V(3).Infof("replicate %s => %s", key, newKey)
 	key = newKey
 	if message.OldEntry != nil && message.NewEntry == nil {

weed/replication/sink/filersink/fetch_write.go

@@ -9,6 +9,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -59,6 +60,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId stri
 	defer readCloser.Close()
 
 	var host string
+	var auth security.EncodedJwt
 
 	if err := fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
 
@@ -76,7 +78,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId stri
 			return err
 		}
 
-		fileId, host = resp.FileId, resp.Url
+		fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
 
 		return nil
 	}); err != nil {
@@ -88,7 +90,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId stri
 	glog.V(4).Infof("replicating %s to %s header:%+v", filename, fileUrl, header)
 
 	uploadResult, err := operation.Upload(fileUrl, filename, readCloser,
-		"gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, "")
+		"gzip" == header.Get("Content-Encoding"), header.Get("Content-Type"), nil, auth)
 	if err != nil {
 		glog.V(0).Infof("upload data %v to %s: %v", filename, fileUrl, err)
 		return "", fmt.Errorf("upload data: %v", err)
@@ -103,7 +105,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk) (fileId stri
 
 func (fs *FilerSink) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(fs.grpcAddress)
+	grpcConnection, err := util.GrpcDial(fs.grpcAddress, fs.grpcDialOption)
 	if err != nil {
 		return fmt.Errorf("fail to dial %s: %v", fs.grpcAddress, err)
 	}

weed/replication/sink/filersink/filer_sink.go

@@ -3,6 +3,9 @@ package filersink
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -13,13 +16,14 @@ import (
 )
 
 type FilerSink struct {
-	filerSource *source.FilerSource
-	grpcAddress string
-	dir         string
-	replication string
-	collection  string
-	ttlSec      int32
-	dataCenter  string
+	filerSource    *source.FilerSource
+	grpcAddress    string
+	dir            string
+	replication    string
+	collection     string
+	ttlSec         int32
+	dataCenter     string
+	grpcDialOption grpc.DialOption
 }
 
 func init() {
@@ -55,6 +59,7 @@ func (fs *FilerSink) initialize(grpcAddress string, dir string,
 	fs.replication = replication
 	fs.collection = collection
 	fs.ttlSec = int32(ttlSec)
+	fs.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 	return nil
 }
 
@@ -66,7 +71,6 @@ func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bo
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    dir,
 			Name:         name,
-			IsDirectory:  isDirectory,
 			IsDeleteData: deleteIncludeChunks,
 		}
 

weed/replication/source/filer_source.go

@@ -3,6 +3,9 @@ package source
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/spf13/viper"
+	"google.golang.org/grpc"
 	"io"
 	"net/http"
 	"strings"
@@ -17,8 +20,9 @@ type ReplicationSource interface {
 }
 
 type FilerSource struct {
-	grpcAddress string
-	Dir         string
+	grpcAddress    string
+	grpcDialOption grpc.DialOption
+	Dir            string
 }
 
 func (fs *FilerSource) Initialize(configuration util.Configuration) error {
@@ -31,6 +35,7 @@ func (fs *FilerSource) Initialize(configuration util.Configuration) error {
 func (fs *FilerSource) initialize(grpcAddress string, dir string) (err error) {
 	fs.grpcAddress = grpcAddress
 	fs.Dir = dir
+	fs.grpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 	return nil
 }
 
@@ -40,7 +45,7 @@ func (fs *FilerSource) LookupFileId(part string) (fileUrl string, err error) {
 
 	vid := volumeId(part)
 
-	err = fs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+	err = fs.withFilerClient(fs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		glog.V(4).Infof("read lookup volume id locations: %v", vid)
 		resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
@@ -84,9 +89,9 @@ func (fs *FilerSource) ReadPart(part string) (filename string, header http.Heade
 	return filename, header, readCloser, err
 }
 
-func (fs *FilerSource) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+func (fs *FilerSource) withFilerClient(grpcDialOption grpc.DialOption, fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(fs.grpcAddress)
+	grpcConnection, err := util.GrpcDial(fs.grpcAddress, grpcDialOption)
 	if err != nil {
 		return fmt.Errorf("fail to dial %s: %v", fs.grpcAddress, err)
 	}

weed/s3api/filer_multipart.go

@@ -1,6 +1,7 @@
 package s3api
 
 import (
+	"encoding/xml"
 	"fmt"
 	"path/filepath"
 	"strconv"
@@ -16,6 +17,7 @@ import (
 )
 
 type InitiateMultipartUploadResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult"`
 	s3.CreateMultipartUploadOutput
 }
 
@@ -34,7 +36,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
 	}
 
 	output = &InitiateMultipartUploadResult{
-		s3.CreateMultipartUploadOutput{
+		CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
 			Bucket:   input.Bucket,
 			Key:      input.Key,
 			UploadId: aws.String(uploadIdString),
@@ -45,6 +47,7 @@ func (s3a *S3ApiServer) createMultipartUpload(input *s3.CreateMultipartUploadInp
 }
 
 type CompleteMultipartUploadResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult"`
 	s3.CompleteMultipartUploadOutput
 }
 
@@ -95,7 +98,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 	}
 
 	output = &CompleteMultipartUploadResult{
-		s3.CompleteMultipartUploadOutput{
+		CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
 			Bucket: input.Bucket,
 			ETag:   aws.String("\"" + filer2.ETag(finalParts) + "\""),
 			Key:    input.Key,
@@ -128,13 +131,14 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
 }
 
 type ListMultipartUploadsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult"`
 	s3.ListMultipartUploadsOutput
 }
 
 func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput) (output *ListMultipartUploadsResult, code ErrorCode) {
 
 	output = &ListMultipartUploadsResult{
-		s3.ListMultipartUploadsOutput{
+		ListMultipartUploadsOutput: s3.ListMultipartUploadsOutput{
 			Bucket:       input.Bucket,
 			Delimiter:    input.Delimiter,
 			EncodingType: input.EncodingType,
@@ -164,12 +168,13 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInput
 }
 
 type ListPartsResult struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"`
 	s3.ListPartsOutput
 }
 
 func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPartsResult, code ErrorCode) {
 	output = &ListPartsResult{
-		s3.ListPartsOutput{
+		ListPartsOutput: s3.ListPartsOutput{
 			Bucket:           input.Bucket,
 			Key:              input.Key,
 			UploadId:         input.UploadId,

weed/s3api/filer_multipart_test.go

@@ -0,0 +1,26 @@
+package s3api
+
+import (
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"testing"
+)
+
+func TestInitiateMultipartUploadResult(t *testing.T) {
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Bucket>example-bucket</Bucket><Key>example-object</Key><UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId></InitiateMultipartUploadResult>`
+	response := &InitiateMultipartUploadResult{
+		CreateMultipartUploadOutput: s3.CreateMultipartUploadOutput{
+			Bucket:   aws.String("example-bucket"),
+			Key:      aws.String("example-object"),
+			UploadId: aws.String("VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"),
+		},
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+
+}

weed/s3api/filer_util.go

@@ -109,7 +109,6 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath string, entryName string, isDirec
 		request := &filer_pb.DeleteEntryRequest{
 			Directory:    parentDirectoryPath,
 			Name:         entryName,
-			IsDirectory:  isDirectory,
 			IsDeleteData: isDeleteData,
 			IsRecursive:  isRecursive,
 		}

weed/s3api/s3api_bucket_handlers.go

@@ -2,11 +2,15 @@ package s3api
 
 import (
 	"context"
+	"encoding/xml"
 	"fmt"
+	"math"
 	"net/http"
 	"os"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/gorilla/mux"
@@ -17,37 +21,39 @@ var (
 	OS_GID = uint32(os.Getgid())
 )
 
+type ListAllMyBucketsResult struct {
+	XMLName   xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult"`
+	Owner   *s3.Owner
+	Buckets []*s3.Bucket `xml:"Buckets>Bucket"`
+}
+
 func (s3a *S3ApiServer) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 
-	var response ListAllMyBucketsResponse
+	var response ListAllMyBucketsResult
 
-	entries, err := s3a.list(s3a.option.BucketsPath, "", "", false, 0)
+	entries, err := s3a.list(s3a.option.BucketsPath, "", "", false, math.MaxInt32)
 
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
 
-	var buckets []ListAllMyBucketsEntry
+	var buckets []*s3.Bucket
 	for _, entry := range entries {
 		if entry.IsDirectory {
-			buckets = append(buckets, ListAllMyBucketsEntry{
-				Name:         entry.Name,
-				CreationDate: time.Unix(entry.Attributes.Crtime, 0),
+			buckets = append(buckets, &s3.Bucket{
+				Name:         aws.String(entry.Name),
+				CreationDate: aws.Time(time.Unix(entry.Attributes.Crtime, 0)),
 			})
 		}
 	}
 
-	response = ListAllMyBucketsResponse{
-		ListAllMyBucketsResponse: ListAllMyBucketsResult{
-			Owner: CanonicalUser{
-				ID:          "",
-				DisplayName: "",
-			},
-			Buckets: ListAllMyBucketsList{
-				Bucket: buckets,
-			},
+	response = ListAllMyBucketsResult{
+		Owner: &s3.Owner{
+			ID:          aws.String(""),
+			DisplayName: aws.String(""),
 		},
+		Buckets: buckets,
 	}
 
 	writeSuccessResponseXML(w, encodeResponse(response))

weed/s3api/s3api_bucket_handlers_test.go

@@ -0,0 +1,39 @@
+package s3api
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func TestListBucketsHandler(t *testing.T) {
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><DisplayName></DisplayName><ID></ID></Owner><Buckets><Bucket><CreationDate>2011-04-09T12:34:49Z</CreationDate><Name>test1</Name></Bucket><Bucket><CreationDate>2011-02-09T12:34:49Z</CreationDate><Name>test2</Name></Bucket></Buckets></ListAllMyBucketsResult>`
+	var response ListAllMyBucketsResult
+
+	var buckets []*s3.Bucket
+	buckets = append(buckets, &s3.Bucket{
+		Name:         aws.String("test1"),
+		CreationDate: aws.Time(time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC)),
+	})
+	buckets = append(buckets, &s3.Bucket{
+		Name:         aws.String("test2"),
+		CreationDate: aws.Time(time.Date(2011, 2, 9, 12, 34, 49, 0, time.UTC)),
+	})
+
+	response = ListAllMyBucketsResult{
+		Owner: &s3.Owner{
+			ID:          aws.String(""),
+			DisplayName: aws.String(""),
+		},
+		Buckets: buckets,
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+}

weed/s3api/s3api_handlers.go

@@ -37,7 +37,7 @@ func encodeResponse(response interface{}) []byte {
 
 func (s3a *S3ApiServer) withFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	grpcConnection, err := util.GrpcDial(s3a.option.FilerGrpcAddress)
+	grpcConnection, err := util.GrpcDial(s3a.option.FilerGrpcAddress, s3a.option.GrpcDialOption)
 	if err != nil {
 		return fmt.Errorf("fail to dial %s: %v", s3a.option.FilerGrpcAddress, err)
 	}
@@ -77,6 +77,7 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	}
 	w.WriteHeader(statusCode)
 	if response != nil {
+		glog.V(4).Infof("status %d %s: %s", statusCode, mType, string(response))
 		w.Write(response)
 		w.(http.Flusher).Flush()
 	}

weed/s3api/s3api_object_handlers.go

@@ -73,7 +73,7 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
 	destUrl := fmt.Sprintf("http://%s%s/%s%s",
 		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
 
-	s3a.proxyToFiler(w, r, destUrl, passThroghResponse)
+	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
 
 }
 
@@ -86,7 +86,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
 	destUrl := fmt.Sprintf("http://%s%s/%s%s",
 		s3a.option.Filer, s3a.option.BucketsPath, bucket, object)
 
-	s3a.proxyToFiler(w, r, destUrl, passThroghResponse)
+	s3a.proxyToFiler(w, r, destUrl, passThroughResponse)
 
 }
 
@@ -147,7 +147,7 @@ func (s3a *S3ApiServer) proxyToFiler(w http.ResponseWriter, r *http.Request, des
 
 	responseFn(resp, w)
 }
-func passThroghResponse(proxyResonse *http.Response, w http.ResponseWriter) {
+func passThroughResponse(proxyResonse *http.Response, w http.ResponseWriter) {
 	for k, v := range proxyResonse.Header {
 		w.Header()[k] = v
 	}

weed/s3api/s3api_objects_list_handlers.go

@@ -9,8 +9,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -85,7 +83,7 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response *s3.ListObjectsOutput, err error) {
+func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys int, marker string) (response ListBucketResult, err error) {
 
 	// convert full path prefix into directory name and prefix for entry name
 	dir, prefix := filepath.Split(originalPrefix)
@@ -106,8 +104,8 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
 			return fmt.Errorf("list buckets: %v", err)
 		}
 
-		var contents []*s3.Object
-		var commonPrefixes []*s3.CommonPrefix
+		var contents []ListEntry
+		var commonPrefixes []PrefixEntry
 		var counter int
 		var lastEntryName string
 		var isTruncated bool
@@ -119,32 +117,32 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
 			}
 			lastEntryName = entry.Name
 			if entry.IsDirectory {
-				commonPrefixes = append(commonPrefixes, &s3.CommonPrefix{
-					Prefix: aws.String(fmt.Sprintf("%s%s/", dir, entry.Name)),
+				commonPrefixes = append(commonPrefixes, PrefixEntry{
+					Prefix: fmt.Sprintf("%s%s/", dir, entry.Name),
 				})
 			} else {
-				contents = append(contents, &s3.Object{
-					Key:          aws.String(fmt.Sprintf("%s%s", dir, entry.Name)),
-					LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
-					ETag:         aws.String("\"" + filer2.ETag(entry.Chunks) + "\""),
-					Size:         aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
-					Owner: &s3.Owner{
-						ID:          aws.String("bcaf161ca5fb16fd081034f"),
-						DisplayName: aws.String("webfile"),
+				contents = append(contents, ListEntry{
+					Key:          fmt.Sprintf("%s%s", dir, entry.Name),
+					LastModified: time.Unix(entry.Attributes.Mtime, 0),
+					ETag:         "\"" + filer2.ETag(entry.Chunks) + "\"",
+					Size:         int64(filer2.TotalSize(entry.Chunks)),
+					Owner: CanonicalUser{
+						ID:          "bcaf161ca5fb16fd081034f",
+						DisplayName: "webfile",
 					},
-					StorageClass: aws.String("STANDARD"),
+					StorageClass: "STANDARD",
 				})
 			}
 		}
 
-		response = &s3.ListObjectsOutput{
-			Name:           aws.String(bucket),
-			Prefix:         aws.String(originalPrefix),
-			Marker:         aws.String(marker),
-			NextMarker:     aws.String(lastEntryName),
-			MaxKeys:        aws.Int64(int64(maxKeys)),
-			Delimiter:      aws.String("/"),
-			IsTruncated:    aws.Bool(isTruncated),
+		response = ListBucketResult{
+			Name:           bucket,
+			Prefix:         originalPrefix,
+			Marker:         marker,
+			NextMarker:     lastEntryName,
+			MaxKeys:        maxKeys,
+			Delimiter:      "/",
+			IsTruncated:    isTruncated,
 			Contents:       contents,
 			CommonPrefixes: commonPrefixes,
 		}

weed/s3api/s3api_objects_list_handlers_test.go

@@ -0,0 +1,38 @@
+package s3api
+
+import (
+	"testing"
+	"time"
+)
+
+func TestListObjectsHandler(t *testing.T) {
+
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html
+
+	expected := `<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Name>test_container</Name><Prefix></Prefix><Marker></Marker><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated><Contents><Key>1.zip</Key><ETag>&#34;4397da7a7649e8085de9916c240e8166&#34;</ETag><Size>1234567</Size><Owner><ID>65a011niqo39cdf8ec533ec3d1ccaafsa932</ID></Owner><StorageClass>STANDARD</StorageClass><LastModified>2011-04-09T12:34:49</LastModified></Contents></ListBucketResult>`
+
+	response := ListBucketResult{
+		Name:        "test_container",
+		Prefix:      "",
+		Marker:      "",
+		NextMarker:  "",
+		MaxKeys:     1000,
+		IsTruncated: false,
+		Contents: []ListEntry{{
+			Key:          "1.zip",
+			LastModified: time.Date(2011, 4, 9, 12, 34, 49, 0, time.UTC),
+			ETag:         "\"4397da7a7649e8085de9916c240e8166\"",
+			Size:         1234567,
+			Owner: CanonicalUser{
+				ID: "65a011niqo39cdf8ec533ec3d1ccaafsa932",
+			},
+			StorageClass: "STANDARD",
+		}},
+	}
+
+	encoded := string(encodeResponse(response))
+	if encoded != expected {
+		t.Errorf("unexpected output: %s\nexpecting:%s", encoded, expected)
+	}
+}

weed/s3api/s3api_server.go

@@ -8,6 +8,7 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/postgres"
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/redis"
 	"github.com/gorilla/mux"
+	"google.golang.org/grpc"
 	"net/http"
 )
 
@@ -16,6 +17,7 @@ type S3ApiServerOption struct {
 	FilerGrpcAddress string
 	DomainName       string
 	BucketsPath      string
+	GrpcDialOption   grpc.DialOption
 }
 
 type S3ApiServer struct {

weed/s3api/s3api_xsd_generated.go

@@ -25,8 +25,8 @@ type BucketLoggingStatus struct {
 }
 
 type CanonicalUser struct {
-	ID          string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ID"`
-	DisplayName string `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DisplayName,omitempty"`
+	ID          string `xml:"ID"`
+	DisplayName string `xml:"DisplayName,omitempty"`
 }
 
 type CopyObject struct {
@@ -506,15 +506,15 @@ func (t *ListAllMyBuckets) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
 }
 
 type ListAllMyBucketsEntry struct {
-	Name         string    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
-	CreationDate time.Time `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+	Name         string    `xml:"Name"`
+	CreationDate time.Time `xml:"CreationDate"`
 }
 
 func (t *ListAllMyBucketsEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type T ListAllMyBucketsEntry
 	var layout struct {
 		*T
-		CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+		CreationDate *xsdDateTime `xml:"CreationDate"`
 	}
 	layout.T = (*T)(t)
 	layout.CreationDate = (*xsdDateTime)(&layout.T.CreationDate)
@@ -524,7 +524,7 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
 	type T ListAllMyBucketsEntry
 	var overlay struct {
 		*T
-		CreationDate *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreationDate"`
+		CreationDate *xsdDateTime `xml:"CreationDate"`
 	}
 	overlay.T = (*T)(t)
 	overlay.CreationDate = (*xsdDateTime)(&overlay.T.CreationDate)
@@ -532,18 +532,13 @@ func (t *ListAllMyBucketsEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElem
 }
 
 type ListAllMyBucketsList struct {
-	Bucket []ListAllMyBucketsEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket,omitempty"`
+	Bucket []ListAllMyBucketsEntry `xml:"Bucket,omitempty"`
 }
 
 type ListAllMyBucketsResponse struct {
 	ListAllMyBucketsResponse ListAllMyBucketsResult `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResponse"`
 }
 
-type ListAllMyBucketsResult struct {
-	Owner   CanonicalUser        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner"`
-	Buckets ListAllMyBucketsList `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Buckets"`
-}
-
 type ListBucket struct {
 	Bucket         string    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Bucket"`
 	Prefix         string    `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix,omitempty"`
@@ -582,32 +577,33 @@ type ListBucketResponse struct {
 }
 
 type ListBucketResult struct {
-	Metadata       []MetadataEntry `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Metadata,omitempty"`
-	Name           string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Name"`
-	Prefix         string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Prefix"`
-	Marker         string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Marker"`
-	NextMarker     string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ NextMarker,omitempty"`
-	MaxKeys        int             `xml:"http://s3.amazonaws.com/doc/2006-03-01/ MaxKeys"`
-	Delimiter      string          `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Delimiter,omitempty"`
-	IsTruncated    bool            `xml:"http://s3.amazonaws.com/doc/2006-03-01/ IsTruncated"`
-	Contents       []ListEntry     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Contents,omitempty"`
-	CommonPrefixes []PrefixEntry   `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CommonPrefixes,omitempty"`
+	XMLName        xml.Name        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
+	Metadata       []MetadataEntry `xml:"Metadata,omitempty"`
+	Name           string          `xml:"Name"`
+	Prefix         string          `xml:"Prefix"`
+	Marker         string          `xml:"Marker"`
+	NextMarker     string          `xml:"NextMarker,omitempty"`
+	MaxKeys        int             `xml:"MaxKeys"`
+	Delimiter      string          `xml:"Delimiter,omitempty"`
+	IsTruncated    bool            `xml:"IsTruncated"`
+	Contents       []ListEntry     `xml:"Contents,omitempty"`
+	CommonPrefixes []PrefixEntry   `xml:"CommonPrefixes,omitempty"`
 }
 
 type ListEntry struct {
-	Key          string        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Key"`
-	LastModified time.Time     `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
-	ETag         string        `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ETag"`
-	Size         int64         `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Size"`
-	Owner        CanonicalUser `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Owner,omitempty"`
-	StorageClass StorageClass  `xml:"http://s3.amazonaws.com/doc/2006-03-01/ StorageClass"`
+	Key          string        `xml:"Key"`
+	LastModified time.Time     `xml:"LastModified"`
+	ETag         string        `xml:"ETag"`
+	Size         int64         `xml:"Size"`
+	Owner        CanonicalUser `xml:"Owner,omitempty"`
+	StorageClass StorageClass  `xml:"StorageClass"`
 }
 
 func (t *ListEntry) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	type T ListEntry
 	var layout struct {
 		*T
-		LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
+		LastModified *xsdDateTime `xml:"LastModified"`
 	}
 	layout.T = (*T)(t)
 	layout.LastModified = (*xsdDateTime)(&layout.T.LastModified)
@@ -617,7 +613,7 @@ func (t *ListEntry) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	type T ListEntry
 	var overlay struct {
 		*T
-		LastModified *xsdDateTime `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LastModified"`
+		LastModified *xsdDateTime `xml:"LastModified"`
 	}
 	overlay.T = (*T)(t)
 	overlay.LastModified = (*xsdDateTime)(&overlay.T.LastModified)

weed/security/guard.go

@@ -41,21 +41,21 @@ https://github.com/pkieltyka/jwtauth/blob/master/jwtauth.go
 
 */
 type Guard struct {
-	whiteList []string
-	SecretKey Secret
+	whiteList  []string
+	SigningKey SigningKey
 
 	isActive bool
 }
 
-func NewGuard(whiteList []string, secretKey string) *Guard {
-	g := &Guard{whiteList: whiteList, SecretKey: Secret(secretKey)}
-	g.isActive = len(g.whiteList) != 0 || len(g.SecretKey) != 0
+func NewGuard(whiteList []string, signingKey string) *Guard {
+	g := &Guard{whiteList: whiteList, SigningKey: SigningKey(signingKey)}
+	g.isActive = len(g.whiteList) != 0 || len(g.SigningKey) != 0
 	return g
 }
 
 func (g *Guard) WhiteList(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
 	if !g.isActive {
-		//if no security needed, just skip all checkings
+		//if no security needed, just skip all checking
 		return f
 	}
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -67,20 +67,6 @@ func (g *Guard) WhiteList(f func(w http.ResponseWriter, r *http.Request)) func(w
 	}
 }
 
-func (g *Guard) Secure(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
-	if !g.isActive {
-		//if no security needed, just skip all checkings
-		return f
-	}
-	return func(w http.ResponseWriter, r *http.Request) {
-		if err := g.checkJwt(w, r); err != nil {
-			w.WriteHeader(http.StatusUnauthorized)
-			return
-		}
-		f(w, r)
-	}
-}
-
 func GetActualRemoteHost(r *http.Request) (host string, err error) {
 	host = r.Header.Get("HTTP_X_FORWARDED_FOR")
 	if host == "" {
@@ -130,33 +116,3 @@ func (g *Guard) checkWhiteList(w http.ResponseWriter, r *http.Request) error {
 	glog.V(0).Infof("Not in whitelist: %s", r.RemoteAddr)
 	return fmt.Errorf("Not in whitelis: %s", r.RemoteAddr)
 }
-
-func (g *Guard) checkJwt(w http.ResponseWriter, r *http.Request) error {
-	if g.checkWhiteList(w, r) == nil {
-		return nil
-	}
-
-	if len(g.SecretKey) == 0 {
-		return nil
-	}
-
-	tokenStr := GetJwt(r)
-
-	if tokenStr == "" {
-		return ErrUnauthorized
-	}
-
-	// Verify the token
-	token, err := DecodeJwt(g.SecretKey, tokenStr)
-	if err != nil {
-		glog.V(1).Infof("Token verification error from %s: %v", r.RemoteAddr, err)
-		return ErrUnauthorized
-	}
-	if !token.Valid {
-		glog.V(1).Infof("Token invliad from %s: %v", r.RemoteAddr, tokenStr)
-		return ErrUnauthorized
-	}
-
-	glog.V(1).Infof("No permission from %s", r.RemoteAddr)
-	return fmt.Errorf("No write permisson from %s", r.RemoteAddr)
-}